Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add openfda drug applications, shortages, device classification and udi tables (#123361)

Tom Owers298362b402df
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index d20ff919bc2e..850befe461a9 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -827,7 +827,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/openaq/openaq.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/openaq/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/opinion_stage/opinion_stage.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/opinion_stage/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 7fe7ff6b1d98..e3703660e98c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8808,10 +8808,10 @@ Today (9): `device_510k`, `device_enforcement`, `device_events`, `drug_enforceme
Diffed against: <https://api.fda.gov/download.json>
-- [ ] `device/classification` — lookup resolving device product codes, classes and regulation numbers referenced by 510k, UDI and device events (high)
-- [ ] `drug/drugsfda` — drug approvals and applications - the core regulatory fact table, absent entirely (high)
-- [ ] `device/udi` — device identifier registry, the lookup that resolves devices named in adverse events and recalls (high)
-- [ ] `drug/shortages` — current and resolved drug shortage records, a headline supply-chain signal (medium)
+- [x] `device/classification` — lookup resolving device product codes, classes and regulation numbers referenced by 510k, UDI and device events (high)
+- [x] `drug/drugsfda` — drug approvals and applications - the core regulatory fact table, absent entirely (high)
+- [x] `device/udi` — device identifier registry, the lookup that resolves devices named in adverse events and recalls (high)
+- [x] `drug/shortages` — current and resolved drug shortage records, a headline supply-chain signal (medium)
- [ ] `device/recall` — recall records distinct from the enforcement reports already synced (medium)
- [ ] `device/pma` — premarket approval submissions, the higher-risk counterpart to 510k (medium)
- [ ] `device/registrationlisting` — establishment registrations, the manufacturer lookup behind device records (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/canonical_descriptions.py
index c2c8b80abd7a..72282e303278 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/canonical_descriptions.py
@@ -44,6 +44,35 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"product_type": "Product type (e.g. HUMAN PRESCRIPTION DRUG, HUMAN OTC DRUG).",
},
},
+ "drug_applications": {
+ "description": "Drugs@FDA: FDA-approved brand-name and generic drug products and their applications (NDA, ANDA, BLA). One row per application.",
+ "docs_url": "https://open.fda.gov/apis/drug/drugsfda/",
+ "columns": {
+ "application_number": "NDA, ANDA, or BLA number of the application.",
+ "sponsor_name": "Company that submitted the application to the FDA to market the drug in the U.S.",
+ "submissions": "Nested list of submissions (original approval, supplements) with their type, status, and status date.",
+ "products": "Nested list of the drug products covered by the application (brand name, dosage form, active ingredients, marketing status).",
+ "openfda": "Nested object of harmonized identifiers (NDC, brand name, generic name, RxCUI, etc.).",
+ },
+ },
+ "drug_shortages": {
+ "description": "Current and resolved drug shortages reported to the FDA. One row per shortage record for a drug presentation; a package NDC can appear more than once.",
+ "docs_url": "https://open.fda.gov/apis/drug/drugshortages/",
+ "columns": {
+ "package_ndc": "National Drug Code identifying the labeler, product, and package size.",
+ "generic_name": "Generic name(s) of the drug product.",
+ "company_name": "Company that makes the drug product.",
+ "presentation": "Drug name, form, strength, and NDC number.",
+ "status": "Shortage status of the presentation (e.g. Current, Resolved, To Be Discontinued).",
+ "availability": "Availability information for the presentation.",
+ "shortage_reason": "Reason for the shortage of the presentation.",
+ "initial_posting_date": "Date the presentation was added to the FDA shortage list (MM/DD/YYYY).",
+ "update_date": "Date the presentation was last updated (MM/DD/YYYY).",
+ "update_type": "Type of the last update (e.g. New, Revised, Reverified).",
+ "discontinued_date": "Date the presentation was discontinued (MM/DD/YYYY).",
+ "therapeutic_category": "List of therapeutic categories for the drug.",
+ },
+ },
"drug_enforcement": {
"description": "Drug recall enforcement reports — recalls of drug products classified by the FDA. One row per recall.",
"docs_url": "https://open.fda.gov/apis/drug/enforcement/",
@@ -82,6 +111,40 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"product_code": "FDA product classification code for the device.",
},
},
+ "device_classification": {
+ "description": "Medical device classification: every FDA device product code with its device class and classification regulation. One row per product code.",
+ "docs_url": "https://open.fda.gov/apis/device/classification/",
+ "columns": {
+ "product_code": "Three-letter identifier assigned to a device category.",
+ "device_name": "Name of the device category.",
+ "device_class": "Risk-based device class (1, 2, or 3; U = unclassified, N = not classified, F = HDE).",
+ "regulation_number": "Classification regulation in 21 CFR under which the device is classified.",
+ "medical_specialty": "Two-letter code of the medical specialty panel for the classification regulation.",
+ "medical_specialty_description": "Readable name of the medical specialty panel.",
+ "review_panel": "510(k) review panel that would review the device.",
+ "submission_type_id": "Submission type the product code is limited to (510(k), PMA, 510(k) exempt, etc.).",
+ "implant_flag": "Whether the device is an implant (Y/N).",
+ "life_sustain_support_flag": "Whether the device is life-sustaining or life-supporting (Y/N).",
+ },
+ },
+ "device_udi": {
+ "description": "Global Unique Device Identification Database (GUDID): device identifier records submitted by device labelers. One row per device identifier record.",
+ "docs_url": "https://open.fda.gov/apis/device/udi/",
+ "columns": {
+ "public_device_record_key": "Unique identifier for the device identifier record.",
+ "publish_date": "Date the record was published to public search (YYYY-MM-DD).",
+ "public_version_date": "Date the current version of the record was published (YYYY-MM-DD).",
+ "brand_name": "Proprietary, trade, or brand name of the device as used on its labeling.",
+ "version_or_model_number": "Version or model shown on the device label or packaging.",
+ "company_name": "Labeler company name associated with the record.",
+ "device_description": "Additional information about the device.",
+ "commercial_distribution_status": "Whether the device is in commercial distribution.",
+ "record_status": "Status of the device identifier record.",
+ "identifiers": "Nested list of device identifiers (primary DI, package DIs) and their issuing agencies.",
+ "product_codes": "Nested list of FDA product codes for the device; joins to the device classification table.",
+ "gmdn_terms": "Nested list of Global Medical Device Nomenclature terms describing the device.",
+ },
+ },
"device_enforcement": {
"description": "Medical device recall enforcement reports. One row per recall.",
"docs_url": "https://open.fda.gov/apis/device/enforcement/",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py
index 80ae038ea020..c66090cdb51f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/openfda.py
@@ -1,8 +1,8 @@
"""openFDA (U.S. Food and Drug Administration) REST API transport.
openFDA (https://api.fda.gov) is the FDA's free public API over drug, device, and food regulatory
-datasets (adverse events, recalls/enforcement reports, drug labeling, 510(k) clearances, the NDC
-directory). Every dataset is its own endpoint with its own schema and date field; responses wrap the
+datasets (adverse events, recalls/enforcement reports, drug labeling, Drugs@FDA applications, drug
+shortages, 510(k) clearances, device classification, the UDI registry, the NDC directory). Every dataset is its own endpoint with its own schema and date field; responses wrap the
records in `{"meta": ..., "results": [...]}`.
Pagination is a `search_after` cursor exposed via the HTTP `Link: rel="next"` header (the `skip`
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.py
index 9017da32bc41..8eca45997d22 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/settings.py
@@ -1,15 +1,17 @@
-from dataclasses import dataclass
from typing import Optional
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@frozen
class OpenFDAEndpointConfig:
name: str
# Path under https://api.fda.gov, e.g. "/drug/enforcement.json".
path: str
- primary_keys: list[str]
+ # None when the dataset has no unique record key; such endpoints must stay full refresh only.
+ primary_keys: Optional[list[str]]
# Server-side date field used both for incremental filtering (search=<field>:[low TO high]) and for
# ascending sort. None means the endpoint has no reliable date cursor, so it ships full refresh only.
incremental_field: Optional[str] = None
@@ -33,9 +35,9 @@ class OpenFDAEndpointConfig:
# openFDA exposes one endpoint per dataset, each with its own schema and date field. We surface the
# datasets a user is most likely to want across the drug, device, and food domains. Adverse-event and
-# enforcement (recall) endpoints carry a genuine server-side date filter, so they sync incrementally;
-# the NDC directory and drug labeling endpoints have no reliable row-level date cursor, so they ship
-# full refresh only.
+# enforcement (recall) endpoints and the UDI registry carry a genuine server-side date filter, so they
+# sync incrementally; the reference datasets (NDC directory, drug labeling, Drugs@FDA, device
+# classification, drug shortages) have no reliable row-level date cursor, so they ship full refresh only.
OPENFDA_ENDPOINTS: dict[str, OpenFDAEndpointConfig] = {
"drug_events": OpenFDAEndpointConfig(
name="drug_events",
@@ -62,6 +64,25 @@ OPENFDA_ENDPOINTS: dict[str, OpenFDAEndpointConfig] = {
incremental_field=None,
partition_key=None,
),
+ "drug_applications": OpenFDAEndpointConfig(
+ name="drug_applications",
+ path="/drug/drugsfda.json",
+ primary_keys=["application_number"],
+ # Submission dates live inside the nested `submissions` list, so there's no top-level date
+ # to filter on — full refresh only.
+ incremental_field=None,
+ partition_key=None,
+ ),
+ "drug_shortages": OpenFDAEndpointConfig(
+ name="drug_shortages",
+ path="/drug/shortages.json",
+ # One NDC can carry several shortage records, and the dataset contains exact duplicate rows,
+ # so no field combination is unique.
+ primary_keys=None,
+ # Dates arrive as MM/DD/YYYY, which the YYYYMMDD search filter can't match — full refresh only.
+ incremental_field=None,
+ partition_key=None,
+ ),
"drug_enforcement": OpenFDAEndpointConfig(
name="drug_enforcement",
path="/drug/enforcement.json",
@@ -83,6 +104,25 @@ OPENFDA_ENDPOINTS: dict[str, OpenFDAEndpointConfig] = {
incremental_field="decision_date",
partition_key="decision_date",
),
+ "device_classification": OpenFDAEndpointConfig(
+ name="device_classification",
+ path="/device/classification.json",
+ primary_keys=["product_code"],
+ # A small reference list with no per-record date — full refresh only.
+ incremental_field=None,
+ partition_key=None,
+ ),
+ "device_udi": OpenFDAEndpointConfig(
+ name="device_udi",
+ path="/device/udi.json",
+ primary_keys=["public_device_record_key"],
+ # `public_version_date` moves forward whenever a record is republished, so it picks up edits;
+ # `publish_date` is fixed at first publication, so it partitions.
+ incremental_field="public_version_date",
+ partition_key="publish_date",
+ # Millions of rows — opt-in so a new source doesn't spend the unauthenticated daily quota on it.
+ should_sync_default=False,
+ ),
"device_enforcement": OpenFDAEndpointConfig(
name="device_enforcement",
path="/device/enforcement.json",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/source.py
index e060ecab15bf..1e1b0b783572 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/source.py
@@ -50,7 +50,7 @@ class OpenFDASource(ResumableSource[OpenFDASourceConfig, OpenFDAResumeConfig]):
category=DataWarehouseSourceCategory.ANALYTICS,
label="openFDA",
releaseStatus=ReleaseStatus.BETA,
- caption="""Pull U.S. FDA drug, device, and food data — adverse event reports, recalls, drug labeling, 510(k) clearances, and the NDC directory — into the PostHog Data warehouse.
+ caption="""Pull U.S. FDA drug, device, and food data — adverse event reports, recalls, drug labeling, drug applications and shortages, 510(k) clearances, device classifications, the UDI device registry, and the NDC directory — into the PostHog Data warehouse.
An API key is optional but recommended. Without one, openFDA limits you to 1,000 requests/day per IP; with one, 120,000 requests/day. Get a free key from the [openFDA API basics page](https://open.fda.gov/apis/authentication/).""",
iconPath="/static/services/openfda.png",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda.py
index 95714ee86fca..f9a87973e1eb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda.py
@@ -144,10 +144,13 @@ class TestOpenfdaSource:
("drug_enforcement", ["recall_number"], "report_date"),
("drug_ndc", ["product_id"], None),
("device_510k", ["k_number"], "decision_date"),
+ # Incremental on `public_version_date` but partitioned on the fixed `publish_date`.
+ ("device_udi", ["public_device_record_key"], "publish_date"),
+ ("drug_shortages", None, None),
]
)
def test_source_response_carries_endpoint_config(
- self, endpoint: str, primary_keys: list[str], partition_key: str | None
+ self, endpoint: str, primary_keys: list[str] | None, partition_key: str | None
) -> None:
result = _driver().run(endpoint, [_page([])])
assert result.raised is None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda_source.py
index c9672156ed16..0a9c699e3ee0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openfda/tests/test_openfda_source.py
@@ -45,10 +45,13 @@ class TestGetSchemas:
# sync silently re-scans the whole dataset.
("drug_labels", False, ["id"]),
("drug_ndc", False, ["product_id"]),
+ ("device_udi", True, ["public_device_record_key"]),
+ # No unique key, so merge-based incremental or append would duplicate rows.
+ ("drug_shortages", False, None),
]
)
def test_incremental_support_matches_endpoint(
- self, endpoint: str, expect_incremental: bool, primary_keys: list[str]
+ self, endpoint: str, expect_incremental: bool, primary_keys: list[str] | None
) -> None:
schema = {s.name: s for s in OpenFDASource().get_schemas(_config(), team_id=1)}[endpoint]
assert schema.supports_incremental is expect_incremental