:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add ramp billing statements table (#126971)
Tom Owers185047234959
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 02a76d3117a1..679b87bae903 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10034,7 +10034,7 @@ Note: No published SDL file; I ran an unauthenticated GraphQL introspection quer## Ramp — **thin**-Today (5): `cards`, `departments`, `reimbursements`, `transactions`, `users`+Today (6): `billing_statements`, `cards`, `departments`, `reimbursements`, `transactions`, `users`Diffed against: <https://docs.ramp.com/openapi/developer-api.json>@@ -10050,6 +10050,7 @@ Diffed against: <https://docs.ramp.com/openapi/developer-api.json>- [ ] `accounting/fields and accounting/field-options` — Custom accounting dimensions and their allowed values - the breakdown dimensions for coded spend (medium)- [ ] `receipts and item-receipts` — Receipt records and line-item detail attached to transactions, for compliance reporting (medium)- [ ] `spend-programs / unified-requests` — Spend limits and the approval-request stream that explains why spend was allowed or blocked (medium)+- [x] `billing/statements` — Ramp billing statements per statement period, with the amount due and statement PDF link (medium)Note: Ramp publishes a full OpenAPI spec (2.1 MB) plus llms-api.txt; the docs HTML explicitly points machine readers at /openapi/developer-api.json. The spec has ~95 GET paths across ~40 resources; PostHog exposes 5. Also note PostHog syncs the legacy /cards path, which no longer appears in the current spec (now /cards/physical and /cards/virtual) - worth confirming it still returns data.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.pyindex da2f389181f9..e825afcd5f16 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py@@ -88,4 +88,14 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"name": "The department's name.",},},+ "billing_statements": {+ "description": "A billing statement for one statement period, with the total amount due.",+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/billing-statements",+ "columns": {+ "id": "Unique identifier of the billing statement.",+ "amount": "The total amount due for this billing statement.",+ "statement_period": "The billing statement period date range (start and end dates; end is null while the period is open).",+ "statement_url": "Presigned URL to download the billing statement PDF, or null if unavailable.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.pyindex c22f6ead8744..7155c83aeb0d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py@@ -54,7 +54,7 @@ def _token_url(environment: str) -> str:return f"{_api_base_url(environment)}/token"-def _make_auth(environment: str, client_id: str, client_secret: str) -> OAuth2Auth:+def _make_auth(environment: str, client_id: str, client_secret: str, scopes: str = TOKEN_SCOPES) -> OAuth2Auth:"""Ramp uses OAuth2 client-credentials with HTTP Basic client auth.Tokens last ~10 days; the framework mints one lazily, caches it for the run, and re-mints@@ -64,7 +64,7 @@ def _make_auth(environment: str, client_id: str, client_secret: str) -> OAuth2Auclient_id=client_id,client_secret=client_secret,grant_type="client_credentials",- scopes=TOKEN_SCOPES,+ scopes=scopes,client_auth_method="basic",)@@ -121,6 +121,26 @@ def validate_credentials(environment: str, client_id: str, client_secret: str) -return True, None+def check_endpoint_access(environment: str, client_id: str, client_secret: str, endpoint: str) -> str | None:+ """Report why the app can't sync an endpoint that mints its own scopes, or None when it can.++ Validation only mints TOKEN_SCOPES, so an app created without an endpoint's extra scope still+ connects; probe that scope here so the table starts deselected instead of failing every sync."""+ scopes = RAMP_ENDPOINTS[endpoint].scopes+ if scopes is None:+ return None+ auth = _make_auth(environment, client_id, client_secret, scopes)+ probe = Request(method="GET", url=_api_base_url(environment)).prepare()+ try:+ auth(probe)+ except OAuth2AuthRequestError as e:+ if e.is_permanent:+ return f"Your Ramp developer app needs the `{scopes}` scope to sync this table."+ except requests.RequestException:+ pass+ return None++def ramp_source(environment: str,client_id: str,@@ -133,7 +153,7 @@ def ramp_source(db_incremental_field_last_value: Optional[Any] = None,) -> SourceResponse:config = RAMP_ENDPOINTS[endpoint]- auth = _make_auth(environment, client_id, client_secret)+ auth = _make_auth(environment, client_id, client_secret, config.scopes or TOKEN_SCOPES)resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.pyindex c1cf8e4ed660..c4c337f51b69 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py@@ -8,7 +8,7 @@ from products.warehouse_sources.backend.types import IncrementalField, IncrementTOKEN_SCOPES = "transactions:read reimbursements:read users:read cards:read departments:read"-@dataclass+@dataclass(frozen=True)class RampEndpointConfig:name: str# Path under {host}/developer/v1.@@ -19,6 +19,9 @@ class RampEndpointConfig:incremental_param: Optional[str] = None# Stable event-time field used for datetime partitioning.partition_key: Optional[str] = None+ # Scopes minted for this endpoint instead of TOKEN_SCOPES. Ramp rejects a token request for a+ # scope the app lacks, so a newer endpoint asks only for its own scope and existing apps keep working.+ scopes: Optional[str] = NoneRAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {@@ -55,6 +58,11 @@ RAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {name="departments",path="/departments",),+ "billing_statements": RampEndpointConfig(+ name="billing_statements",+ path="/billing/statements",+ scopes="billing_statements:read",+ ),}ENDPOINTS = tuple(RAMP_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.pyindex 934ae4c65288..62ab5bdb8e7d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.ramp import RampSourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.ramp.ramp import (RampResumeConfig,+ check_endpoint_access,ramp_source,validate_credentials as validate_ramp_credentials,)@@ -78,7 +79,7 @@ class RampSource(ResumableSource[RampSourceConfig, RampResumeConfig]):label="Ramp",caption="""Connect your Ramp account to pull your spend data into the PostHog Data warehouse.-A Ramp admin can create a developer app under Settings > Developer API. Grant it the `transactions:read`, `reimbursements:read`, `users:read`, `cards:read`, and `departments:read` scopes (scopes are fixed at app creation) and enable client credentials, then enter the app's client ID and secret here.""",+A Ramp admin can create a developer app under Settings > Developer API. Grant it the `transactions:read`, `reimbursements:read`, `users:read`, `cards:read`, and `departments:read` scopes, plus `billing_statements:read` to sync billing statements (scopes are fixed at app creation) and enable client credentials, then enter the app's client ID and secret here.""",iconPath="/static/services/ramp.png",docsUrl="https://posthog.com/docs/cdp/sources/ramp",releaseStatus=ReleaseStatus.GA,@@ -131,6 +132,14 @@ A Ramp admin can create a developer app under Settings > Developer API. Grant it) -> tuple[bool, str | None]:return validate_ramp_credentials(config.environment, config.client_id, config.client_secret)+ def get_endpoint_permissions(+ self, config: RampSourceConfig, team_id: int, endpoints: list[str], api_version: str | None = None+ ) -> dict[str, str | None]:+ return {+ endpoint: check_endpoint_access(config.environment, config.client_id, config.client_secret, endpoint)+ for endpoint in endpoints+ }+def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[RampResumeConfig]:return ResumableSourceManager[RampResumeConfig](inputs, RampResumeConfig)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.pyindex d15980c067e4..0ea44b102f08 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py@@ -18,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.rampRampResumeConfig,_base_url,_format_timestamp,+ check_endpoint_access,validate_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.settings import TOKEN_SCOPES@@ -39,8 +40,8 @@ def _token(status_code: int = 200, expires_in: int = 864000) -> ScriptedResponsereturn ScriptedResponse(status=status_code, json=body)-def _source(endpoint: str, pages: list[ScriptedResponse], **inputs: Any) -> DriveResult:- script = route({"/token": [_token()], f"/developer/v1/{endpoint}": pages})+def _source(endpoint: str, pages: list[ScriptedResponse], path: str | None = None, **inputs: Any) -> DriveResult:+ script = route({"/token": [_token()], f"/developer/v1/{path or endpoint}": pages})config = RampSourceConfig(environment="production", client_id="cid", client_secret="sec")return SourceDriver(RampSource(), config).run(endpoint, script, **inputs)@@ -87,6 +88,21 @@ class TestValidateCredentials:assert "credentials" in (message or "")+class TestCheckEndpointAccess:+ @pytest.mark.parametrize(+ "token_status, expected_denied",+ [(200, False), (400, True)],+ )+ def test_billing_statements_probe_their_own_scope(self, token_status, expected_denied):+ with scripted_network([_token(status_code=token_status)]) as network:+ reason = check_endpoint_access("production", "cid", "sec", "billing_statements")++ assert parse_qs((network.requests_log[0].body or b"").decode())["scope"] == ["billing_statements:read"]+ assert (reason is not None) == expected_denied+ if expected_denied:+ assert "billing_statements:read" in (reason or "")++class TestGetRows:def test_incremental_transactions_use_from_date(self):result = _source(@@ -100,6 +116,21 @@ class TestGetRows:assert data_requests[0].param("from_date") == "2024-01-02T00:00:00Z"assert data_requests[0].param("page_size") == str(PAGE_SIZE)+ def test_billing_statements_mint_only_their_own_scope_and_walk_pages(self):+ result = _source(+ "billing_statements",+ [+ _page([{"id": "s1"}], next_url="https://api.ramp.com/developer/v1/billing/statements?start=s1"),+ _page([{"id": "s2"}]),+ ],+ path="billing/statements",+ )++ assert result.raised is None+ token_request = next(request for request in result.requests if request.path.endswith("/token"))+ assert parse_qs((token_request.body or b"").decode())["scope"] == ["billing_statements:read"]+ assert [row["id"] for row in result.rows] == ["s1", "s2"]+def test_a_run_with_no_saved_state_starts_from_the_first_page(self):result = _source("transactions", [_page([{"id": "t1"}])])