Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add ramp billing statements table (#126971)

Tom Owers185047234959
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 02a76d3117a1..679b87bae903 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10034,7 +10034,7 @@ Note: No published SDL file; I ran an unauthenticated GraphQL introspection quer
## Ramp — **thin**
-Today (5): `cards`, `departments`, `reimbursements`, `transactions`, `users`
+Today (6): `billing_statements`, `cards`, `departments`, `reimbursements`, `transactions`, `users`
Diffed against: <https://docs.ramp.com/openapi/developer-api.json>
@@ -10050,6 +10050,7 @@ Diffed against: <https://docs.ramp.com/openapi/developer-api.json>
- [ ] `accounting/fields and accounting/field-options` — Custom accounting dimensions and their allowed values - the breakdown dimensions for coded spend (medium)
- [ ] `receipts and item-receipts` — Receipt records and line-item detail attached to transactions, for compliance reporting (medium)
- [ ] `spend-programs / unified-requests` — Spend limits and the approval-request stream that explains why spend was allowed or blocked (medium)
+- [x] `billing/statements` — Ramp billing statements per statement period, with the amount due and statement PDF link (medium)
Note: Ramp publishes a full OpenAPI spec (2.1 MB) plus llms-api.txt; the docs HTML explicitly points machine readers at /openapi/developer-api.json. The spec has ~95 GET paths across ~40 resources; PostHog exposes 5. Also note PostHog syncs the legacy /cards path, which no longer appears in the current spec (now /cards/physical and /cards/virtual) - worth confirming it still returns data.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py
index da2f389181f9..e825afcd5f16 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py
@@ -88,4 +88,14 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"name": "The department's name.",
},
},
+ "billing_statements": {
+ "description": "A billing statement for one statement period, with the total amount due.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/billing-statements",
+ "columns": {
+ "id": "Unique identifier of the billing statement.",
+ "amount": "The total amount due for this billing statement.",
+ "statement_period": "The billing statement period date range (start and end dates; end is null while the period is open).",
+ "statement_url": "Presigned URL to download the billing statement PDF, or null if unavailable.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py
index c22f6ead8744..7155c83aeb0d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py
@@ -54,7 +54,7 @@ def _token_url(environment: str) -> str:
return f"{_api_base_url(environment)}/token"
-def _make_auth(environment: str, client_id: str, client_secret: str) -> OAuth2Auth:
+def _make_auth(environment: str, client_id: str, client_secret: str, scopes: str = TOKEN_SCOPES) -> OAuth2Auth:
"""Ramp uses OAuth2 client-credentials with HTTP Basic client auth.
Tokens last ~10 days; the framework mints one lazily, caches it for the run, and re-mints
@@ -64,7 +64,7 @@ def _make_auth(environment: str, client_id: str, client_secret: str) -> OAuth2Au
client_id=client_id,
client_secret=client_secret,
grant_type="client_credentials",
- scopes=TOKEN_SCOPES,
+ scopes=scopes,
client_auth_method="basic",
)
@@ -121,6 +121,26 @@ def validate_credentials(environment: str, client_id: str, client_secret: str) -
return True, None
+def check_endpoint_access(environment: str, client_id: str, client_secret: str, endpoint: str) -> str | None:
+ """Report why the app can't sync an endpoint that mints its own scopes, or None when it can.
+
+ Validation only mints TOKEN_SCOPES, so an app created without an endpoint's extra scope still
+ connects; probe that scope here so the table starts deselected instead of failing every sync."""
+ scopes = RAMP_ENDPOINTS[endpoint].scopes
+ if scopes is None:
+ return None
+ auth = _make_auth(environment, client_id, client_secret, scopes)
+ probe = Request(method="GET", url=_api_base_url(environment)).prepare()
+ try:
+ auth(probe)
+ except OAuth2AuthRequestError as e:
+ if e.is_permanent:
+ return f"Your Ramp developer app needs the `{scopes}` scope to sync this table."
+ except requests.RequestException:
+ pass
+ return None
+
+
def ramp_source(
environment: str,
client_id: str,
@@ -133,7 +153,7 @@ def ramp_source(
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
config = RAMP_ENDPOINTS[endpoint]
- auth = _make_auth(environment, client_id, client_secret)
+ auth = _make_auth(environment, client_id, client_secret, config.scopes or TOKEN_SCOPES)
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py
index c1cf8e4ed660..c4c337f51b69 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py
@@ -8,7 +8,7 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
TOKEN_SCOPES = "transactions:read reimbursements:read users:read cards:read departments:read"
-@dataclass
+@dataclass(frozen=True)
class RampEndpointConfig:
name: str
# Path under {host}/developer/v1.
@@ -19,6 +19,9 @@ class RampEndpointConfig:
incremental_param: Optional[str] = None
# Stable event-time field used for datetime partitioning.
partition_key: Optional[str] = None
+ # Scopes minted for this endpoint instead of TOKEN_SCOPES. Ramp rejects a token request for a
+ # scope the app lacks, so a newer endpoint asks only for its own scope and existing apps keep working.
+ scopes: Optional[str] = None
RAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {
@@ -55,6 +58,11 @@ RAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {
name="departments",
path="/departments",
),
+ "billing_statements": RampEndpointConfig(
+ name="billing_statements",
+ path="/billing/statements",
+ scopes="billing_statements:read",
+ ),
}
ENDPOINTS = tuple(RAMP_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py
index 934ae4c65288..62ab5bdb8e7d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py
@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.ramp import RampSourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.ramp import (
RampResumeConfig,
+ check_endpoint_access,
ramp_source,
validate_credentials as validate_ramp_credentials,
)
@@ -78,7 +79,7 @@ class RampSource(ResumableSource[RampSourceConfig, RampResumeConfig]):
label="Ramp",
caption="""Connect your Ramp account to pull your spend data into the PostHog Data warehouse.
-A Ramp admin can create a developer app under Settings > Developer API. Grant it the `transactions:read`, `reimbursements:read`, `users:read`, `cards:read`, and `departments:read` scopes (scopes are fixed at app creation) and enable client credentials, then enter the app's client ID and secret here.""",
+A Ramp admin can create a developer app under Settings > Developer API. Grant it the `transactions:read`, `reimbursements:read`, `users:read`, `cards:read`, and `departments:read` scopes, plus `billing_statements:read` to sync billing statements (scopes are fixed at app creation) and enable client credentials, then enter the app's client ID and secret here.""",
iconPath="/static/services/ramp.png",
docsUrl="https://posthog.com/docs/cdp/sources/ramp",
releaseStatus=ReleaseStatus.GA,
@@ -131,6 +132,14 @@ A Ramp admin can create a developer app under Settings > Developer API. Grant it
) -> tuple[bool, str | None]:
return validate_ramp_credentials(config.environment, config.client_id, config.client_secret)
+ def get_endpoint_permissions(
+ self, config: RampSourceConfig, team_id: int, endpoints: list[str], api_version: str | None = None
+ ) -> dict[str, str | None]:
+ return {
+ endpoint: check_endpoint_access(config.environment, config.client_id, config.client_secret, endpoint)
+ for endpoint in endpoints
+ }
+
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[RampResumeConfig]:
return ResumableSourceManager[RampResumeConfig](inputs, RampResumeConfig)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py
index d15980c067e4..0ea44b102f08 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py
@@ -18,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.ramp
RampResumeConfig,
_base_url,
_format_timestamp,
+ check_endpoint_access,
validate_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.settings import TOKEN_SCOPES
@@ -39,8 +40,8 @@ def _token(status_code: int = 200, expires_in: int = 864000) -> ScriptedResponse
return ScriptedResponse(status=status_code, json=body)
-def _source(endpoint: str, pages: list[ScriptedResponse], **inputs: Any) -> DriveResult:
- script = route({"/token": [_token()], f"/developer/v1/{endpoint}": pages})
+def _source(endpoint: str, pages: list[ScriptedResponse], path: str | None = None, **inputs: Any) -> DriveResult:
+ script = route({"/token": [_token()], f"/developer/v1/{path or endpoint}": pages})
config = RampSourceConfig(environment="production", client_id="cid", client_secret="sec")
return SourceDriver(RampSource(), config).run(endpoint, script, **inputs)
@@ -87,6 +88,21 @@ class TestValidateCredentials:
assert "credentials" in (message or "")
+class TestCheckEndpointAccess:
+ @pytest.mark.parametrize(
+ "token_status, expected_denied",
+ [(200, False), (400, True)],
+ )
+ def test_billing_statements_probe_their_own_scope(self, token_status, expected_denied):
+ with scripted_network([_token(status_code=token_status)]) as network:
+ reason = check_endpoint_access("production", "cid", "sec", "billing_statements")
+
+ assert parse_qs((network.requests_log[0].body or b"").decode())["scope"] == ["billing_statements:read"]
+ assert (reason is not None) == expected_denied
+ if expected_denied:
+ assert "billing_statements:read" in (reason or "")
+
+
class TestGetRows:
def test_incremental_transactions_use_from_date(self):
result = _source(
@@ -100,6 +116,21 @@ class TestGetRows:
assert data_requests[0].param("from_date") == "2024-01-02T00:00:00Z"
assert data_requests[0].param("page_size") == str(PAGE_SIZE)
+ def test_billing_statements_mint_only_their_own_scope_and_walk_pages(self):
+ result = _source(
+ "billing_statements",
+ [
+ _page([{"id": "s1"}], next_url="https://api.ramp.com/developer/v1/billing/statements?start=s1"),
+ _page([{"id": "s2"}]),
+ ],
+ path="billing/statements",
+ )
+
+ assert result.raised is None
+ token_request = next(request for request in result.requests if request.path.endswith("/token"))
+ assert parse_qs((token_request.body or b"").decode())["scope"] == ["billing_statements:read"]
+ assert [row["id"] for row in result.rows] == ["s1", "s2"]
+
def test_a_run_with_no_saved_state_starts_from_the_first_page(self):
result = _source("transactions", [_page([{"id": "t1"}])])