:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ramp): add locations, entities, transfers and statements tables (#128422)
Tom Owersa5b1878e878b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex adc4c49ae723..c8218af2f135 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10055,10 +10055,10 @@ Diffed against: <https://docs.ramp.com/openapi/developer-api.json>- [x] `merchants` — Lookup resolving merchant IDs on every transaction - required for any spend-by-merchant analysis (high)- [x] `bills (and bills/drafts)` — Accounts-payable bills are a core Ramp spend object entirely missing from the warehouse (high)- [x] `accounting/accounts` — General-ledger account lookup that resolves the accounting codes applied to transactions (high)-- [ ] `locations` — Lookup resolving location IDs on transactions, users and cards (high)-- [ ] `entities` — Legal-entity lookup needed to split spend by entity in multi-entity orgs (high)-- [ ] `transfers` — Money movement in and out of the Ramp balance - the settlement side of card spend (medium)-- [ ] `statements` — Billing-period statements that reconcile transactions to what was actually charged (medium)+- [x] `locations` — Lookup resolving location IDs on transactions, users and cards (high)+- [x] `entities` — Legal-entity lookup needed to split spend by entity in multi-entity orgs (high)+- [x] `transfers` — Money movement in and out of the Ramp balance - the settlement side of card spend (medium)+- [x] `statements` — Billing-period statements that reconcile transactions to what was actually charged (medium)- [ ] `cashbacks` — Rewards earned per period, a headline Ramp value metric (medium)- [ ] `accounting/fields and accounting/field-options` — Custom accounting dimensions and their allowed values - the breakdown dimensions for coded spend (medium)- [ ] `receipts and item-receipts` — Receipt records and line-item detail attached to transactions, for compliance reporting (medium)diff --git a/products/warehouse_sources/sources/ramp/canonical_descriptions.py b/products/warehouse_sources/sources/ramp/canonical_descriptions.pyindex 573ce41af60c..a37a5adef621 100644--- a/products/warehouse_sources/sources/ramp/canonical_descriptions.py+++ b/products/warehouse_sources/sources/ramp/canonical_descriptions.py@@ -192,4 +192,60 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"statement_url": "Presigned URL to download the billing statement PDF, or null if unavailable.",},},+ "locations": {+ "description": "A location of the business, such as an office, that spend and users can be assigned to.",+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/locations",+ "columns": {+ "id": "Unique identifier for the location.",+ "name": "Name of the location.",+ "entity_id": "Identifier of the business entity this location belongs to.",+ },+ },+ "entities": {+ "description": "A business entity (legal entity) of a multi-entity Ramp business.",+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/entities",+ "columns": {+ "id": "Unique identifier for the business entity.",+ "entity_name": "Name of the business entity.",+ "currency": "Currency used by this business entity.",+ "is_primary": "Whether this is the primary entity for the business.",+ "accounts": "Information about the entity's active accounts.",+ "payment_accounts": "Bank accounts eligible for bill payment for this entity.",+ "default_bill_pay_payment_account_id": "Bank account ID of the entity's default Bill Pay payment account, if one exists.",+ "location_ids": "Identifiers of the locations associated with this entity.",+ "connected_subsidiary": "The subsidiary in the ERP system that this entity is mapped to.",+ },+ },+ "transfers": {+ "description": "A transfer payment that moves money between a business bank account and the Ramp balance.",+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/transfers",+ "columns": {+ "id": "Unique identifier for the transfer.",+ "amount": "Amount of the transfer.",+ "status": "Status of the transfer (e.g. INITIATED, COMPLETED, RETURNED_BY_RDFI).",+ "sync_status": "Accounting sync status of the transfer.",+ "created_at": "Time at which the transfer was created.",+ "entity_id": "Business entity associated with this transfer.",+ "bank_account_id": "Identifier of the bank account used for this transfer payment.",+ "payment_id": "Unique payment identifier for the transfer.",+ },+ },+ "statements": {+ "description": "A Ramp card statement for one statement period, with its balances and statement lines.",+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/statements",+ "columns": {+ "id": "Unique identifier for the statement.",+ "start_date": "Start date of the statement period.",+ "end_date": "End date of the statement period.",+ "opening_balance": "Balance at the beginning of the statement period.",+ "ending_balance": "Balance at the end of the statement period.",+ "charges": "Total charges incurred during the statement period.",+ "credits": "Total credits accumulated during the statement period.",+ "payments": "Total payments made during the statement period.",+ "balance_sections": "Breakdown of statement balances by entity and currency.",+ "statement_lines": "Financial activities in the period, such as card transactions, transfers and cashbacks.",+ "preceding_statement_id": "Identifier of the previous statement in sequence.",+ "statement_url": "Presigned URL to download the statement PDF, valid for 1 hour after generation.",+ },+ },}diff --git a/products/warehouse_sources/sources/ramp/ramp.py b/products/warehouse_sources/sources/ramp/ramp.pyindex 757c84151c73..ec2c8c23f319 100644--- a/products/warehouse_sources/sources/ramp/ramp.py+++ b/products/warehouse_sources/sources/ramp/ramp.py@@ -5,7 +5,7 @@ from typing import Any, Optionalimport requestsfrom requests import Request, Response-from products.warehouse_sources.sources.ramp.settings import PAGE_SIZE, RAMP_ENDPOINTS, TOKEN_SCOPES+from products.warehouse_sources.sources.ramp.settings import RAMP_ENDPOINTS, TOKEN_SCOPESfrom products.warehouse_sources.sources.sdk import (ClientConfig,Endpoint,@@ -155,7 +155,7 @@ def ramp_source(resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- params: dict[str, Any] = {"page_size": PAGE_SIZE}+ params: dict[str, Any] = {"page_size": config.page_size}if (config.incremental_param is not Noneand should_use_incremental_field@@ -236,7 +236,5 @@ def ramp_source(partition_mode="datetime" if config.partition_key else None,partition_format="month" if config.partition_key else None,partition_keys=[config.partition_key] if config.partition_key else None,- # Result ordering within a from_date window is not documented, so the- # pipeline defers incremental watermark commits until a run completes.- sort_mode="desc" if config.incremental_fields else "asc",+ sort_mode=config.sort_mode,)diff --git a/products/warehouse_sources/sources/ramp/settings.py b/products/warehouse_sources/sources/ramp/settings.pyindex a7380d6d6d6d..03ba593a9c56 100644--- a/products/warehouse_sources/sources/ramp/settings.py+++ b/products/warehouse_sources/sources/ramp/settings.py@@ -1,7 +1,12 @@from dataclasses import dataclass, fieldfrom typing import Optional-from products.warehouse_sources.sources.sdk import DependentEndpointConfig, IncrementalField, IncrementalFieldType+from products.warehouse_sources.sources.sdk import (+ DependentEndpointConfig,+ IncrementalField,+ IncrementalFieldType,+ SortMode,+)# Scopes requested when validating a new connection. Scopes are fixed on the# developer app at creation, so the caption tells users to grant these.@@ -27,6 +32,8 @@ class RampEndpointConfig:fanout: Optional[DependentEndpointConfig] = Nonepage_size: int = PAGE_SIZEdefault_incremental_field: Optional[str] = None+ sort_mode: Optional[SortMode] = "asc"+ default_incremental_lookback_seconds: Optional[int] = None_VENDORS_FANOUT = DependentEndpointConfig(@@ -48,6 +55,9 @@ RAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {# so late state changes (pending→cleared) need an occasional full# refresh — the same trade-off Fivetran makes.incremental_param="from_date",+ # Result ordering within a from_date window is not documented, so the+ # pipeline defers incremental watermark commits until a run completes.+ sort_mode="desc",incremental_fields=[{"label": "user_transaction_time",@@ -123,6 +133,56 @@ RAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {path="/billing/statements",scope="billing_statements:read",),+ "locations": RampEndpointConfig(+ name="locations",+ path="/locations",+ scope="locations:read",+ ),+ "entities": RampEndpointConfig(+ name="entities",+ path="/entities",+ scope="entities:read",+ ),+ # Ramp does not document the list order of transfers or statements, so their incremental+ # reads promise no sort order.+ "transfers": RampEndpointConfig(+ name="transfers",+ path="/transfers",+ scope="transfers:read",+ partition_key="created_at",+ # from_date filters on when the transfer occurred, but the status keeps changing while an+ # ACH transfer settles, so incremental runs re-read the trailing week.+ incremental_param="from_date",+ sort_mode=None,+ default_incremental_lookback_seconds=7 * 24 * 60 * 60,+ incremental_fields=[+ {+ "label": "created_at",+ "type": IncrementalFieldType.DateTime,+ "field": "created_at",+ "field_type": IncrementalFieldType.DateTime,+ },+ ],+ ),+ "statements": RampEndpointConfig(+ name="statements",+ path="/statements",+ scope="statements:read",+ partition_key="end_date",+ # from_date filters on end_date (inclusive).+ incremental_param="from_date",+ sort_mode=None,+ incremental_fields=[+ {+ "label": "end_date",+ "type": IncrementalFieldType.DateTime,+ "field": "end_date",+ "field_type": IncrementalFieldType.DateTime,+ },+ ],+ # Each statement embeds every line of its period, so a full page can be very large.+ page_size=10,+ ),}ENDPOINTS = tuple(RAMP_ENDPOINTS.keys())@@ -131,6 +191,12 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {name: config.incremental_fields for name, config in RAMP_ENDPOINTS.items() if config.incremental_fields}+DEFAULT_INCREMENTAL_LOOKBACK_SECONDS: dict[str, int] = {+ name: config.default_incremental_lookback_seconds+ for name, config in RAMP_ENDPOINTS.items()+ if config.default_incremental_lookback_seconds is not None+}+# Credential validation only proves TOKEN_SCOPES, so tables needing any other scope start# unselected rather than failing every sync on apps that were never granted it.SHOULD_SYNC_DEFAULT: dict[str, bool] = {diff --git a/products/warehouse_sources/sources/ramp/source.py b/products/warehouse_sources/sources/ramp/source.pyindex 634b7e999825..e0180a5f6805 100644--- a/products/warehouse_sources/sources/ramp/source.py+++ b/products/warehouse_sources/sources/ramp/source.py@@ -7,7 +7,12 @@ from products.warehouse_sources.sources.ramp.ramp import (ramp_source,validate_credentials as validate_ramp_credentials,)-from products.warehouse_sources.sources.ramp.settings import ENDPOINTS, INCREMENTAL_FIELDS, SHOULD_SYNC_DEFAULT+from products.warehouse_sources.sources.ramp.settings import (+ DEFAULT_INCREMENTAL_LOOKBACK_SECONDS,+ ENDPOINTS,+ INCREMENTAL_FIELDS,+ SHOULD_SYNC_DEFAULT,+)from products.warehouse_sources.sources.sdk import (OAUTH2_PERMANENT_ERROR_MARKER,CanonicalDescriptions,@@ -75,7 +80,7 @@ class RampSource(ResumableSource[RampSourceConfig, RampResumeConfig]):A Ramp admin can create a developer app under Settings > Developer API. Grant it the `transactions:read`, `reimbursements:read`, `users:read`, `cards:read`, and `departments:read` scopes (scopes are fixed at app creation) and enable client credentials, then enter the app's client ID and secret here.-To also sync billing statements, vendors, merchants, bills, or general ledger accounts, grant `billing_statements:read`, `vendors:read`, `merchants:read`, `bills:read`, or `accounting:read` as well.""",+To also sync billing statements, vendors, merchants, bills, general ledger accounts, locations, business entities, transfers, or statements, grant `billing_statements:read`, `vendors:read`, `merchants:read`, `bills:read`, `accounting:read`, `locations:read`, `entities:read`, `transfers:read`, or `statements:read` as well.""",iconPath="/static/services/ramp.png",docsUrl="https://posthog.com/docs/cdp/sources/ramp",releaseStatus=ReleaseStatus.GA,@@ -121,7 +126,13 @@ To also sync billing statements, vendors, merchants, bills, or general ledger acforce_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, should_sync_default=SHOULD_SYNC_DEFAULT)+ return build_endpoint_schemas(+ ENDPOINTS,+ INCREMENTAL_FIELDS,+ names,+ should_sync_default=SHOULD_SYNC_DEFAULT,+ default_incremental_lookback_seconds=DEFAULT_INCREMENTAL_LOOKBACK_SECONDS,+ )def validate_credentials(self, config: RampSourceConfig, team_id: int, schema_name: Optional[str] = None, api_version: str | None = Nonediff --git a/products/warehouse_sources/sources/ramp/tests/test_ramp.py b/products/warehouse_sources/sources/ramp/tests/test_ramp.pyindex e68b38518b7f..c693d27442cc 100644--- a/products/warehouse_sources/sources/ramp/tests/test_ramp.py+++ b/products/warehouse_sources/sources/ramp/tests/test_ramp.py@@ -7,14 +7,13 @@ import pytestfrom products.warehouse_sources.sources.ramp._config import RampSourceConfigfrom products.warehouse_sources.sources.ramp.ramp import (- PAGE_SIZE,RampResumeConfig,_base_url,_format_timestamp,check_endpoint_access,validate_credentials,)-from products.warehouse_sources.sources.ramp.settings import TOKEN_SCOPES+from products.warehouse_sources.sources.ramp.settings import PAGE_SIZE, TOKEN_SCOPESfrom products.warehouse_sources.sources.ramp.source import RampSourcefrom products.warehouse_sources.sources.sdk.testing import (DriveResult,@@ -111,32 +110,59 @@ class TestCheckEndpointAccess:class TestGetRows:- def test_incremental_transactions_use_from_date(self):+ @pytest.mark.parametrize(+ "endpoint, incremental, expected_from_date, expected_page_size, expected_sort_mode",+ [+ ("transactions", True, "2024-01-02T00:00:00Z", PAGE_SIZE, "desc"),+ ("transfers", True, "2024-01-02T00:00:00Z", PAGE_SIZE, None),+ ("statements", True, "2024-01-02T00:00:00Z", 10, None),+ ("transfers", False, None, PAGE_SIZE, None),+ ("statements", False, None, 10, None),+ ("locations", False, None, PAGE_SIZE, "asc"),+ ],+ )+ def test_request_shaping_and_sort_mode(+ self, endpoint, incremental, expected_from_date, expected_page_size, expected_sort_mode+ ):result = _source(- "transactions",+ endpoint,[_page([])],- should_use_incremental_field=True,- db_incremental_field_last_value=datetime(2024, 1, 2, tzinfo=UTC),+ should_use_incremental_field=incremental,+ db_incremental_field_last_value=datetime(2024, 1, 2, tzinfo=UTC) if incremental else None,)- data_requests = [request for request in result.requests if request.path.endswith("/transactions")]- assert data_requests[0].param("from_date") == "2024-01-02T00:00:00Z"- assert data_requests[0].param("page_size") == str(PAGE_SIZE)+ assert result.raised is None+ data_requests = [request for request in result.requests if request.path.endswith(f"/{endpoint}")]+ assert data_requests[0].param("from_date") == expected_from_date+ assert data_requests[0].param("page_size") == str(expected_page_size)+ assert result.response is not None+ assert result.response.sort_mode == expected_sort_mode- def test_billing_statements_mint_only_their_own_scope_and_walk_pages(self):+ @pytest.mark.parametrize(+ "endpoint, path, scope",+ [+ ("billing_statements", "billing/statements", "billing_statements:read"),+ ("locations", "locations", "locations:read"),+ ("entities", "entities", "entities:read"),+ ("transfers", "transfers", "transfers:read"),+ ("statements", "statements", "statements:read"),+ ],+ )+ def test_endpoints_mint_only_their_own_scope_and_walk_pages(self, endpoint, path, scope):result = _source(- "billing_statements",+ endpoint,[- _page([{"id": "s1"}], next_url="https://api.ramp.com/developer/v1/billing/statements?start=s1"),- _page([{"id": "s2"}]),+ _page([{"id": "r1"}], next_url=f"https://api.ramp.com/developer/v1/{path}?start=r1"),+ _page([{"id": "r2"}]),],- path="billing/statements",+ path=path,)assert result.raised is Nonetoken_request = next(request for request in result.requests if request.path.endswith("/token"))- assert parse_qs((token_request.body or b"").decode())["scope"] == ["billing_statements:read"]- assert [row["id"] for row in result.rows] == ["s1", "s2"]+ assert parse_qs((token_request.body or b"").decode())["scope"] == [scope]+ assert [row["id"] for row in result.rows] == ["r1", "r2"]+ assert result.urls[-1] == f"https://api.ramp.com/developer/v1/{path}?start=r1"def test_a_run_with_no_saved_state_starts_from_the_first_page(self):result = _source("transactions", [_page([{"id": "t1"}])])diff --git a/products/warehouse_sources/sources/ramp/tests/test_ramp_source.py b/products/warehouse_sources/sources/ramp/tests/test_ramp_source.pyindex c4145db06f96..3816ac59b5ae 100644--- a/products/warehouse_sources/sources/ramp/tests/test_ramp_source.py+++ b/products/warehouse_sources/sources/ramp/tests/test_ramp_source.py@@ -1,3 +1,5 @@+import pytest+from products.warehouse_sources.sources.ramp._config import RampSourceConfigfrom products.warehouse_sources.sources.ramp.source import RampSource@@ -12,3 +14,16 @@ class TestRampSource:# Changing environment retargets where the stored client secret is sent, so it must force# re-entering secrets.assert self.source.connection_host_fields == ["environment"]++ @pytest.mark.parametrize(+ "endpoint, expected_lookback_seconds",+ [+ ("transfers", 7 * 24 * 60 * 60),+ ("statements", None),+ ("transactions", None),+ ],+ )+ def test_incremental_lookback_default_reaches_the_schema(self, endpoint, expected_lookback_seconds):+ [schema] = self.source.get_schemas(self.config, self.team_id, names=[endpoint])++ assert schema.default_incremental_lookback_seconds == expected_lookback_seconds