Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ramp): add locations, entities, transfers and statements tables (#128422)

Tom Owersa5b1878e878b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index adc4c49ae723..c8218af2f135 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10055,10 +10055,10 @@ Diffed against: <https://docs.ramp.com/openapi/developer-api.json>
- [x] `merchants` — Lookup resolving merchant IDs on every transaction - required for any spend-by-merchant analysis (high)
- [x] `bills (and bills/drafts)` — Accounts-payable bills are a core Ramp spend object entirely missing from the warehouse (high)
- [x] `accounting/accounts` — General-ledger account lookup that resolves the accounting codes applied to transactions (high)
-- [ ] `locations` — Lookup resolving location IDs on transactions, users and cards (high)
-- [ ] `entities` — Legal-entity lookup needed to split spend by entity in multi-entity orgs (high)
-- [ ] `transfers` — Money movement in and out of the Ramp balance - the settlement side of card spend (medium)
-- [ ] `statements` — Billing-period statements that reconcile transactions to what was actually charged (medium)
+- [x] `locations` — Lookup resolving location IDs on transactions, users and cards (high)
+- [x] `entities` — Legal-entity lookup needed to split spend by entity in multi-entity orgs (high)
+- [x] `transfers` — Money movement in and out of the Ramp balance - the settlement side of card spend (medium)
+- [x] `statements` — Billing-period statements that reconcile transactions to what was actually charged (medium)
- [ ] `cashbacks` — Rewards earned per period, a headline Ramp value metric (medium)
- [ ] `accounting/fields and accounting/field-options` — Custom accounting dimensions and their allowed values - the breakdown dimensions for coded spend (medium)
- [ ] `receipts and item-receipts` — Receipt records and line-item detail attached to transactions, for compliance reporting (medium)
diff --git a/products/warehouse_sources/sources/ramp/canonical_descriptions.py b/products/warehouse_sources/sources/ramp/canonical_descriptions.py
index 573ce41af60c..a37a5adef621 100644
--- a/products/warehouse_sources/sources/ramp/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/ramp/canonical_descriptions.py
@@ -192,4 +192,60 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"statement_url": "Presigned URL to download the billing statement PDF, or null if unavailable.",
},
},
+ "locations": {
+ "description": "A location of the business, such as an office, that spend and users can be assigned to.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/locations",
+ "columns": {
+ "id": "Unique identifier for the location.",
+ "name": "Name of the location.",
+ "entity_id": "Identifier of the business entity this location belongs to.",
+ },
+ },
+ "entities": {
+ "description": "A business entity (legal entity) of a multi-entity Ramp business.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/entities",
+ "columns": {
+ "id": "Unique identifier for the business entity.",
+ "entity_name": "Name of the business entity.",
+ "currency": "Currency used by this business entity.",
+ "is_primary": "Whether this is the primary entity for the business.",
+ "accounts": "Information about the entity's active accounts.",
+ "payment_accounts": "Bank accounts eligible for bill payment for this entity.",
+ "default_bill_pay_payment_account_id": "Bank account ID of the entity's default Bill Pay payment account, if one exists.",
+ "location_ids": "Identifiers of the locations associated with this entity.",
+ "connected_subsidiary": "The subsidiary in the ERP system that this entity is mapped to.",
+ },
+ },
+ "transfers": {
+ "description": "A transfer payment that moves money between a business bank account and the Ramp balance.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/transfers",
+ "columns": {
+ "id": "Unique identifier for the transfer.",
+ "amount": "Amount of the transfer.",
+ "status": "Status of the transfer (e.g. INITIATED, COMPLETED, RETURNED_BY_RDFI).",
+ "sync_status": "Accounting sync status of the transfer.",
+ "created_at": "Time at which the transfer was created.",
+ "entity_id": "Business entity associated with this transfer.",
+ "bank_account_id": "Identifier of the bank account used for this transfer payment.",
+ "payment_id": "Unique payment identifier for the transfer.",
+ },
+ },
+ "statements": {
+ "description": "A Ramp card statement for one statement period, with its balances and statement lines.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/statements",
+ "columns": {
+ "id": "Unique identifier for the statement.",
+ "start_date": "Start date of the statement period.",
+ "end_date": "End date of the statement period.",
+ "opening_balance": "Balance at the beginning of the statement period.",
+ "ending_balance": "Balance at the end of the statement period.",
+ "charges": "Total charges incurred during the statement period.",
+ "credits": "Total credits accumulated during the statement period.",
+ "payments": "Total payments made during the statement period.",
+ "balance_sections": "Breakdown of statement balances by entity and currency.",
+ "statement_lines": "Financial activities in the period, such as card transactions, transfers and cashbacks.",
+ "preceding_statement_id": "Identifier of the previous statement in sequence.",
+ "statement_url": "Presigned URL to download the statement PDF, valid for 1 hour after generation.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/ramp/ramp.py b/products/warehouse_sources/sources/ramp/ramp.py
index 757c84151c73..ec2c8c23f319 100644
--- a/products/warehouse_sources/sources/ramp/ramp.py
+++ b/products/warehouse_sources/sources/ramp/ramp.py
@@ -5,7 +5,7 @@ from typing import Any, Optional
import requests
from requests import Request, Response
-from products.warehouse_sources.sources.ramp.settings import PAGE_SIZE, RAMP_ENDPOINTS, TOKEN_SCOPES
+from products.warehouse_sources.sources.ramp.settings import RAMP_ENDPOINTS, TOKEN_SCOPES
from products.warehouse_sources.sources.sdk import (
ClientConfig,
Endpoint,
@@ -155,7 +155,7 @@ def ramp_source(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- params: dict[str, Any] = {"page_size": PAGE_SIZE}
+ params: dict[str, Any] = {"page_size": config.page_size}
if (
config.incremental_param is not None
and should_use_incremental_field
@@ -236,7 +236,5 @@ def ramp_source(
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
- # Result ordering within a from_date window is not documented, so the
- # pipeline defers incremental watermark commits until a run completes.
- sort_mode="desc" if config.incremental_fields else "asc",
+ sort_mode=config.sort_mode,
)
diff --git a/products/warehouse_sources/sources/ramp/settings.py b/products/warehouse_sources/sources/ramp/settings.py
index a7380d6d6d6d..03ba593a9c56 100644
--- a/products/warehouse_sources/sources/ramp/settings.py
+++ b/products/warehouse_sources/sources/ramp/settings.py
@@ -1,7 +1,12 @@
from dataclasses import dataclass, field
from typing import Optional
-from products.warehouse_sources.sources.sdk import DependentEndpointConfig, IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import (
+ DependentEndpointConfig,
+ IncrementalField,
+ IncrementalFieldType,
+ SortMode,
+)
# Scopes requested when validating a new connection. Scopes are fixed on the
# developer app at creation, so the caption tells users to grant these.
@@ -27,6 +32,8 @@ class RampEndpointConfig:
fanout: Optional[DependentEndpointConfig] = None
page_size: int = PAGE_SIZE
default_incremental_field: Optional[str] = None
+ sort_mode: Optional[SortMode] = "asc"
+ default_incremental_lookback_seconds: Optional[int] = None
_VENDORS_FANOUT = DependentEndpointConfig(
@@ -48,6 +55,9 @@ RAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {
# so late state changes (pending→cleared) need an occasional full
# refresh — the same trade-off Fivetran makes.
incremental_param="from_date",
+ # Result ordering within a from_date window is not documented, so the
+ # pipeline defers incremental watermark commits until a run completes.
+ sort_mode="desc",
incremental_fields=[
{
"label": "user_transaction_time",
@@ -123,6 +133,56 @@ RAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {
path="/billing/statements",
scope="billing_statements:read",
),
+ "locations": RampEndpointConfig(
+ name="locations",
+ path="/locations",
+ scope="locations:read",
+ ),
+ "entities": RampEndpointConfig(
+ name="entities",
+ path="/entities",
+ scope="entities:read",
+ ),
+ # Ramp does not document the list order of transfers or statements, so their incremental
+ # reads promise no sort order.
+ "transfers": RampEndpointConfig(
+ name="transfers",
+ path="/transfers",
+ scope="transfers:read",
+ partition_key="created_at",
+ # from_date filters on when the transfer occurred, but the status keeps changing while an
+ # ACH transfer settles, so incremental runs re-read the trailing week.
+ incremental_param="from_date",
+ sort_mode=None,
+ default_incremental_lookback_seconds=7 * 24 * 60 * 60,
+ incremental_fields=[
+ {
+ "label": "created_at",
+ "type": IncrementalFieldType.DateTime,
+ "field": "created_at",
+ "field_type": IncrementalFieldType.DateTime,
+ },
+ ],
+ ),
+ "statements": RampEndpointConfig(
+ name="statements",
+ path="/statements",
+ scope="statements:read",
+ partition_key="end_date",
+ # from_date filters on end_date (inclusive).
+ incremental_param="from_date",
+ sort_mode=None,
+ incremental_fields=[
+ {
+ "label": "end_date",
+ "type": IncrementalFieldType.DateTime,
+ "field": "end_date",
+ "field_type": IncrementalFieldType.DateTime,
+ },
+ ],
+ # Each statement embeds every line of its period, so a full page can be very large.
+ page_size=10,
+ ),
}
ENDPOINTS = tuple(RAMP_ENDPOINTS.keys())
@@ -131,6 +191,12 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in RAMP_ENDPOINTS.items() if config.incremental_fields
}
+DEFAULT_INCREMENTAL_LOOKBACK_SECONDS: dict[str, int] = {
+ name: config.default_incremental_lookback_seconds
+ for name, config in RAMP_ENDPOINTS.items()
+ if config.default_incremental_lookback_seconds is not None
+}
+
# Credential validation only proves TOKEN_SCOPES, so tables needing any other scope start
# unselected rather than failing every sync on apps that were never granted it.
SHOULD_SYNC_DEFAULT: dict[str, bool] = {
diff --git a/products/warehouse_sources/sources/ramp/source.py b/products/warehouse_sources/sources/ramp/source.py
index 634b7e999825..e0180a5f6805 100644
--- a/products/warehouse_sources/sources/ramp/source.py
+++ b/products/warehouse_sources/sources/ramp/source.py
@@ -7,7 +7,12 @@ from products.warehouse_sources.sources.ramp.ramp import (
ramp_source,
validate_credentials as validate_ramp_credentials,
)
-from products.warehouse_sources.sources.ramp.settings import ENDPOINTS, INCREMENTAL_FIELDS, SHOULD_SYNC_DEFAULT
+from products.warehouse_sources.sources.ramp.settings import (
+ DEFAULT_INCREMENTAL_LOOKBACK_SECONDS,
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ SHOULD_SYNC_DEFAULT,
+)
from products.warehouse_sources.sources.sdk import (
OAUTH2_PERMANENT_ERROR_MARKER,
CanonicalDescriptions,
@@ -75,7 +80,7 @@ class RampSource(ResumableSource[RampSourceConfig, RampResumeConfig]):
A Ramp admin can create a developer app under Settings > Developer API. Grant it the `transactions:read`, `reimbursements:read`, `users:read`, `cards:read`, and `departments:read` scopes (scopes are fixed at app creation) and enable client credentials, then enter the app's client ID and secret here.
-To also sync billing statements, vendors, merchants, bills, or general ledger accounts, grant `billing_statements:read`, `vendors:read`, `merchants:read`, `bills:read`, or `accounting:read` as well.""",
+To also sync billing statements, vendors, merchants, bills, general ledger accounts, locations, business entities, transfers, or statements, grant `billing_statements:read`, `vendors:read`, `merchants:read`, `bills:read`, `accounting:read`, `locations:read`, `entities:read`, `transfers:read`, or `statements:read` as well.""",
iconPath="/static/services/ramp.png",
docsUrl="https://posthog.com/docs/cdp/sources/ramp",
releaseStatus=ReleaseStatus.GA,
@@ -121,7 +126,13 @@ To also sync billing statements, vendors, merchants, bills, or general ledger ac
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, should_sync_default=SHOULD_SYNC_DEFAULT)
+ return build_endpoint_schemas(
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ should_sync_default=SHOULD_SYNC_DEFAULT,
+ default_incremental_lookback_seconds=DEFAULT_INCREMENTAL_LOOKBACK_SECONDS,
+ )
def validate_credentials(
self, config: RampSourceConfig, team_id: int, schema_name: Optional[str] = None, api_version: str | None = None
diff --git a/products/warehouse_sources/sources/ramp/tests/test_ramp.py b/products/warehouse_sources/sources/ramp/tests/test_ramp.py
index e68b38518b7f..c693d27442cc 100644
--- a/products/warehouse_sources/sources/ramp/tests/test_ramp.py
+++ b/products/warehouse_sources/sources/ramp/tests/test_ramp.py
@@ -7,14 +7,13 @@ import pytest
from products.warehouse_sources.sources.ramp._config import RampSourceConfig
from products.warehouse_sources.sources.ramp.ramp import (
- PAGE_SIZE,
RampResumeConfig,
_base_url,
_format_timestamp,
check_endpoint_access,
validate_credentials,
)
-from products.warehouse_sources.sources.ramp.settings import TOKEN_SCOPES
+from products.warehouse_sources.sources.ramp.settings import PAGE_SIZE, TOKEN_SCOPES
from products.warehouse_sources.sources.ramp.source import RampSource
from products.warehouse_sources.sources.sdk.testing import (
DriveResult,
@@ -111,32 +110,59 @@ class TestCheckEndpointAccess:
class TestGetRows:
- def test_incremental_transactions_use_from_date(self):
+ @pytest.mark.parametrize(
+ "endpoint, incremental, expected_from_date, expected_page_size, expected_sort_mode",
+ [
+ ("transactions", True, "2024-01-02T00:00:00Z", PAGE_SIZE, "desc"),
+ ("transfers", True, "2024-01-02T00:00:00Z", PAGE_SIZE, None),
+ ("statements", True, "2024-01-02T00:00:00Z", 10, None),
+ ("transfers", False, None, PAGE_SIZE, None),
+ ("statements", False, None, 10, None),
+ ("locations", False, None, PAGE_SIZE, "asc"),
+ ],
+ )
+ def test_request_shaping_and_sort_mode(
+ self, endpoint, incremental, expected_from_date, expected_page_size, expected_sort_mode
+ ):
result = _source(
- "transactions",
+ endpoint,
[_page([])],
- should_use_incremental_field=True,
- db_incremental_field_last_value=datetime(2024, 1, 2, tzinfo=UTC),
+ should_use_incremental_field=incremental,
+ db_incremental_field_last_value=datetime(2024, 1, 2, tzinfo=UTC) if incremental else None,
)
- data_requests = [request for request in result.requests if request.path.endswith("/transactions")]
- assert data_requests[0].param("from_date") == "2024-01-02T00:00:00Z"
- assert data_requests[0].param("page_size") == str(PAGE_SIZE)
+ assert result.raised is None
+ data_requests = [request for request in result.requests if request.path.endswith(f"/{endpoint}")]
+ assert data_requests[0].param("from_date") == expected_from_date
+ assert data_requests[0].param("page_size") == str(expected_page_size)
+ assert result.response is not None
+ assert result.response.sort_mode == expected_sort_mode
- def test_billing_statements_mint_only_their_own_scope_and_walk_pages(self):
+ @pytest.mark.parametrize(
+ "endpoint, path, scope",
+ [
+ ("billing_statements", "billing/statements", "billing_statements:read"),
+ ("locations", "locations", "locations:read"),
+ ("entities", "entities", "entities:read"),
+ ("transfers", "transfers", "transfers:read"),
+ ("statements", "statements", "statements:read"),
+ ],
+ )
+ def test_endpoints_mint_only_their_own_scope_and_walk_pages(self, endpoint, path, scope):
result = _source(
- "billing_statements",
+ endpoint,
[
- _page([{"id": "s1"}], next_url="https://api.ramp.com/developer/v1/billing/statements?start=s1"),
- _page([{"id": "s2"}]),
+ _page([{"id": "r1"}], next_url=f"https://api.ramp.com/developer/v1/{path}?start=r1"),
+ _page([{"id": "r2"}]),
],
- path="billing/statements",
+ path=path,
)
assert result.raised is None
token_request = next(request for request in result.requests if request.path.endswith("/token"))
- assert parse_qs((token_request.body or b"").decode())["scope"] == ["billing_statements:read"]
- assert [row["id"] for row in result.rows] == ["s1", "s2"]
+ assert parse_qs((token_request.body or b"").decode())["scope"] == [scope]
+ assert [row["id"] for row in result.rows] == ["r1", "r2"]
+ assert result.urls[-1] == f"https://api.ramp.com/developer/v1/{path}?start=r1"
def test_a_run_with_no_saved_state_starts_from_the_first_page(self):
result = _source("transactions", [_page([{"id": "t1"}])])
diff --git a/products/warehouse_sources/sources/ramp/tests/test_ramp_source.py b/products/warehouse_sources/sources/ramp/tests/test_ramp_source.py
index c4145db06f96..3816ac59b5ae 100644
--- a/products/warehouse_sources/sources/ramp/tests/test_ramp_source.py
+++ b/products/warehouse_sources/sources/ramp/tests/test_ramp_source.py
@@ -1,3 +1,5 @@
+import pytest
+
from products.warehouse_sources.sources.ramp._config import RampSourceConfig
from products.warehouse_sources.sources.ramp.source import RampSource
@@ -12,3 +14,16 @@ class TestRampSource:
# Changing environment retargets where the stored client secret is sent, so it must force
# re-entering secrets.
assert self.source.connection_host_fields == ["environment"]
+
+ @pytest.mark.parametrize(
+ "endpoint, expected_lookback_seconds",
+ [
+ ("transfers", 7 * 24 * 60 * 60),
+ ("statements", None),
+ ("transactions", None),
+ ],
+ )
+ def test_incremental_lookback_default_reaches_the_schema(self, endpoint, expected_lookback_seconds):
+ [schema] = self.source.get_schemas(self.config, self.team_id, names=[endpoint])
+
+ assert schema.default_incremental_lookback_seconds == expected_lookback_seconds