Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add ramp vendors, merchants, bills and gl account tables (#126980)

Tom Owersed43268e793f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 97c77ed33b46..fabb70176105 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10038,10 +10038,10 @@ Today (6): `billing_statements`, `cards`, `departments`, `reimbursements`, `tran
Diffed against: <https://docs.ramp.com/openapi/developer-api.json>
-- [ ] `vendors (and vendors/{id}/contacts, /accounts)` — Lookup table resolving the vendor IDs carried on bills, transactions and purchase orders (high)
-- [ ] `merchants` — Lookup resolving merchant IDs on every transaction - required for any spend-by-merchant analysis (high)
-- [ ] `bills (and bills/drafts)` — Accounts-payable bills are a core Ramp spend object entirely missing from the warehouse (high)
-- [ ] `accounting/accounts` — General-ledger account lookup that resolves the accounting codes applied to transactions (high)
+- [x] `vendors (and vendors/{id}/contacts, /accounts)` — Lookup table resolving the vendor IDs carried on bills, transactions and purchase orders (high)
+- [x] `merchants` — Lookup resolving merchant IDs on every transaction - required for any spend-by-merchant analysis (high)
+- [x] `bills (and bills/drafts)` — Accounts-payable bills are a core Ramp spend object entirely missing from the warehouse (high)
+- [x] `accounting/accounts` — General-ledger account lookup that resolves the accounting codes applied to transactions (high)
- [ ] `locations` — Lookup resolving location IDs on transactions, users and cards (high)
- [ ] `entities` — Legal-entity lookup needed to split spend by entity in multi-entity orgs (high)
- [ ] `transfers` — Money movement in and out of the Ramp balance - the settlement side of card spend (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py
index e825afcd5f16..6bc569ae513e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/canonical_descriptions.py
@@ -88,6 +88,102 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"name": "The department's name.",
},
},
+ "vendors": {
+ "description": "A vendor the business pays through Ramp bill pay or card spend.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/vendors",
+ "columns": {
+ "id": "Unique identifier of the vendor.",
+ "name": "Name of the vendor.",
+ "name_legal": "Legal name of the vendor.",
+ "display_name": "Display name of the vendor.",
+ "merchant_id": "ID of the card merchant associated with the vendor.",
+ "external_vendor_id": "Customer-defined external identifier for the vendor.",
+ "accounting_vendor_remote_id": "ID of the vendor in the connected accounting system.",
+ "parent_vendor_id": "ID of the parent vendor, if this vendor has one.",
+ "vendor_owner_id": "ID of the user who owns this vendor.",
+ "default_entity_id": "ID of the default business entity associated with this vendor.",
+ "contacts": "IDs of the contacts associated with this vendor.",
+ "is_active": "Whether this vendor is active.",
+ "net_payment_terms": "Number of days after receipt that payment is due. 0 means due on receipt.",
+ "sk_category_name": "Category name of the vendor.",
+ "total_spend_all_time": "Total spend with this vendor for all time.",
+ "total_spend_ytd": "Total spend with this vendor year to date.",
+ "created_at": "Time at which the vendor was created.",
+ "updated_at": "Time at which the vendor was last updated.",
+ },
+ },
+ "vendor_contacts": {
+ "description": "A contact person at a vendor.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/vendors",
+ "columns": {
+ "id": "Unique identifier of the vendor contact.",
+ "vendor_id": "ID of the vendor the contact belongs to.",
+ "is_default": "Whether this is the vendor's default contact.",
+ },
+ },
+ "merchants": {
+ "description": "A card merchant that Ramp transactions are made with.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/merchants",
+ "columns": {
+ "id": "Unique identifier for the merchant.",
+ "merchant_name": "Display name of the merchant.",
+ "sk_category_name": "Category classification of the merchant.",
+ "is_auto_approved": "Whether transactions from this merchant are exempt from receipt submission requirements.",
+ },
+ },
+ "bills": {
+ "description": "An accounts payable bill from a vendor, paid through Ramp bill pay.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/bills",
+ "columns": {
+ "id": "Unique identifier for the bill.",
+ "amount": "Amount of the bill, in the lowest denomination of the currency (e.g. cents for USD).",
+ "vendor": "Vendor information of the bill.",
+ "vendor_contact_id": "ID of the associated vendor contact.",
+ "invoice_number": "The invoice number on the bill.",
+ "issued_at": "The date the bill was issued on the invoice.",
+ "due_at": "The date the bill is due.",
+ "paid_at": "Time at which the bill's payment status was set to PAID.",
+ "approval_status": "Approval status of the bill.",
+ "status_summary": "Status summary of the bill.",
+ "sync_status": "Sync status of the bill with the accounting system.",
+ "entity_id": "ID of the associated business entity.",
+ "purchase_order_id": "ID of the matched purchase order, if one exists.",
+ "draft_bill_id": "ID of the draft bill this bill was created from, if one exists.",
+ "line_items": "Line items of the bill.",
+ "accounting_field_selections": "Accounting field options selected to code the bill.",
+ "payment": "Payment information for the bill.",
+ },
+ },
+ "bill_drafts": {
+ "description": "A draft accounts payable bill that has not been submitted yet.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/bills",
+ "columns": {
+ "id": "Unique identifier for the draft bill.",
+ "amount": "Amount of the bill, in the lowest denomination of the currency (e.g. cents for USD).",
+ "vendor": "Vendor information of the bill.",
+ "invoice_number": "The invoice number on the bill.",
+ "created_at": "The date the draft bill was created.",
+ "issued_at": "The date the bill was issued on the invoice.",
+ "due_at": "The date the bill is due.",
+ "status": "Payment status of the bill. Always DRAFT for draft bills.",
+ "entity_id": "ID of the associated business entity.",
+ },
+ },
+ "accounting_accounts": {
+ "description": "A general ledger account from the connected accounting system.",
+ "docs_url": "https://docs.ramp.com/developer-api/v1/api/accounting",
+ "columns": {
+ "ramp_id": "Unique identifier of the general ledger account in Ramp.",
+ "id": "ID of the general ledger account in the accounting system.",
+ "code": "Account code (e.g. 400-100).",
+ "name": "Account name (e.g. Travel : Travel - Lodging).",
+ "classification": "Classification of the account.",
+ "is_active": "Whether the account is active.",
+ "accounting_connection_id": "ID of the accounting connection the account belongs to.",
+ "provider_name": "Name of the accounting connection the account belongs to.",
+ "entity_remote_ids": "IDs of the entities this account is available for. Null means all entities.",
+ },
+ },
"billing_statements": {
"description": "A billing statement for one statement period, with the total amount due.",
"docs_url": "https://docs.ramp.com/developer-api/v1/api/billing-statements",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py
index 7155c83aeb0d..1a648c83a56b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/ramp.py
@@ -13,6 +13,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
OAuth2Auth,
OAuth2AuthRequestError,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponsePaginator,
)
@@ -22,21 +25,24 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.settings import RAMP_ENDPOINTS, TOKEN_SCOPES
+from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.settings import (
+ PAGE_SIZE,
+ RAMP_ENDPOINTS,
+ TOKEN_SCOPES,
+)
RAMP_HOSTS = {
"production": "https://api.ramp.com",
"sandbox": "https://demo-api.ramp.com",
}
-# Ramp list pages cap at 100 items.
-PAGE_SIZE = 100
[email protected](frozen=True)
class RampResumeConfig:
# Ramp paginates via the self-contained page.next URL. Optional/defaulted so previously
# saved state (which always carried next_url) still parses via dataclass(**saved).
next_url: Optional[str] = None
+ fanout_state: Optional[dict[str, Any]] = None
def _base_url(environment: str) -> str:
@@ -54,7 +60,7 @@ def _token_url(environment: str) -> str:
return f"{_api_base_url(environment)}/token"
-def _make_auth(environment: str, client_id: str, client_secret: str, scopes: str = TOKEN_SCOPES) -> OAuth2Auth:
+def _make_auth(environment: str, client_id: str, client_secret: str, scopes: str) -> OAuth2Auth:
"""Ramp uses OAuth2 client-credentials with HTTP Basic client auth.
Tokens last ~10 days; the framework mints one lazily, caches it for the run, and re-mints
@@ -97,13 +103,17 @@ class RampPaginator(JSONResponsePaginator):
super().update_state(response, data)
-def validate_credentials(environment: str, client_id: str, client_secret: str) -> tuple[bool, str | None]:
+def validate_credentials(
+ environment: str, client_id: str, client_secret: str, schema_name: Optional[str] = None
+) -> tuple[bool, str | None]:
"""Confirm the developer app credentials are valid by minting a token.
Distinguishes a genuine credential rejection (permanent 4xx from the token endpoint) from a
transient connectivity problem so the user sees an actionable message instead of a blanket
"invalid credentials"."""
- auth = _make_auth(environment, client_id, client_secret)
+ endpoint_config = RAMP_ENDPOINTS.get(schema_name) if schema_name else None
+ scopes = endpoint_config.scope if endpoint_config else TOKEN_SCOPES
+ auth = _make_auth(environment, client_id, client_secret, scopes)
# Force the lazy token mint through the public auth callable; a bad credential raises here.
probe = Request(method="GET", url=_api_base_url(environment)).prepare()
try:
@@ -122,20 +132,20 @@ def validate_credentials(environment: str, client_id: str, client_secret: str) -
def check_endpoint_access(environment: str, client_id: str, client_secret: str, endpoint: str) -> str | None:
- """Report why the app can't sync an endpoint that mints its own scopes, or None when it can.
+ """Report why the app can't sync an endpoint whose scope is outside TOKEN_SCOPES, or None when it can.
Validation only mints TOKEN_SCOPES, so an app created without an endpoint's extra scope still
connects; probe that scope here so the table starts deselected instead of failing every sync."""
- scopes = RAMP_ENDPOINTS[endpoint].scopes
- if scopes is None:
+ scope = RAMP_ENDPOINTS[endpoint].scope
+ if scope in TOKEN_SCOPES.split():
return None
- auth = _make_auth(environment, client_id, client_secret, scopes)
+ auth = _make_auth(environment, client_id, client_secret, scope)
probe = Request(method="GET", url=_api_base_url(environment)).prepare()
try:
auth(probe)
except OAuth2AuthRequestError as e:
if e.is_permanent:
- return f"Your Ramp developer app needs the `{scopes}` scope to sync this table."
+ return f"Your Ramp developer app needs the `{scope}` scope to sync this table."
except requests.RequestException:
pass
return None
@@ -153,7 +163,7 @@ def ramp_source(
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
config = RAMP_ENDPOINTS[endpoint]
- auth = _make_auth(environment, client_id, client_secret, config.scopes or TOKEN_SCOPES)
+ auth = _make_auth(environment, client_id, client_secret, config.scope)
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
@@ -165,6 +175,41 @@ def ramp_source(
):
params[config.incremental_param] = _format_timestamp(db_incremental_field_last_value)
+ client_config: ClientConfig = {
+ "base_url": _api_base_url(environment),
+ "auth": auth,
+ # Pin every request — including page.next links and the seeded resume URL — to the
+ # configured Ramp host so a tampered next/resume link can't exfiltrate the bearer token.
+ "allowed_hosts": [],
+ }
+
+ if config.fanout is not None:
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(RampResumeConfig(fanout_state=dict(state)))
+
+ items = build_dependent_resource(
+ endpoint_configs=RAMP_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=config.fanout,
+ client_config=client_config,
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra={"data_selector": "data", "paginator": RampPaginator()},
+ child_endpoint_extra={"data_selector": "data", "paginator": RampPaginator()},
+ page_size_param="page_size",
+ resume_hook=save_fanout_checkpoint,
+ initial_paginator_state=resume.fanout_state if resume is not None else None,
+ )
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: items,
+ primary_keys=config.primary_keys,
+ )
+
initial_paginator_state: Optional[dict[str, Any]] = None
if resume is not None and resume.next_url is not None:
initial_paginator_state = {"next_url": resume.next_url}
@@ -181,13 +226,6 @@ def ramp_source(
"data_selector": "data",
"paginator": RampPaginator(),
}
- client_config: ClientConfig = {
- "base_url": _api_base_url(environment),
- "auth": auth,
- # Pin every request — including page.next links and the seeded resume URL — to the
- # configured Ramp host so a tampered next/resume link can't exfiltrate the bearer token.
- "allowed_hosts": [],
- }
rest_config: RESTAPIConfig = {
"client": client_config,
"resources": [{"name": endpoint, "endpoint": endpoint_config}],
@@ -204,7 +242,7 @@ def ramp_source(
return SourceResponse(
name=endpoint,
items=lambda: resource,
- primary_keys=[config.primary_key],
+ primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
partition_mode="datetime" if config.partition_key else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py
index c4c337f51b69..e149b1b5eb01 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/settings.py
@@ -1,11 +1,16 @@
from dataclasses import dataclass, field
from typing import Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-# Scopes requested when minting the client-credentials token. Scopes are fixed
-# on the developer app at creation, so the caption tells users to grant these.
+# Scopes requested when validating a new connection. Scopes are fixed on the
+# developer app at creation, so the caption tells users to grant these.
TOKEN_SCOPES = "transactions:read reimbursements:read users:read cards:read departments:read"
+# Ramp list pages cap at 100 items.
+PAGE_SIZE = 100
@dataclass(frozen=True)
@@ -13,21 +18,34 @@ class RampEndpointConfig:
name: str
# Path under {host}/developer/v1.
path: str
- primary_key: str = "id"
+ # The token for a sync requests only this scope: Ramp rejects a mint that asks for a
+ # scope the app lacks, so an app without a newer scope still syncs its other tables.
+ scope: str
+ primary_keys: list[str] = field(default_factory=lambda: ["id"])
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Query param that pushes the incremental cursor server-side.
incremental_param: Optional[str] = None
# Stable event-time field used for datetime partitioning.
partition_key: Optional[str] = None
- # Scopes minted for this endpoint instead of TOKEN_SCOPES. Ramp rejects a token request for a
- # scope the app lacks, so a newer endpoint asks only for its own scope and existing apps keep working.
- scopes: Optional[str] = None
+ fanout: Optional[DependentEndpointConfig] = None
+ page_size: int = PAGE_SIZE
+ default_incremental_field: Optional[str] = None
+
+
+_VENDORS_FANOUT = DependentEndpointConfig(
+ parent_name="vendors",
+ resolve_param="vendor_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "vendor_id"},
+)
RAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {
"transactions": RampEndpointConfig(
name="transactions",
path="/transactions",
+ scope="transactions:read",
partition_key="user_transaction_time",
# from_date filters on the transaction time. There is no updated_after,
# so late state changes (pending→cleared) need an occasional full
@@ -45,23 +63,68 @@ RAMP_ENDPOINTS: dict[str, RampEndpointConfig] = {
"reimbursements": RampEndpointConfig(
name="reimbursements",
path="/reimbursements",
+ scope="reimbursements:read",
),
"users": RampEndpointConfig(
name="users",
path="/users",
+ scope="users:read",
),
"cards": RampEndpointConfig(
name="cards",
path="/cards",
+ scope="cards:read",
),
"departments": RampEndpointConfig(
name="departments",
path="/departments",
+ scope="departments:read",
+ ),
+ # Bills and vendors change after creation (payment status, approvals, details), but the
+ # bills endpoints only filter on created_at and vendors.updated_at is nullable, so these
+ # lookups stay full refresh rather than risk silently missing changed or new rows.
+ "vendors": RampEndpointConfig(
+ name="vendors",
+ path="/vendors",
+ scope="vendors:read",
+ partition_key="created_at",
+ ),
+ "vendor_contacts": RampEndpointConfig(
+ name="vendor_contacts",
+ path="/vendors/{vendor_id}/contacts",
+ scope="vendors:read",
+ primary_keys=["vendor_id", "id"],
+ fanout=_VENDORS_FANOUT,
+ ),
+ "merchants": RampEndpointConfig(
+ name="merchants",
+ path="/merchants",
+ scope="merchants:read",
+ ),
+ "bills": RampEndpointConfig(
+ name="bills",
+ path="/bills",
+ scope="bills:read",
+ partition_key="created_at",
+ ),
+ "bill_drafts": RampEndpointConfig(
+ name="bill_drafts",
+ path="/bills/drafts",
+ scope="bills:read",
+ partition_key="created_at",
+ ),
+ "accounting_accounts": RampEndpointConfig(
+ name="accounting_accounts",
+ path="/accounting/accounts",
+ scope="accounting:read",
+ # `id` is the ERP's remote ID and can be absent; `ramp_id` is Ramp's own unique ID.
+ primary_keys=["ramp_id"],
+ partition_key="created_at",
),
"billing_statements": RampEndpointConfig(
name="billing_statements",
path="/billing/statements",
- scopes="billing_statements:read",
+ scope="billing_statements:read",
),
}
@@ -70,3 +133,9 @@ ENDPOINTS = tuple(RAMP_ENDPOINTS.keys())
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in RAMP_ENDPOINTS.items() if config.incremental_fields
}
+
+# Credential validation only proves TOKEN_SCOPES, so tables needing any other scope start
+# unselected rather than failing every sync on apps that were never granted it.
+SHOULD_SYNC_DEFAULT: dict[str, bool] = {
+ name: False for name, config in RAMP_ENDPOINTS.items() if config.scope not in TOKEN_SCOPES.split()
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py
index 62ab5bdb8e7d..5a3aef1e5751 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/source.py
@@ -31,7 +31,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.ramp
ramp_source,
validate_credentials as validate_ramp_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.settings import ENDPOINTS, INCREMENTAL_FIELDS
+from products.warehouse_sources.backend.temporal.data_imports.sources.ramp.settings import (
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ SHOULD_SYNC_DEFAULT,
+)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -79,7 +83,9 @@ class RampSource(ResumableSource[RampSourceConfig, RampResumeConfig]):
label="Ramp",
caption="""Connect your Ramp account to pull your spend data into the PostHog Data warehouse.
-A Ramp admin can create a developer app under Settings > Developer API. Grant it the `transactions:read`, `reimbursements:read`, `users:read`, `cards:read`, and `departments:read` scopes, plus `billing_statements:read` to sync billing statements (scopes are fixed at app creation) and enable client credentials, then enter the app's client ID and secret here.""",
+A Ramp admin can create a developer app under Settings > Developer API. Grant it the `transactions:read`, `reimbursements:read`, `users:read`, `cards:read`, and `departments:read` scopes (scopes are fixed at app creation) and enable client credentials, then enter the app's client ID and secret here.
+
+To also sync billing statements, vendors, merchants, bills, or general ledger accounts, grant `billing_statements:read`, `vendors:read`, `merchants:read`, `bills:read`, or `accounting:read` as well.""",
iconPath="/static/services/ramp.png",
docsUrl="https://posthog.com/docs/cdp/sources/ramp",
releaseStatus=ReleaseStatus.GA,
@@ -125,12 +131,12 @@ A Ramp admin can create a developer app under Settings > Developer API. Grant it
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, should_sync_default=SHOULD_SYNC_DEFAULT)
def validate_credentials(
self, config: RampSourceConfig, team_id: int, schema_name: Optional[str] = None, api_version: str | None = None
) -> tuple[bool, str | None]:
- return validate_ramp_credentials(config.environment, config.client_id, config.client_secret)
+ return validate_ramp_credentials(config.environment, config.client_id, config.client_secret, schema_name)
def get_endpoint_permissions(
self, config: RampSourceConfig, team_id: int, endpoints: list[str], api_version: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py
index 0ea44b102f08..9af3ab931be0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ramp/tests/test_ramp.py
@@ -67,16 +67,23 @@ class TestFormatTimestamp:
class TestValidateCredentials:
- def test_mint_requests_documented_scopes(self):
+ @pytest.mark.parametrize(
+ "schema_name, expected_scope",
+ [
+ (None, TOKEN_SCOPES),
+ ("bills", "bills:read"),
+ ],
+ )
+ def test_mint_requests_documented_scopes(self, schema_name, expected_scope):
with scripted_network([_token()]) as network:
- validate_credentials("production", "cid", "sec")
+ validate_credentials("production", "cid", "sec", schema_name)
request = network.requests_log[0]
# HTTP Basic client auth: credentials ride in the auth header, scopes in the form body.
assert request.url == TOKEN_URL
assert parse_qs((request.body or b"").decode()) == {
"grant_type": ["client_credentials"],
- "scope": [TOKEN_SCOPES],
+ "scope": [expected_scope],
}
assert request.headers["authorization"] == "Basic " + base64.b64encode(b"cid:sec").decode()
@@ -173,3 +180,27 @@ class TestGetRows:
assert isinstance(result.raised, ValueError)
assert not any("evil.example.com" in url for url in result.urls)
assert result.saved_states == []
+
+ def test_vendor_contacts_fan_out_over_vendors_with_a_vendor_scoped_token(self):
+ script = route(
+ {
+ "/token": [_token()],
+ "/developer/v1/vendors": [
+ _page([{"id": "v1"}], next_url="https://api.ramp.com/developer/v1/vendors?start=v1"),
+ _page([{"id": "v2"}]),
+ ],
+ "/vendors/v1/contacts": [_page([{"id": "c1"}])],
+ "/vendors/v2/contacts": [
+ _page([{"id": "c2"}], next_url="https://api.ramp.com/developer/v1/vendors/v2/contacts?start=c2"),
+ _page([{"id": "c3"}]),
+ ],
+ }
+ )
+ config = RampSourceConfig(environment="production", client_id="cid", client_secret="sec")
+ result = SourceDriver(RampSource(), config).run("vendor_contacts", script)
+
+ assert result.raised is None
+ assert [(row["vendor_id"], row["id"]) for row in result.rows] == [("v1", "c1"), ("v2", "c2"), ("v2", "c3")]
+ token_request = next(request for request in result.requests if request.path.endswith("/token"))
+ token_body = parse_qs((token_request.body or b"").decode())
+ assert token_body["scope"] == ["vendors:read"]