Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add razorpay payment links, recon and razorpayx tables (#126985)

Tom Owersbcfd3f7a1e80
diff --git a/posthog/test/repo_invariants/source_test_mocked_resume_manager_baseline.txt b/posthog/test/repo_invariants/source_test_mocked_resume_manager_baseline.txt
index 98f415594f89..be6b6f8048af 100644
--- a/posthog/test/repo_invariants/source_test_mocked_resume_manager_baseline.txt
+++ b/posthog/test/repo_invariants/source_test_mocked_resume_manager_baseline.txt
@@ -41,7 +41,6 @@ products/warehouse_sources/backend/temporal/data_imports/sources/moengage/tests/
products/warehouse_sources/backend/temporal/data_imports/sources/nager_date/tests/test_nager_date.py
products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py
products/warehouse_sources/backend/temporal/data_imports/sources/proofpoint_tap/tests/test_proofpoint_tap.py
-products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/tests/test_razorpay_source.py
products/warehouse_sources/backend/temporal/data_imports/sources/scalr/tests/test_scalr.py
products/warehouse_sources/backend/temporal/data_imports/sources/sentry/tests/test_sentry.py
products/warehouse_sources/backend/temporal/data_imports/sources/servicem8/tests/test_servicem8_source.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 0be057813c84..d5c8532cb448 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10056,13 +10056,13 @@ Today (11): `Customers`, `Disputes`, `Invoices`, `Items`, `Orders`, `Payments`,
Diffed against: <https://razorpay.com/docs/build/sitemap/razorpay/IN/urls.txt>
-- [ ] `settlements/recon (settlement reconciliation report)` — Per-transaction settlement breakdown with fees and tax - the headline reconciliation report for finance users, not derivable from the settlement summary we sync (high)
-- [ ] `payment_links` — Payment Links are a primary collection channel with their own conversion funnel and are entirely absent (high)
+- [x] `settlements/recon (settlement reconciliation report)` — Per-transaction settlement breakdown with fees and tax - the headline reconciliation report for finance users, not derivable from the settlement summary we sync (high)
+- [x] `payment_links` — Payment Links are a primary collection channel with their own conversion funnel and are entirely absent (high)
- [ ] `transfers and reversals (Route)` — Marketplace split-payment transfers and their reversals - the line-item breakdown of how each payment was divided (medium)
- [ ] `qr_codes` — QR code collection channel and its associated payments (medium)
- [ ] `linked_accounts (Route)` — Lookup resolving the linked-account IDs that Route transfers point at (medium)
- [ ] `payouts (RazorpayX)` — Outbound money movement; the payables counterpart to the payments already synced (medium)
-- [ ] `x/transactions (RazorpayX)` — Bank account transaction ledger for balance and cash-flow reporting (medium)
+- [x] `x/transactions (RazorpayX)` — Bank account transaction ledger for balance and cash-flow reporting (medium)
- [ ] `contacts and fund_accounts (RazorpayX)` — Lookup tables resolving the payee IDs on payouts (medium)
- [ ] `subscriptions add-ons` — Line-item charges added to subscriptions we already sync, needed for correct MRR math (medium)
- [ ] `settlements/instant` — Instant settlement records with their own fee structure, separate from standard settlements (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv b/products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv
index c7cbe201e682..b8dabff3f67f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv
@@ -1023,7 +1023,7 @@ RakutenAdvertising,,,,,,rakuten-advertising
Ramp,environment client_id client_secret,,,,,ramp
Rapid7Insightvm,api_key region,,,,,rapid7-insightvm
Raygun,personal_access_token,,,,,raygun
-Razorpay,key_id key_secret,,,,,razorpay
+Razorpay,key_id key_secret razorpayx_account_number,,,,,razorpay
RecallAI,api_key region,,,,,recall-ai
Recharge,api_key,,,,,recharge
Recreation,api_key,,,,,recreation
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/razorpay.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/razorpay.py
index 952e046fbef2..27e4bde9ec60 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/razorpay.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/razorpay.py
@@ -8,3 +8,4 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp
class RazorpaySourceConfig(config.Config):
key_id: str
key_secret: str
+ razorpayx_account_number: str | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/canonical_descriptions.py
index 0a5720ab9434..11a4ed7b6e0c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/canonical_descriptions.py
@@ -82,6 +82,32 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "UNIX timestamp (seconds) when the order was created.",
},
},
+ "PaymentLinks": {
+ "description": "A Payment Link shared with a customer to collect a payment, with its amount, amount paid, status, and the captured payments made against it.",
+ "docs_url": "https://razorpay.com/docs/api/payments/payment-links/fetch-all-standard/",
+ "columns": {
+ "id": "Unique identifier of the Payment Link.",
+ "amount": "Amount to be paid using the Payment Link, in the smallest currency sub-unit (e.g. paise).",
+ "amount_paid": "Amount paid by the customer so far.",
+ "currency": "ISO currency code of the Payment Link.",
+ "accept_partial": "Whether the customer can make partial payments.",
+ "first_min_partial_amount": "Minimum amount the customer must pay as the first partial payment.",
+ "status": "State of the Payment Link: created, partially_paid, expired, cancelled, or paid.",
+ "description": "Description of the Payment Link.",
+ "reference_id": "Your unique reference number for the Payment Link.",
+ "customer": "Name, email, and contact number of the customer.",
+ "payments": "Captured payments made against the Payment Link, or null until one is captured.",
+ "short_url": "Short URL generated for the Payment Link.",
+ "upi_link": "Whether the Payment Link is a UPI Payment Link.",
+ "expire_by": "UNIX timestamp (seconds) when the Payment Link expires.",
+ "expired_at": "UNIX timestamp (seconds) when the Payment Link expired.",
+ "cancelled_at": "UNIX timestamp (seconds) when the Payment Link was cancelled.",
+ "notes": "Key-value pairs of additional information about the Payment Link.",
+ "user_id": "Identifier of the user role that created the Payment Link.",
+ "created_at": "UNIX timestamp (seconds) when the Payment Link was created.",
+ "updated_at": "UNIX timestamp (seconds) when the Payment Link was last updated.",
+ },
+ },
"Payments": {
"description": "A payment made by a customer, covering every payment method (card, UPI, netbanking, wallet) and its full lifecycle status.",
"docs_url": "https://razorpay.com/docs/api/payments/",
@@ -136,6 +162,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "UNIX timestamp (seconds) when the refund was created.",
},
},
+ "RazorpayXTransactions": {
+ "description": "A credit or debit on a RazorpayX business account, such as a payout or a bank transfer that added funds, with the running balance.",
+ "docs_url": "https://razorpay.com/docs/api/x/transactions/fetch-all/",
+ "columns": {
+ "id": "Unique identifier of the transaction.",
+ "account_number": "RazorpayX business account number the transaction belongs to.",
+ "amount": "Amount transferred, in paise, excluding fees and tax.",
+ "currency": "ISO currency code of the transaction.",
+ "credit": "Amount credited to the account, in paise. 0 for debits.",
+ "debit": "Amount debited from the account, in paise. 0 for credits.",
+ "balance": "Account balance, in paise, after the transaction.",
+ "source": "The payout or bank transfer that created the transaction.",
+ "created_at": "UNIX timestamp (seconds) when the transaction was created.",
+ },
+ },
"Settlements": {
"description": "A settlement of captured payments transferred to your bank account, with fees, tax, and the bank UTR reference.",
"docs_url": "https://razorpay.com/docs/api/settlements/",
@@ -149,6 +190,36 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "UNIX timestamp (seconds) when the settlement was created.",
},
},
+ "SettlementRecon": {
+ "description": "One settled transaction (payment, refund, transfer, or adjustment) with its fee, tax, and the settlement it was paid out in, from the settlement reconciliation report.",
+ "docs_url": "https://razorpay.com/docs/api/settlements/fetch-recon/",
+ "columns": {
+ "entity_id": "Identifier of the settled payment, refund, transfer, or adjustment.",
+ "type": "Type of the transaction: payment, refund, transfer, or adjustment.",
+ "debit": "Amount debited from your account for the transaction, in the smallest currency sub-unit.",
+ "credit": "Amount credited to your account for the transaction, in the smallest currency sub-unit.",
+ "amount": "Transaction amount, in the smallest currency sub-unit.",
+ "currency": "ISO currency code of the transaction.",
+ "fee": "Fee charged for the transaction, including tax.",
+ "tax": "Tax charged on the fee.",
+ "on_hold": "Whether the settlement of the transaction is on hold.",
+ "settled": "Whether the transaction is settled.",
+ "settlement_id": "Identifier of the settlement the transaction was paid out in.",
+ "settlement_utr": "Bank UTR reference of the settlement.",
+ "payment_id": "Identifier of the payment a refund or transfer belongs to.",
+ "order_id": "Identifier of the order linked to the transaction.",
+ "order_receipt": "Receipt of the order linked to the transaction.",
+ "method": "Payment method, such as card, netbanking, wallet, or upi.",
+ "card_network": "Card network for card payments.",
+ "card_issuer": "Card issuer for card payments.",
+ "card_type": "Card type for card payments, such as credit or debit.",
+ "dispute_id": "Identifier of the dispute linked to the transaction.",
+ "description": "Description of the transaction.",
+ "notes": "Key-value pairs of additional information about the transaction.",
+ "created_at": "UNIX timestamp (seconds) when the transaction was created.",
+ "settled_at": "UNIX timestamp (seconds) when the transaction was settled.",
+ },
+ },
"Subscriptions": {
"description": "A recurring billing relationship between a customer and a plan, with its cycle counts and lifecycle status.",
"docs_url": "https://razorpay.com/docs/api/payments/subscriptions/",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/razorpay.py b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/razorpay.py
index 9530897ffaf6..f49b504b9fa0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/razorpay.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/razorpay.py
@@ -1,21 +1,30 @@
import dataclasses
+from collections.abc import Iterator
+from datetime import datetime
from typing import Any, Optional
+from zoneinfo import ZoneInfo
+
+from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
+ RESTClient,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import HttpBasicAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
OffsetPaginator,
+ SinglePagePaginator,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.razorpay.settings import (
API_BASE_URL,
ENDPOINT_CONFIGS,
PAGE_SIZE,
+ RazorpayEndpointConfig,
)
# Razorpay has no updated-at filter, so incremental syncs window on created_at only. Re-read a
@@ -27,10 +36,41 @@ INCREMENTAL_LOOKBACK_SECONDS = 3 * 24 * 60 * 60
# The API rejects `from` values before 2000-01-01 with a 400.
MIN_FROM_TIMESTAMP = 946_684_800
+# The recon report has no "all months" query, so a full sync walks every month from before
+# Razorpay's launch to now. Months are settlement months on Razorpay's IST calendar.
+RECON_TIMEZONE = ZoneInfo("Asia/Kolkata")
+
+
+@frozen(order=True)
+class ReconMonth:
+ year: int
+ month: int
+
+ @classmethod
+ def at(cls, timestamp: float) -> "ReconMonth":
+ moment = datetime.fromtimestamp(timestamp, tz=RECON_TIMEZONE)
+ return cls(year=moment.year, month=moment.month)
+
+ def next(self) -> "ReconMonth":
+ if self.month == 12:
+ return ReconMonth(year=self.year + 1, month=1)
+ return ReconMonth(year=self.year, month=self.month + 1)
+
+
+RECON_FIRST_MONTH = ReconMonth(year=2014, month=1)
+
+MISSING_ACCOUNT_NUMBER_ERROR = (
+ "Add your RazorpayX account number to the Razorpay source settings to sync RazorpayX transactions."
+)
+
+
[email protected](frozen=True)
class RazorpayResumeConfig:
skip: int
+ # Set only for the settlement recon report: the month that `skip` applies to.
+ recon_year: Optional[int] = None
+ recon_month: Optional[int] = None
def build_from_param(db_incremental_field_last_value: Optional[Any]) -> Optional[int]:
@@ -62,10 +102,13 @@ def get_resource(
name: str,
should_use_incremental_field: bool,
db_incremental_field_last_value: Optional[Any],
+ account_number: Optional[str] = None,
) -> EndpointResource:
config = ENDPOINT_CONFIGS[name]
params: dict[str, Any] = {}
+ if config.requires_account_number:
+ params["account_number"] = account_number
if config.supports_created_filter and should_use_incremental_field:
# None values are dropped before the request, so the first incremental sync (no
# watermark yet) walks the full history without a `from` bound.
@@ -81,7 +124,7 @@ def get_resource(
if should_use_incremental_field
else "replace",
"endpoint": {
- "data_selector": "items",
+ "data_selector": config.data_selector,
"path": config.path,
"params": params,
"paginator": _build_paginator(),
@@ -90,6 +133,61 @@ def get_resource(
}
+def _current_recon_month() -> ReconMonth:
+ return ReconMonth.at(datetime.now(tz=RECON_TIMEZONE).timestamp())
+
+
+def recon_start_month(db_incremental_field_last_value: Optional[Any]) -> ReconMonth:
+ from_timestamp = build_from_param(db_incremental_field_last_value)
+ if from_timestamp is None:
+ return RECON_FIRST_MONTH
+ return max(ReconMonth.at(from_timestamp), RECON_FIRST_MONTH)
+
+
+def _iter_settlement_recon(
+ key_id: str,
+ key_secret: str,
+ config: RazorpayEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[RazorpayResumeConfig],
+ db_incremental_field_last_value: Optional[Any],
+) -> Iterator[list[dict[str, Any]]]:
+ current = recon_start_month(db_incremental_field_last_value)
+ skip = 0
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume is not None and resume.recon_year is not None and resume.recon_month is not None:
+ current = ReconMonth(year=resume.recon_year, month=resume.recon_month)
+ skip = resume.skip
+
+ end = _current_recon_month()
+ client = RESTClient(
+ base_url=API_BASE_URL,
+ auth=HttpBasicAuth(username=key_id, password=key_secret),
+ paginator=SinglePagePaginator(),
+ )
+
+ try:
+ while current <= end:
+ params = {"year": current.year, "month": current.month, "count": PAGE_SIZE, "skip": skip}
+ page = next(client.paginate(config.path, params=params, data_selector=config.data_selector), [])
+ rows = [row for row in page if isinstance(row, dict)]
+
+ if len(page) < PAGE_SIZE:
+ current = current.next()
+ skip = 0
+ else:
+ skip += len(page)
+
+ resumable_source_manager.save_state(
+ RazorpayResumeConfig(skip=skip, recon_year=current.year, recon_month=current.month)
+ )
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+ finally:
+ client.session.close()
+
+
def razorpay_source(
key_id: str,
key_secret: str,
@@ -99,10 +197,36 @@ def razorpay_source(
resumable_source_manager: ResumableSourceManager[RazorpayResumeConfig],
db_incremental_field_last_value: Optional[Any],
should_use_incremental_field: bool = False,
-) -> Resource:
+ account_number: Optional[str] = None,
+) -> SourceResponse:
+ endpoint_config = ENDPOINT_CONFIGS[endpoint]
+ if endpoint_config.requires_account_number and not account_number:
+ raise ValueError(MISSING_ACCOUNT_NUMBER_ERROR)
+
+ if not should_use_incremental_field:
+ db_incremental_field_last_value = None
+
+ if endpoint_config.monthly_recon:
+ return SourceResponse(
+ name=endpoint_config.name,
+ items=lambda: _iter_settlement_recon(
+ key_id, key_secret, endpoint_config, resumable_source_manager, db_incremental_field_last_value
+ ),
+ primary_keys=list(endpoint_config.primary_keys),
+ partition_count=1,
+ partition_size=1,
+ partition_mode="datetime",
+ partition_format="week",
+ partition_keys=[endpoint_config.partition_key],
+ # Months are read oldest first, but the order of rows within a month is undocumented.
+ sort_mode=None,
+ )
+
config: RESTAPIConfig = {
"client": {
"base_url": API_BASE_URL,
+ # The RazorpayX account number rides in the query string, so mask it in request logs too.
+ "session": make_tracked_session(redact_values=tuple(v for v in (key_secret, account_number) if v)),
"auth": {
"type": "http_basic",
"username": key_id,
@@ -121,7 +245,8 @@ def razorpay_source(
get_resource(
endpoint,
should_use_incremental_field,
- db_incremental_field_last_value if should_use_incremental_field else None,
+ db_incremental_field_last_value,
+ account_number,
)
],
}
@@ -138,7 +263,7 @@ def razorpay_source(
if state and state.get("offset") is not None:
resumable_source_manager.save_state(RazorpayResumeConfig(skip=int(state["offset"])))
- return rest_api_resource(
+ resource = rest_api_resource(
config,
team_id,
job_id,
@@ -146,6 +271,19 @@ def razorpay_source(
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
+ return SourceResponse(
+ name=resource.name,
+ items=lambda: resource,
+ primary_keys=list(endpoint_config.primary_keys),
+ column_hints=resource.column_hints,
+ partition_count=1,
+ partition_size=1,
+ partition_mode="datetime",
+ partition_format="week",
+ partition_keys=[endpoint_config.partition_key],
+ # List endpoints return newest-first and expose no sort param.
+ sort_mode="desc",
+ )
def validate_credentials(key_id: str, key_secret: str) -> bool:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/settings.py
index ebcaa58d107c..d2fc371ba455 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/settings.py
@@ -15,8 +15,18 @@ class RazorpayEndpointConfig:
# Whether the endpoint documents the standard `from`/`to` created_at window filters.
# Endpoints without them sync as full refresh only.
supports_created_filter: bool
- primary_key: str = "id"
+ primary_keys: tuple[str, ...] = ("id",)
partition_key: str = "created_at"
+ data_selector: str = "items"
+ # RazorpayX endpoints are scoped to one business account via the `account_number` param.
+ requires_account_number: bool = False
+ # The settlement recon report only lists one settlement month per request (`year`/`month`).
+ monthly_recon: bool = False
+ incremental_field: str = "created_at"
+
+ @property
+ def supports_incremental(self) -> bool:
+ return self.supports_created_filter or self.monthly_recon
ENDPOINT_CONFIGS: dict[str, RazorpayEndpointConfig] = {
@@ -49,6 +59,14 @@ ENDPOINT_CONFIGS: dict[str, RazorpayEndpointConfig] = {
path="/v1/orders",
supports_created_filter=True,
),
+ # The payment links docs list no `count`/`skip` or `from`/`to`, but the official SDKs page with
+ # `count`/`skip`. `from`/`to` stays unused until it is documented.
+ "PaymentLinks": RazorpayEndpointConfig(
+ name="PaymentLinks",
+ path="/v1/payment_links",
+ supports_created_filter=False,
+ data_selector="payment_links",
+ ),
"Payments": RazorpayEndpointConfig(
name="Payments",
path="/v1/payments",
@@ -64,11 +82,27 @@ ENDPOINT_CONFIGS: dict[str, RazorpayEndpointConfig] = {
path="/v1/refunds",
supports_created_filter=True,
),
+ "RazorpayXTransactions": RazorpayEndpointConfig(
+ name="RazorpayXTransactions",
+ path="/v1/transactions",
+ supports_created_filter=True,
+ requires_account_number=True,
+ ),
"Settlements": RazorpayEndpointConfig(
name="Settlements",
path="/v1/settlements",
supports_created_filter=True,
),
+ # One row per settled payment, refund, transfer or adjustment. `entity_id` identifies the
+ # transaction and `settlement_id` the settlement it was paid out in.
+ "SettlementRecon": RazorpayEndpointConfig(
+ name="SettlementRecon",
+ path="/v1/settlements/recon/combined",
+ supports_created_filter=False,
+ primary_keys=("entity_id", "settlement_id"),
+ monthly_recon=True,
+ incremental_field="settled_at",
+ ),
"Subscriptions": RazorpayEndpointConfig(
name="Subscriptions",
path="/v1/subscriptions",
@@ -84,17 +118,17 @@ ENDPOINT_CONFIGS: dict[str, RazorpayEndpointConfig] = {
ENDPOINTS = tuple(ENDPOINT_CONFIGS.keys())
-# Razorpay only filters on created_at (there is no updated-at filter), advertised as a DateTime
-# cursor stored as a UNIX-seconds integer.
+# Razorpay only filters on created_at (there is no updated-at filter), and the recon report only on
+# settlement month. Both are advertised as a DateTime cursor stored as a UNIX-seconds integer.
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: [
{
- "label": "created_at",
+ "label": config.incremental_field,
"type": IncrementalFieldType.DateTime,
- "field": "created_at",
+ "field": config.incremental_field,
"field_type": IncrementalFieldType.Integer,
}
]
for name, config in ENDPOINT_CONFIGS.items()
- if config.supports_created_filter
+ if config.supports_incremental
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/source.py
index 342dd9cb7a99..f25ccfdc8350 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/source.py
@@ -23,6 +23,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
RazorpaySourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.razorpay.razorpay import (
+ MISSING_ACCOUNT_NUMBER_ERROR,
RazorpayResumeConfig,
razorpay_source,
validate_credentials as validate_razorpay_credentials,
@@ -76,6 +77,7 @@ class RazorpaySource(ResumableSource[RazorpaySourceConfig, RazorpayResumeConfig]
return {
"401 Client Error: Unauthorized for url: https://api.razorpay.com": "Razorpay authentication failed. Please check your key ID and key secret.",
"400 Client Error: Bad Request for url: https://api.razorpay.com": None,
+ MISSING_ACCOUNT_NUMBER_ERROR: MISSING_ACCOUNT_NUMBER_ERROR,
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
@@ -96,6 +98,17 @@ class RazorpaySource(ResumableSource[RazorpaySourceConfig, RazorpayResumeConfig]
) -> list[SourceSchema]:
return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ def get_endpoint_permissions(
+ self, config: RazorpaySourceConfig, team_id: int, endpoints: list[str], api_version: str | None = None
+ ) -> dict[str, str | None]:
+ return {
+ endpoint: MISSING_ACCOUNT_NUMBER_ERROR
+ if ENDPOINT_CONFIGS[endpoint].requires_account_number and not config.razorpayx_account_number
+ else None
+ for endpoint in endpoints
+ if endpoint in ENDPOINT_CONFIGS
+ }
+
def validate_credentials(
self,
config: RazorpaySourceConfig,
@@ -120,8 +133,7 @@ class RazorpaySource(ResumableSource[RazorpaySourceConfig, RazorpayResumeConfig]
resumable_source_manager: ResumableSourceManager[RazorpayResumeConfig],
inputs: SourceInputs,
) -> SourceResponse:
- endpoint_config = ENDPOINT_CONFIGS[inputs.schema_name]
- resource = razorpay_source(
+ return razorpay_source(
key_id=config.key_id,
key_secret=config.key_secret,
endpoint=inputs.schema_name,
@@ -132,19 +144,7 @@ class RazorpaySource(ResumableSource[RazorpaySourceConfig, RazorpayResumeConfig]
db_incremental_field_last_value=inputs.db_incremental_field_last_value
if inputs.should_use_incremental_field
else None,
- )
- return SourceResponse(
- name=resource.name,
- items=lambda: resource,
- primary_keys=[endpoint_config.primary_key],
- column_hints=resource.column_hints,
- partition_count=1,
- partition_size=1,
- partition_mode="datetime",
- partition_format="week",
- partition_keys=[endpoint_config.partition_key],
- # List endpoints return newest-first and expose no sort param.
- sort_mode="desc",
+ account_number=config.razorpayx_account_number,
)
@property
@@ -176,6 +176,15 @@ class RazorpaySource(ResumableSource[RazorpaySourceConfig, RazorpayResumeConfig]
placeholder="",
secret=True,
),
+ SourceFieldInputConfig(
+ name="razorpayx_account_number",
+ label="RazorpayX account number",
+ type=SourceFieldInputConfigType.TEXT,
+ required=False,
+ placeholder="7878780080316316",
+ caption="Only needed to sync RazorpayX transactions. Find it in the RazorpayX Dashboard under My Account & Settings → Banking. Live mode and test mode use different numbers.",
+ secret=False,
+ ),
],
),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/tests/test_razorpay.py b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/tests/test_razorpay.py
index bde3a1913e0a..7b5e08e0b61e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/tests/test_razorpay.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/tests/test_razorpay.py
@@ -1,6 +1,8 @@
+from datetime import datetime
from typing import Any, Optional
import pytest
+from unittest.mock import patch
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
DriveResult,
@@ -15,7 +17,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
from products.warehouse_sources.backend.temporal.data_imports.sources.razorpay.razorpay import (
INCREMENTAL_LOOKBACK_SECONDS,
MIN_FROM_TIMESTAMP,
+ MISSING_ACCOUNT_NUMBER_ERROR,
+ RECON_TIMEZONE,
RazorpayResumeConfig,
+ ReconMonth,
build_from_param,
validate_credentials,
)
@@ -34,12 +39,28 @@ def _items(count: int, start: int = 0) -> list[dict[str, Any]]:
def _drive(
responses: list[ScriptedResponse | BaseException],
endpoint: str = "Payments",
+ account_number: Optional[str] = None,
**inputs: Any,
) -> DriveResult:
- config = RazorpaySourceConfig(key_id="rzp_test_key", key_secret="secret")
+ config = RazorpaySourceConfig(key_id="rzp_test_key", key_secret="secret", razorpayx_account_number=account_number)
return SourceDriver(RazorpaySource(), config).run(endpoint, responses, **inputs)
+def _ist_timestamp(year: int, month: int, day: int) -> int:
+ return int(datetime(year, month, day, tzinfo=RECON_TIMEZONE).timestamp())
+
+
+def _recon_items(count: int) -> list[dict[str, Any]]:
+ return [{"entity_id": f"pay_{i}", "settlement_id": "setl_1", "created_at": 1_750_000_000} for i in range(count)]
+
+
+def _current_month(year: int, month: int) -> Any:
+ return patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.razorpay.razorpay._current_recon_month",
+ return_value=ReconMonth(year=year, month=month),
+ )
+
+
class TestRazorpayPagination:
def test_resume_seeds_skip_from_saved_state(self) -> None:
result = _drive([_collection_response(_items(1))], resume_state=RazorpayResumeConfig(skip=200))
@@ -94,6 +115,99 @@ class TestRazorpayIncremental:
assert build_from_param(last_value) == expected
+class TestRazorpaySettlementRecon:
+ def test_walks_each_month_page_by_page_up_to_the_current_month(self) -> None:
+ with _current_month(2026, 2):
+ result = _drive(
+ [
+ _collection_response(_recon_items(PAGE_SIZE)),
+ _collection_response(_recon_items(2)),
+ _collection_response([]),
+ ],
+ endpoint="SettlementRecon",
+ incremental_field="settled_at",
+ db_incremental_field_last_value=_ist_timestamp(2026, 1, 20),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/v1/settlements/recon/combined"] * 3
+ assert list(zip(result.params("year"), result.params("month"), result.params("skip"))) == [
+ ("2026", "1", "0"),
+ ("2026", "1", str(PAGE_SIZE)),
+ ("2026", "2", "0"),
+ ]
+ assert len(result.rows) == PAGE_SIZE + 2
+ assert result.committed_states[-1] == RazorpayResumeConfig(skip=0, recon_year=2026, recon_month=3)
+
+ @pytest.mark.parametrize(
+ ("current_month", "inputs", "expected_first_request"),
+ [
+ ((2014, 1), {}, ("2014", "1", "0")),
+ (
+ (2026, 2),
+ {"incremental_field": "settled_at", "db_incremental_field_last_value": _ist_timestamp(2026, 3, 2)},
+ ("2026", "2", "0"),
+ ),
+ (
+ (2025, 6),
+ {"resume_state": RazorpayResumeConfig(skip=200, recon_year=2025, recon_month=6)},
+ ("2025", "6", "200"),
+ ),
+ ],
+ )
+ def test_start_month(
+ self, current_month: tuple[int, int], inputs: dict[str, Any], expected_first_request: tuple[str, str, str]
+ ) -> None:
+ with _current_month(*current_month):
+ result = _drive([_collection_response([])], endpoint="SettlementRecon", **inputs)
+
+ assert result.raised is None
+ assert (result.params("year")[0], result.params("month")[0], result.params("skip")[0]) == expected_first_request
+
+
+class TestRazorpayPaymentLinks:
+ def test_pages_through_links_with_count_and_skip(self) -> None:
+ links = [{"id": f"plink_{i}", "created_at": 1_750_000_000} for i in range(PAGE_SIZE + 1)]
+
+ result = _drive(
+ [
+ ScriptedResponse(json={"payment_links": links[:PAGE_SIZE]}),
+ ScriptedResponse(json={"payment_links": links[PAGE_SIZE:]}),
+ ],
+ endpoint="PaymentLinks",
+ )
+
+ assert result.raised is None
+ assert result.params("skip") == ["0", str(PAGE_SIZE)]
+ assert result.params("count") == [str(PAGE_SIZE)] * 2
+ assert len(result.rows) == PAGE_SIZE + 1
+
+
+class TestRazorpayXTransactions:
+ def test_sends_account_number_on_every_page(self) -> None:
+ result = _drive(
+ [
+ _collection_response(_items(PAGE_SIZE)),
+ _collection_response(_items(1, start=PAGE_SIZE)),
+ ],
+ endpoint="RazorpayXTransactions",
+ account_number="7878780080316316",
+ incremental_field="created_at",
+ db_incremental_field_last_value=1_750_000_000,
+ )
+
+ assert result.params("account_number") == ["7878780080316316", "7878780080316316"]
+ assert "7878780080316316" in result.session_options[0]["redact_values"]
+ assert result.params("from") == [str(1_750_000_000 - INCREMENTAL_LOOKBACK_SECONDS)] * 2
+
+ def test_fails_without_an_account_number(self) -> None:
+ result = _drive([], endpoint="RazorpayXTransactions")
+
+ assert result.requests == []
+ assert result.raised is not None
+ assert MISSING_ACCOUNT_NUMBER_ERROR in str(result.raised)
+
+
class TestRazorpayValidateCredentials:
@pytest.mark.parametrize(
("status_code", "expected"),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/tests/test_razorpay_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/tests/test_razorpay_source.py
index b3ea931f3e25..24fd4cf8b32f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/tests/test_razorpay_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/razorpay/tests/test_razorpay_source.py
@@ -1,14 +1,12 @@
-from typing import Any, Optional
+from typing import Optional
import pytest
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.razorpay import (
RazorpaySourceConfig,
)
@@ -17,32 +15,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.razorpay.c
)
from products.warehouse_sources.backend.temporal.data_imports.sources.razorpay.razorpay import (
INCREMENTAL_LOOKBACK_SECONDS,
+ MISSING_ACCOUNT_NUMBER_ERROR,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.razorpay.settings import ENDPOINTS
from products.warehouse_sources.backend.temporal.data_imports.sources.razorpay.source import RazorpaySource
-def _source_inputs(
- schema_name: str = "Payments",
- should_use_incremental_field: bool = False,
- db_incremental_field_last_value: Optional[Any] = None,
-) -> SourceInputs:
- return SourceInputs(
- schema_name=schema_name,
- schema_id="schema_id",
- source_id="source_id",
- team_id=1,
- should_use_incremental_field=should_use_incremental_field,
- db_incremental_field_last_value=db_incremental_field_last_value,
- db_incremental_field_earliest_value=None,
- incremental_field="created_at" if should_use_incremental_field else None,
- incremental_field_type=None,
- job_id="job_id",
- logger=MagicMock(),
- reset_pipeline=False,
- )
-
-
class TestRazorpaySource:
def setup_method(self) -> None:
self.source = RazorpaySource()
@@ -93,55 +71,20 @@ class TestRazorpaySource:
assert response.partition_mode == "datetime"
assert response.partition_keys == ["created_at"]
- def test_canonical_descriptions_cover_only_known_endpoints(self) -> None:
- assert set(CANONICAL_DESCRIPTIONS.keys()) == set(ENDPOINTS)
-
-
-class TestRazorpaySourceForwarding:
- def setup_method(self) -> None:
- self.source = RazorpaySource()
- self.config = RazorpaySourceConfig(key_id="rzp_test_key", key_secret="secret")
-
@pytest.mark.parametrize(
- ("should_use_incremental_field", "last_value", "expected_last_value"),
- [
- (True, 1_750_000_000, 1_750_000_000),
- (False, 1_750_000_000, None),
- ],
+ ("account_number", "expected"),
+ [(None, MISSING_ACCOUNT_NUMBER_ERROR), ("7878780080316316", None)],
)
- def test_source_for_pipeline_plumbing(
- self,
- should_use_incremental_field: bool,
- last_value: Optional[int],
- expected_last_value: Optional[int],
+ def test_razorpayx_transactions_permission_needs_an_account_number(
+ self, account_number: Optional[str], expected: Optional[str]
) -> None:
- inputs = _source_inputs(
- should_use_incremental_field=should_use_incremental_field,
- db_incremental_field_last_value=last_value,
+ config = RazorpaySourceConfig(
+ key_id="rzp_test_key", key_secret="secret", razorpayx_account_number=account_number
)
- manager = MagicMock(spec=ResumableSourceManager)
- mock_resource = MagicMock()
- mock_resource.name = "Payments"
- mock_resource.column_hints = None
- with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.razorpay.source.razorpay_source",
- return_value=mock_resource,
- ) as mock_source:
- response = self.source.source_for_pipeline(self.config, manager, inputs)
+ permissions = self.source.get_endpoint_permissions(config, 1, ["Payments", "RazorpayXTransactions"])
- mock_source.assert_called_once_with(
- key_id="rzp_test_key",
- key_secret="secret",
- endpoint="Payments",
- team_id=1,
- job_id="job_id",
- resumable_source_manager=manager,
- should_use_incremental_field=should_use_incremental_field,
- db_incremental_field_last_value=expected_last_value,
- )
- assert response.name == "Payments"
- assert response.primary_keys == ["id"]
- assert response.sort_mode == "desc"
- assert response.partition_mode == "datetime"
- assert response.partition_keys == ["created_at"]
+ assert permissions == {"Payments": None, "RazorpayXTransactions": expected}
+
+ def test_canonical_descriptions_cover_only_known_endpoints(self) -> None:
+ assert set(CANONICAL_DESCRIPTIONS.keys()) == set(ENDPOINTS)