:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add kalshi portfolio tables (#122044)
Tom Owers468ec059a2dd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 3248942a5f2e..89626e0a00c4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -6763,10 +6763,10 @@ Diffed against: <https://docs.kalshi.com/openapi.yaml>- [x] `Market candlesticks (GET /trade-api/v2/markets/candlesticks)` — analyze price, volume, and liquidity trends using historical time buckets (high)- [x] `Historical markets (GET /trade-api/v2/historical/markets)` — retain archived contracts needed to interpret older trades and long-term market outcomes (high)- [x] `Historical trades (GET /trade-api/v2/historical/trades)` — recover trades before the historical cutoff for complete volume and price history (high)-- [ ] `Positions (GET /trade-api/v2/portfolio/positions; GET /trade-api/v2/historical/positions)` — measure account exposure, realized profit, and trading fees for current and archived positions (high)-- [ ] `Fills (GET /trade-api/v2/portfolio/fills; GET /trade-api/v2/historical/fills)` — analyze account execution prices, fees, and trading volume across current and archived fills (high)-- [ ] `Settlements (GET /trade-api/v2/portfolio/settlements)` — reconcile settled contracts with payouts and account trading results (high)-- [ ] `Orders (GET /trade-api/v2/portfolio/orders; GET /trade-api/v2/historical/orders)` — compare submitted, executed, and canceled orders across current and archived history (medium)+- [x] `Positions (GET /trade-api/v2/portfolio/positions; GET /trade-api/v2/historical/positions)` — measure account exposure, realized profit, and trading fees for current and archived positions (high)+- [x] `Fills (GET /trade-api/v2/portfolio/fills; GET /trade-api/v2/historical/fills)` — analyze account execution prices, fees, and trading volume across current and archived fills (high)+- [x] `Settlements (GET /trade-api/v2/portfolio/settlements)` — reconcile settled contracts with payouts and account trading results (high)+- [x] `Orders (GET /trade-api/v2/portfolio/orders; GET /trade-api/v2/historical/orders)` — compare submitted, executed, and canceled orders across current and archived history (medium)- [ ] `Market orderbook (GET /trade-api/v2/markets/{ticker}/orderbook)` — measure market depth and available liquidity at each price level (medium)- [ ] `Series fee changes (GET /trade-api/v2/series/fee_changes)` — explain trading cost changes across series and time (medium)- [x] `Weather index (GET /trade-api/v2/live_data/weather/{city})` — analyze the minute-resolution city temperature index behind hourly temperature markets (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.pyindex 7d955a01335e..ab29ea90d58c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py@@ -81,6 +81,7 @@ _AUTH_HEADER_NAMES: frozenset[str] = frozenset("x-auth-token","x-metabase-session","ob-token-v1",+ "kalshi-access-signature","cookie","set-cookie","proxy-authorization",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_http_sampling.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_http_sampling.pyindex 902f13ba6eab..171da6775d60 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_http_sampling.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_http_sampling.py@@ -444,6 +444,7 @@ def test_sample_payload_omits_streamed_response_body():"x-sn-apikey","x-auth-token","X-Metabase-Session",+ "KALSHI-ACCESS-SIGNATURE","Cookie","Set-Cookie","Proxy-Authorization",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/kalshi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/kalshi.pyindex 558256c6a879..0329525bf34d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/kalshi.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/kalshi.py@@ -6,4 +6,5 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp@config.configclass KalshiSourceConfig(config.Config):- pass+ api_key_id: str | None = None+ private_key: str | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.pyindex bebb36daee6b..7294767e7f4d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py@@ -45,6 +45,55 @@ _TRADE_COLUMNS = {"is_block_trade": "Whether the trade was negotiated as a block rather than matched on the book.",}+_FILL_COLUMNS = {+ "fill_id": "Unique identifier for the fill.",+ "trade_id": "Legacy name for fill_id.",+ "order_id": "Identifier of the order that produced this fill.",+ "ticker": "Ticker of the market the fill is in.",+ "outcome_side": "Outcome side the fill positioned the account for, yes or no.",+ "book_side": "Order book side of the fill, bid or ask.",+ "count_fp": "Number of contracts filled.",+ "yes_price_dollars": "Fill price for the yes side, in dollars.",+ "no_price_dollars": "Fill price for the no side, in dollars.",+ "is_taker": "True when the fill removed liquidity from the order book.",+ "fee_cost": "Fee charged for the fill, in dollars.",+ "created_time": "When the fill executed.",+ "subaccount_number": "Subaccount the fill belongs to. 0 is the primary account.",+}++_ORDER_COLUMNS = {+ "order_id": "Unique identifier for the order.",+ "client_order_id": "Identifier the client set when it placed the order.",+ "ticker": "Ticker of the market the order is in.",+ "outcome_side": "Outcome side the order is positioned for, yes or no.",+ "book_side": "Order book side of the order, bid or ask.",+ "type": "Order type, limit or market.",+ "status": "Order status, for example resting, canceled, or executed.",+ "yes_price_dollars": "Limit price for the yes side, in dollars.",+ "no_price_dollars": "Limit price for the no side, in dollars.",+ "initial_count_fp": "Number of contracts the order asked for.",+ "fill_count_fp": "Number of contracts filled so far.",+ "remaining_count_fp": "Number of contracts still resting.",+ "taker_fill_cost_dollars": "Cost of the contracts filled as taker, in dollars.",+ "maker_fill_cost_dollars": "Cost of the contracts filled as maker, in dollars.",+ "taker_fees_dollars": "Fees paid on contracts filled as taker, in dollars.",+ "maker_fees_dollars": "Fees paid on contracts filled as maker, in dollars.",+ "created_time": "When the order was placed.",+ "last_update_time": "When the order last changed.",+ "expiration_time": "When the order expires, if it has an expiration.",+}++_POSITION_COLUMNS = {+ "ticker": "Ticker of the market the position is in.",+ "position_fp": "Net number of contracts held. Positive is yes, negative is no.",+ "total_traded_dollars": "Total spent on this market, in dollars.",+ "market_exposure_dollars": "Cost of the open position, in dollars.",+ "realized_pnl_dollars": "Locked-in profit and loss, in dollars.",+ "fees_paid_dollars": "Fees paid on fills in this market, in dollars.",+ "last_updated_ts": "When the position last changed.",+}++CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"markets": {"description": "One row per tradeable event contract, carrying its rules, current quotes, and settlement outcome.",@@ -144,4 +193,51 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"receipt_basis": "Set only on points from the historical backfill for the period before a city went live. These points are not settlement-eligible.",},},+ "fills": {+ "description": "Your account's fills. A fill is one match of your order against another order.",+ "docs_url": "https://docs.kalshi.com/api-reference/portfolio/get-fills",+ "columns": _FILL_COLUMNS,+ },+ "historical_fills": {+ "description": "Your account's fills from before the historical cutoff, which the fills table no longer returns.",+ "docs_url": "https://docs.kalshi.com/api-reference/historical/get-historical-fills",+ "columns": _FILL_COLUMNS,+ },+ "orders": {+ "description": "Your account's orders that are resting, or that were canceled or executed after the historical cutoff.",+ "docs_url": "https://docs.kalshi.com/api-reference/orders/get-orders",+ "columns": _ORDER_COLUMNS,+ },+ "historical_orders": {+ "description": "Your account's canceled and executed orders from before the historical cutoff.",+ "docs_url": "https://docs.kalshi.com/api-reference/historical/get-historical-orders",+ "columns": _ORDER_COLUMNS,+ },+ "positions": {+ "description": "Your account's position in each market, settled or not, that is still in the live data set.",+ "docs_url": "https://docs.kalshi.com/api-reference/portfolio/get-positions",+ "columns": _POSITION_COLUMNS,+ },+ "historical_positions": {+ "description": "Your account's settled positions after their event moved to the historical archive.",+ "docs_url": "https://docs.kalshi.com/api-reference/historical/get-historical-positions",+ "columns": _POSITION_COLUMNS,+ },+ "settlements": {+ "description": "Payouts for each settled market your account held contracts in, while still in the live data set.",+ "docs_url": "https://docs.kalshi.com/api-reference/portfolio/get-settlements",+ "columns": {+ "ticker": "Ticker of the market that settled.",+ "event_ticker": "Ticker of the event the market belongs to.",+ "market_result": "Settlement outcome, yes, no, or scalar.",+ "yes_count_fp": "Number of yes contracts held at settlement.",+ "yes_total_cost_dollars": "Cost basis of the yes contracts, in dollars.",+ "no_count_fp": "Number of no contracts held at settlement.",+ "no_total_cost_dollars": "Cost basis of the no contracts, in dollars.",+ "revenue": "Revenue from the settlement, in cents.",+ "value": "Payout of a single yes contract, in cents.",+ "fee_cost": "Total fees paid on the market, in dollars.",+ "settled_time": "When the market settled and payouts were made.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.pyindex 7b1566c78639..07e117fe49bf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py@@ -1,6 +1,14 @@+import time+import base64from collections.abc import Iterable, Iteratorfrom datetime import UTC, date, datetimefrom typing import Any, Optional+from urllib.parse import urlsplit++from cryptography.exceptions import UnsupportedAlgorithm+from cryptography.hazmat.primitives import hashes, serialization+from cryptography.hazmat.primitives.asymmetric import ed25519, padding, rsa+from requests import PreparedRequestfrom posthog.dataclasses import frozen@@ -9,6 +17,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import AuthConfigBasefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponseCursorPaginator,SinglePagePaginator,@@ -39,10 +48,69 @@ _WEATHER_INDEX_EARLIEST_MS = 1_WEATHER_INDEX_OVERLAP_MS = 60 * 60 * 1000+class KalshiCredentialsError(Exception):+ pass+++MISSING_CREDENTIALS_MESSAGE = "Portfolio tables need a Kalshi API key ID and private key"+INVALID_PRIVATE_KEY_MESSAGE = "The Kalshi private key is not a valid unencrypted RSA or Ed25519 PEM key"+++class KalshiSignedAuth(AuthConfigBase):+ """Signs `{timestamp}{METHOD}{path}` with the account's API key, as Kalshi's portfolio endpoints require."""++ def __init__(self, api_key_id: str, private_key: str) -> None:+ self.api_key_id = api_key_id+ self.private_key_pem = private_key+ try:+ key = serialization.load_pem_private_key(private_key.strip().encode(), password=None)+ except (ValueError, TypeError, UnsupportedAlgorithm) as e:+ raise KalshiCredentialsError(INVALID_PRIVATE_KEY_MESSAGE) from e+ if not isinstance(key, rsa.RSAPrivateKey | ed25519.Ed25519PrivateKey):+ raise KalshiCredentialsError(INVALID_PRIVATE_KEY_MESSAGE)+ self._key = key++ def _sign(self, message: bytes) -> bytes:+ if isinstance(self._key, ed25519.Ed25519PrivateKey):+ return self._key.sign(message)+ return self._key.sign(+ message,+ padding.PSS(mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.DIGEST_LENGTH),+ hashes.SHA256(),+ )++ def __call__(self, request: PreparedRequest) -> PreparedRequest:+ timestamp = str(int(time.time() * 1000))+ # The signed path keeps the `/trade-api/v2` prefix and drops the query string.+ path = urlsplit(request.url or "").path+ signature = self._sign(f"{timestamp}{request.method}{path}".encode())+ request.headers["KALSHI-ACCESS-KEY"] = self.api_key_id+ request.headers["KALSHI-ACCESS-TIMESTAMP"] = timestamp+ request.headers["KALSHI-ACCESS-SIGNATURE"] = base64.b64encode(signature).decode()+ return request++ def secret_values(self) -> tuple[str, ...]:+ return (self.private_key_pem,)+++def build_auth(api_key_id: Optional[str], private_key: Optional[str]) -> Optional[KalshiSignedAuth]:+ """Return the signer, or `None` when neither field is set. Raises when only one is set or the key is bad."""+ api_key_id = (api_key_id or "").strip()+ private_key = (private_key or "").strip()+ if not api_key_id and not private_key:+ return None+ if not api_key_id or not private_key:+ raise KalshiCredentialsError(MISSING_CREDENTIALS_MESSAGE)+ return KalshiSignedAuth(api_key_id, private_key)++@frozenclass KalshiResumeConfig:# Opaque pagination cursor from the previous response's `cursor` field.cursor: str+ # The subaccount the cursor belongs to, for endpoints read once per subaccount. An empty+ # cursor starts that subaccount at its first page.+ subaccount: Optional[int] = Nonedef _to_epoch_seconds(value: Any) -> Optional[int]:@@ -250,6 +318,91 @@ def _weather_index_source()+def _subaccounts(client: RESTClient) -> list[int]:+ numbers: set[int] = set()+ for page in client.paginate(+ path="/portfolio/subaccounts/balances",+ paginator=SinglePagePaginator(),+ data_selector="subaccount_balances",+ data_selector_required=True,+ ):+ numbers.update(int(balance["subaccount_number"]) for balance in page if "subaccount_number" in balance)+ return sorted(numbers) or [0]+++def _subaccount_rows(+ client: RESTClient,+ config: KalshiEndpointConfig,+ resumable_source_manager: ResumableSourceManager[KalshiResumeConfig],+ min_ts: Optional[int],+) -> Iterator[list[dict[str, Any]]]:+ subaccounts = _subaccounts(client)+ page_cursor: Optional[str] = None+ if resumable_source_manager.can_resume():+ resume = resumable_source_manager.load_state()+ if resume is not None and resume.subaccount in subaccounts:+ subaccounts = subaccounts[subaccounts.index(resume.subaccount) :]+ page_cursor = resume.cursor or None++ for position, subaccount in enumerate(subaccounts):+ following = subaccounts[position + 1] if position + 1 < len(subaccounts) else None+ params: dict[str, Any] = {"limit": config.page_size, "subaccount": subaccount, **config.extra_params}+ if min_ts is not None:+ params["min_ts"] = min_ts++ def stage_next(+ state: Optional[dict[str, Any]],+ has_next_page: bool,+ subaccount: int = subaccount,+ following: Optional[int] = following,+ ) -> None:+ if has_next_page and state and state.get("cursor"):+ resumable_source_manager.save_state(+ KalshiResumeConfig(cursor=str(state["cursor"]), subaccount=subaccount)+ )+ elif following is not None:+ resumable_source_manager.save_state(KalshiResumeConfig(cursor="", subaccount=following))++ for page in client.paginate(+ path=config.path,+ params=params,+ paginator=JSONResponseCursorPaginator(cursor_path="cursor", cursor_param="cursor"),+ data_selector=config.data_key,+ data_selector_required=True,+ initial_paginator_state={"cursor": page_cursor} if page_cursor else None,+ page_state_hook=stage_next,+ ):+ if page:+ yield [{**row, "subaccount_number": subaccount} for row in page]+ else:+ resumable_source_manager.safe_point()+ page_cursor = None+++def _subaccount_source(+ config: KalshiEndpointConfig,+ resumable_source_manager: ResumableSourceManager[KalshiResumeConfig],+ auth: KalshiSignedAuth,+ last_value: Optional[Any],+) -> SourceResponse:+ client = RESTClient(base_url=KALSHI_BASE_URL, headers={"Accept": "application/json"}, auth=auth)+ min_ts = _to_epoch_seconds(last_value) if config.supports_incremental else None++ return SourceResponse(+ name=config.name,+ items=lambda: _subaccount_rows(client, config, resumable_source_manager, min_ts),+ primary_keys=config.primary_keys,+ # Rows arrive grouped by subaccount, not by time, so the watermark can only advance once+ # the walk completes.+ sort_mode="desc",+ partition_count=1 if config.partition_key else None,+ partition_size=1 if config.partition_key else None,+ partition_mode="datetime" if config.partition_key else None,+ partition_format="day" if config.partition_key else None,+ partition_keys=[config.partition_key] if config.partition_key else None,+ )++def _build_endpoint(config: KalshiEndpointConfig, should_use_incremental_field: bool) -> Endpoint:endpoint: Endpoint = {"path": config.path,@@ -259,7 +412,7 @@ def _build_endpoint(config: KalshiEndpointConfig, should_use_incremental_field:}if config.paginated:- endpoint["params"] = {"limit": config.page_size}+ endpoint["params"] = {"limit": config.page_size, **config.extra_params}# Kalshi pages with an opaque `cursor` echoed at the response root, and omits it on the# last page. The paginator implements resume state, so a heartbeat timeout picks up mid-walk.endpoint["paginator"] = JSONResponseCursorPaginator(cursor_path="cursor", cursor_param="cursor")@@ -269,7 +422,7 @@ def _build_endpoint(config: KalshiEndpointConfig, should_use_incremental_field:if config.supports_incremental and should_use_incremental_field:endpoint["incremental"] = {"start_param": "min_ts",- "cursor_path": "created_time",+ "cursor_path": config.incremental_fields[0]["field"],"convert": _to_epoch_seconds,}@@ -283,8 +436,11 @@ def kalshi_source(resumable_source_manager: ResumableSourceManager[KalshiResumeConfig],should_use_incremental_field: bool = False,db_incremental_field_last_value: Optional[Any] = None,+ auth: Optional[KalshiSignedAuth] = None,) -> SourceResponse:config = KALSHI_ENDPOINTS[endpoint]+ if config.requires_auth and auth is None:+ raise KalshiCredentialsError(MISSING_CREDENTIALS_MESSAGE)last_value = db_incremental_field_last_value if should_use_incremental_field else Noneif config.market_fan_out:@@ -293,11 +449,15 @@ def kalshi_source(if config.city_fan_out:return _weather_index_source(config, resumable_source_manager, last_value)- # Kalshi's market-data endpoints are public and take no credential, so no `auth` is configured.+ if config.subaccount_fan_out and auth is not None:+ return _subaccount_source(config, resumable_source_manager, auth, last_value)++ # Market-data endpoints are public, so only portfolio endpoints send the signature.rest_config: RESTAPIConfig = {"client": {"base_url": KALSHI_BASE_URL,"headers": {"Accept": "application/json"},+ "auth": auth if config.requires_auth else None,},"resource_defaults": {},"resources": [@@ -333,7 +493,7 @@ def kalshi_source(name=endpoint,items=lambda: resource,primary_keys=config.primary_keys,- # Trades arrive strictly newest-first and the API offers no ascending option, so the+ # Trades, fills, and settlements arrive strictly newest-first and the API offers no ascending option, so the# watermark can only advance once the walk completes.sort_mode="desc" if config.newest_first else "asc",partition_count=1,@@ -345,12 +505,22 @@ def kalshi_source()-def validate_credentials() -> bool:- # Nothing to authenticate; confirm the public API is reachable and answering so a source is- # never created against an endpoint that has moved.+def validate_credentials(auth: Optional[KalshiSignedAuth] = None) -> bool:+ if auth is None:+ # Nothing to authenticate; confirm the public API is reachable and answering so a source is+ # never created against an endpoint that has moved.+ ok, _status = validate_via_probe(+ make_tracked_session,+ f"{KALSHI_BASE_URL}/exchange/status",+ headers={"Accept": "application/json"},+ )+ return ok+ok, _status = validate_via_probe(make_tracked_session,- f"{KALSHI_BASE_URL}/exchange/status",+ f"{KALSHI_BASE_URL}/portfolio/balance",headers={"Accept": "application/json"},+ auth=auth,+ allow_redirects=False,)return okdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.pyindex a28309d2bb43..8e33e1bebfb6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py@@ -14,11 +14,11 @@ class KalshiEndpointConfig:primary_keys: list[str] = field(default_factory=lambda: ["ticker"])partition_key: Optional[str] = None # stable creation-time field for datetime partitioningincremental_fields: list[IncrementalField] = field(default_factory=list)- # Only the trade endpoints expose a server-side time filter (`min_ts`), and candlesticks take an- # explicit time window. Everything else accepts the param and silently ignores it, so those+ # Only the trade, fill, and settlement endpoints expose a server-side time filter (`min_ts`), and+ # candlesticks take an explicit time window. Everything else accepts the param and silently ignores it, so those# endpoints are full-refresh only.supports_incremental: bool = False- # Trades come back newest-first; every other endpoint is treated as ascending.+ # Trades, fills, and settlements come back newest-first; every other endpoint is treated as ascending.newest_first: bool = False# `/series` returns the whole collection in one response with no `cursor` key.paginated: bool = True@@ -26,6 +26,12 @@ class KalshiEndpointConfig:market_fan_out: bool = False# The weather index is served one city at a time, with no listing endpoint for the cities.city_fan_out: bool = False+ # Portfolio endpoints read the account's own data and need a signed API key.+ requires_auth: bool = False+ # Kalshi reads only subaccount 0 unless asked for another, or the rows do not say which+ # subaccount they belong to, so these endpoints are read once per subaccount.+ subaccount_fan_out: bool = False+ extra_params: dict[str, str] = field(default_factory=dict)page_size: int = 200@@ -40,6 +46,17 @@ def _created_time_field() -> list[IncrementalField]:]+def _settled_time_field() -> list[IncrementalField]:+ return [+ {+ "label": "settled_time",+ "type": IncrementalFieldType.DateTime,+ "field": "settled_time",+ "field_type": IncrementalFieldType.DateTime,+ }+ ]++def _end_period_ts_field() -> list[IncrementalField]:return [{@@ -175,6 +192,89 @@ KALSHI_ENDPOINTS: dict[str, KalshiEndpointConfig] = {paginated=False,city_fan_out=True,),+ # The account's own executions. `min_ts` filters on the fill time, and fills come back+ # newest-first like public trades.+ "fills": KalshiEndpointConfig(+ name="fills",+ path="/portfolio/fills",+ data_key="fills",+ primary_keys=["fill_id"],+ partition_key="created_time",+ incremental_fields=_created_time_field(),+ supports_incremental=True,+ newest_first=True,+ requires_auth=True,+ page_size=1000,+ ),+ # Fills before the historical cutoff, which `/portfolio/fills` no longer returns.+ "historical_fills": KalshiEndpointConfig(+ name="historical_fills",+ path="/historical/fills",+ data_key="fills",+ primary_keys=["fill_id"],+ partition_key="created_time",+ incremental_fields=_created_time_field(),+ supports_incremental=True,+ newest_first=True,+ requires_auth=True,+ page_size=1000,+ ),+ # Resting orders change status and fill counts in place, and the docs do not say which+ # timestamp `min_ts` filters on, so orders are a full refresh.+ "orders": KalshiEndpointConfig(+ name="orders",+ path="/portfolio/orders",+ data_key="orders",+ primary_keys=["order_id"],+ requires_auth=True,+ page_size=1000,+ ),+ # Canceled and executed orders archived before the historical cutoff.+ "historical_orders": KalshiEndpointConfig(+ name="historical_orders",+ path="/historical/orders",+ data_key="orders",+ primary_keys=["order_id"],+ requires_auth=True,+ page_size=1000,+ ),+ # One row per market position, settled or not, that is still in the live data set. Exposure and+ # realized P&L change on every fill, so this is a full refresh.+ "positions": KalshiEndpointConfig(+ name="positions",+ path="/portfolio/positions",+ data_key="market_positions",+ primary_keys=["subaccount_number", "ticker"],+ requires_auth=True,+ subaccount_fan_out=True,+ extra_params={"settlement_status": "all"},+ page_size=1000,+ ),+ # Settled positions after their event moves to the archive.+ "historical_positions": KalshiEndpointConfig(+ name="historical_positions",+ path="/historical/positions",+ data_key="market_positions",+ primary_keys=["subaccount_number", "ticker"],+ requires_auth=True,+ subaccount_fan_out=True,+ page_size=1000,+ ),+ # Payouts per settled market. A settlement carries only `settled_time`, which is what `min_ts`+ # filters on.+ "settlements": KalshiEndpointConfig(+ name="settlements",+ path="/portfolio/settlements",+ data_key="settlements",+ primary_keys=["subaccount_number", "ticker"],+ partition_key="settled_time",+ incremental_fields=_settled_time_field(),+ supports_incremental=True,+ newest_first=True,+ requires_auth=True,+ subaccount_fan_out=True,+ page_size=1000,+ ),}ENDPOINTS = tuple(KALSHI_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.pyindex 50361a33b4f0..51112b038912 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py@@ -5,6 +5,8 @@ from products.warehouse_sources.backend.facade.source_config import (DataWarehouseSourceCategory,ReleaseStatus,SourceConfig,+ SourceFieldInputConfig,+ SourceFieldInputConfigType,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (@@ -19,13 +21,17 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.schfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.kalshi import KalshiSourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.kalshi.kalshi import (+ MISSING_CREDENTIALS_MESSAGE,+ KalshiCredentialsError,KalshiResumeConfig,+ build_auth,kalshi_source,validate_credentials as validate_kalshi_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.kalshi.settings import (ENDPOINTS,INCREMENTAL_FIELDS,+ KALSHI_ENDPOINTS,)from products.warehouse_sources.backend.types import ExternalDataSourceType@@ -54,13 +60,35 @@ class KalshiSource(ResumableSource[KalshiSourceConfig, KalshiResumeConfig]):category=DataWarehouseSourceCategory.FINANCE___ACCOUNTING,label="Kalshi",releaseStatus=ReleaseStatus.BETA,- caption="""Kalshi's market data endpoints are public, so this source needs no API key.+ caption="""Kalshi's market data endpoints are public, so this source needs no API key for them.-It syncs markets, events, series, trades, milestones, daily market candlesticks, and the per-minute weather index for each city. It also syncs the historical archive of markets and trades. Portfolio data (your own orders, fills, and positions) needs a signed API key and is not supported yet.""",+It syncs markets, events, series, trades, milestones, daily market candlesticks, and the per-minute weather index for each city. It also syncs the historical archive of markets and trades.++To sync your own fills, orders, positions, and settlements, add an API key. Create one in your [Kalshi account settings](https://kalshi.com/account/profile) and paste the key ID and the private key file contents below. The key only needs read access.""",iconPath="/static/services/kalshi.png",docsUrl="https://posthog.com/docs/cdp/sources/kalshi",keywords=["prediction market", "event contracts", "trading"],- fields=cast(list[FieldType], []),+ fields=cast(+ list[FieldType],+ [+ SourceFieldInputConfig(+ name="api_key_id",+ label="API key ID (optional)",+ type=SourceFieldInputConfigType.TEXT,+ required=False,+ placeholder="a952bcbe-ec3b-4b5b-b8f9-11dae589608c",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="private_key",+ label="Private key (optional)",+ type=SourceFieldInputConfigType.TEXTAREA,+ required=False,+ placeholder="-----BEGIN RSA PRIVATE KEY-----",+ secret=True,+ ),+ ],+ ),)def get_canonical_descriptions(self) -> CanonicalDescriptions:@@ -75,6 +103,9 @@ It syncs markets, events, series, trades, milestones, daily market candlesticks,# There is no credential to fix, so a 403 means Kalshi is refusing this caller (most# often a geo-block on the exchange). Retrying can't change that."403 Client Error: Forbidden for url: https://api.elections.kalshi.com": "Kalshi refused the request. The exchange restricts access from some regions, so this source may not be usable from where PostHog runs.",+ "401 Client Error: Unauthorized for url: https://api.elections.kalshi.com": "Kalshi rejected the API key. Check the key ID and private key, or create a new key and update the source.",+ MISSING_CREDENTIALS_MESSAGE: "Portfolio tables need a Kalshi API key. Add the key ID and private key to the source, or turn off the portfolio tables.",+ "The Kalshi private key is not a valid": "The Kalshi private key could not be read. Paste the full contents of the private key file, including the BEGIN and END lines.",}def get_schemas(@@ -86,7 +117,7 @@ It syncs markets, events, series, trades, milestones, daily market candlesticks,force_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # Only the trade, candlestick, and weather index endpoints carry incremental fields, so build_endpoint_schemas+ # Only the trade, fill, settlement, candlestick, and weather index endpoints carry incremental fields, so build_endpoint_schemas# marks them incremental and leaves the rest full-refresh.# Append would duplicate the hour each weather index run re-reads; merge dedupes it.return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=("weather_index",))@@ -98,11 +129,32 @@ It syncs markets, events, series, trades, milestones, daily market candlesticks,schema_name: Optional[str] = None,api_version: str | None = None,) -> tuple[bool, str | None]:- if validate_kalshi_credentials():+ try:+ auth = build_auth(config.api_key_id, config.private_key)+ except KalshiCredentialsError as e:+ return False, f"{e}."++ if auth is None and schema_name in KALSHI_ENDPOINTS and KALSHI_ENDPOINTS[schema_name].requires_auth:+ return False, f"{MISSING_CREDENTIALS_MESSAGE}."++ if validate_kalshi_credentials(auth):return True, None+ if auth is not None:+ return False, "Kalshi rejected the API key. Check the key ID and private key and try again."return False, "Couldn't reach the Kalshi API. Try again in a few minutes."+ def get_endpoint_permissions(+ self, config: KalshiSourceConfig, team_id: int, endpoints: list[str], api_version: str | None = None+ ) -> dict[str, str | None]:+ has_key = bool((config.api_key_id or "").strip() and (config.private_key or "").strip())+ return {+ name: MISSING_CREDENTIALS_MESSAGE+ if not has_key and name in KALSHI_ENDPOINTS and KALSHI_ENDPOINTS[name].requires_auth+ else None+ for name in endpoints+ }+def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[KalshiResumeConfig]:return ResumableSourceManager[KalshiResumeConfig](inputs, KalshiResumeConfig)@@ -121,4 +173,5 @@ It syncs markets, events, series, trades, milestones, daily market candlesticks,db_incremental_field_last_value=inputs.db_incremental_field_last_valueif inputs.should_use_incremental_fieldelse None,+ auth=build_auth(config.api_key_id, config.private_key),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.pyindex 5325d5c5a05f..525322e1855f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py@@ -1,6 +1,13 @@+import base64from datetime import UTC, date, datetime, timedeltafrom typing import Any+import pytest++from cryptography.exceptions import InvalidSignature+from cryptography.hazmat.primitives import hashes, serialization+from cryptography.hazmat.primitives.asymmetric import ec, ed25519, padding, rsa+from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypesfrom parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (@@ -12,7 +19,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.kalshi import KalshiSourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.kalshi.kalshi import (+ INVALID_PRIVATE_KEY_MESSAGE,+ KalshiCredentialsError,KalshiResumeConfig,+ KalshiSignedAuth,_batch_markets,_candlestick_market,_CandlestickMarket,@@ -26,8 +36,35 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.kalshi.setfrom products.warehouse_sources.backend.temporal.data_imports.sources.kalshi.source import KalshiSource+def _pem(key: PrivateKeyTypes) -> str:+ return key.private_bytes(+ serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()+ ).decode()+++_RSA_KEY = rsa.generate_private_key(public_exponent=65537, key_size=2048)+_ED25519_KEY = ed25519.Ed25519PrivateKey.generate()++def _driver() -> SourceDriver:- return SourceDriver(KalshiSource(), KalshiSourceConfig())+ return SourceDriver(KalshiSource(), KalshiSourceConfig(api_key_id="key-id", private_key=_pem(_RSA_KEY)))+++def _verify(key: PrivateKeyTypes, signature: bytes, message: bytes) -> bool:+ try:+ if isinstance(key, ed25519.Ed25519PrivateKey):+ key.public_key().verify(signature, message)+ else:+ assert isinstance(key, rsa.RSAPrivateKey)+ key.public_key().verify(+ signature,+ message,+ padding.PSS(mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.DIGEST_LENGTH),+ hashes.SHA256(),+ )+ except InvalidSignature:+ return False+ return Truedef _page(data_key: str, items: list[dict[str, Any]], *, cursor: str | None = None) -> ScriptedResponse:@@ -75,15 +112,17 @@ class TestKalshiTransport:@parameterized.expand([- ("trades", "/trade-api/v2/markets/trades"),- ("historical_trades", "/trade-api/v2/historical/trades"),+ ("trades", "/trade-api/v2/markets/trades", "created_time"),+ ("historical_trades", "/trade-api/v2/historical/trades", "created_time"),+ ("fills", "/trade-api/v2/portfolio/fills", "created_time"),+ ("historical_fills", "/trade-api/v2/historical/fills", "created_time"),])- def test_trades_incremental_sends_min_ts(self, endpoint: str, path: str) -> None:+ def test_trades_incremental_sends_min_ts(self, endpoint: str, path: str, incremental_field: str) -> None:result = _driver().run(endpoint,- [_page("trades", [{"trade_id": "t1"}])],- incremental_field="created_time",+ [_page(KALSHI_ENDPOINTS[endpoint].data_key, [{"id": "1"}])],+ incremental_field=incremental_field,db_incremental_field_last_value=datetime(2021, 1, 1, tzinfo=UTC),)@@ -92,7 +131,7 @@ class TestKalshiTransport:assert result.requests[0].param("min_ts") == "1609459200"assert result.requests[0].param("limit") == str(KALSHI_ENDPOINTS[endpoint].page_size)- @parameterized.expand(["markets", "events", "milestones", "historical_markets"])+ @parameterized.expand(["markets", "events", "milestones", "historical_markets", "orders", "historical_orders"])def test_non_incremental_endpoints_never_send_min_ts(self, endpoint: str) -> None:# Kalshi answers 200 and ignores params it doesn't recognise, so a filter wired onto an# endpoint that has none would look like it worked while syncing everything every run.@@ -136,6 +175,7 @@ class TestKalshiTransport:[("trades", "desc"),("historical_trades", "desc"),+ ("fills", "desc"),("markets", "asc"),("series", "asc"),]@@ -195,6 +235,115 @@ class TestKalshiTransport:assert network.requests_log[0].path == "/trade-api/v2/exchange/status"+class TestKalshiPortfolio:+ @parameterized.expand([("rsa", _RSA_KEY), ("ed25519", _ED25519_KEY)])+ def test_signs_method_and_path_without_the_query(self, _name: str, key: PrivateKeyTypes) -> None:+ driver = SourceDriver(KalshiSource(), KalshiSourceConfig(api_key_id="key-id", private_key=_pem(key)))+ result = driver.run("fills", [_page("fills", [{"fill_id": "f1"}])])++ assert result.raised is None+ request = result.requests[0]+ assert request.param("limit") == "1000"+ assert request.headers["kalshi-access-key"] == "key-id"+ message = f"{request.headers['kalshi-access-timestamp']}GET/trade-api/v2/portfolio/fills".encode()+ assert _verify(key, base64.b64decode(request.headers["kalshi-access-signature"]), message)++ def test_public_endpoints_are_not_signed(self) -> None:+ result = _driver().run("markets", [_page("markets", [{"ticker": "A"}])])++ assert result.raised is None+ assert "kalshi-access-signature" not in result.requests[0].headers++ @parameterized.expand(+ [+ ("no_key", None, None),+ ("key_id_only", "key-id", None),+ ]+ )+ def test_portfolio_endpoint_without_a_key_fails_before_any_request(+ self, _name: str, api_key_id: str | None, private_key: str | None+ ) -> None:+ driver = SourceDriver(KalshiSource(), KalshiSourceConfig(api_key_id=api_key_id, private_key=private_key))+ result = driver.run("orders", [_page("orders", [])])++ assert isinstance(result.raised, KalshiCredentialsError)+ assert result.requests == []++ @parameterized.expand(+ [+ ("not_pem", "not a key"),+ ("unsupported_type", _pem(ec.generate_private_key(ec.SECP256R1()))),+ ]+ )+ def test_rejects_an_invalid_private_key(self, _name: str, private_key: str) -> None:+ with pytest.raises(KalshiCredentialsError, match=INVALID_PRIVATE_KEY_MESSAGE):+ KalshiSignedAuth("key-id", private_key)++ @parameterized.expand(+ [+ ("positions", "/trade-api/v2/portfolio/positions", "market_positions", "all"),+ ("historical_positions", "/trade-api/v2/historical/positions", "market_positions", None),+ ("settlements", "/trade-api/v2/portfolio/settlements", "settlements", None),+ ]+ )+ def test_subaccount_endpoints_read_every_subaccount(+ self, endpoint: str, path: str, data_key: str, settlement_status: str | None+ ) -> None:+ # Positions default to subaccount 0, and settlement rows carry no subaccount, so each+ # subaccount is read on its own and stamped onto its rows.+ result = _driver().run(+ endpoint,+ [+ ScriptedResponse(json={"subaccount_balances": [{"subaccount_number": 3}, {"subaccount_number": 0}]}),+ _page(data_key, [{"ticker": "A"}], cursor="c1"),+ _page(data_key, [{"ticker": "B"}]),+ _page(data_key, [{"ticker": "A"}]),+ ],+ )++ assert result.raised is None+ assert result.paths == ["/trade-api/v2/portfolio/subaccounts/balances", path, path, path]+ assert result.params("subaccount") == [None, "0", "0", "3"]+ assert result.params("cursor") == [None, None, "c1", None]+ assert result.params("settlement_status") == [None, settlement_status, settlement_status, settlement_status]+ assert [(row["subaccount_number"], row["ticker"]) for row in result.rows] == [(0, "A"), (0, "B"), (3, "A")]+ assert result.response is not None+ assert result.response.primary_keys == ["subaccount_number", "ticker"]++ def test_settlements_incremental_sends_min_ts_for_each_subaccount(self) -> None:+ result = _driver().run(+ "settlements",+ [+ ScriptedResponse(json={"subaccount_balances": [{"subaccount_number": 0}, {"subaccount_number": 1}]}),+ _page("settlements", [{"ticker": "A"}]),+ _page("settlements", [{"ticker": "B"}]),+ ],+ incremental_field="settled_time",+ db_incremental_field_last_value=datetime(2021, 1, 1, tzinfo=UTC),+ )++ assert result.raised is None+ assert result.params("min_ts") == [None, "1609459200", "1609459200"]+ assert result.response is not None+ assert result.response.sort_mode == "desc"++ def test_subaccount_walk_resumes_and_stages_the_next_subaccount(self) -> None:+ result = _driver().run(+ "positions",+ [+ ScriptedResponse(json={"subaccount_balances": [{"subaccount_number": n} for n in (0, 1, 2)]}),+ _page("market_positions", [{"ticker": "B"}]),+ _page("market_positions", [{"ticker": "C"}]),+ ],+ resume_state=KalshiResumeConfig(cursor="saved", subaccount=1),+ )++ assert result.raised is None+ assert result.params("subaccount") == [None, "1", "2"]+ assert result.params("cursor") == [None, "saved", None]+ assert KalshiResumeConfig(cursor="", subaccount=2) in result.committed_states++class TestKalshiCandlesticks:def test_walks_market_pages_and_batches_candlestick_requests(self) -> None:now = datetime.now(UTC)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi_source.pyindex 9a9a7baebea1..37a99c222bf1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi_source.py@@ -1,3 +1,5 @@+from cryptography.hazmat.primitives import serialization+from cryptography.hazmat.primitives.asymmetric import ed25519from parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (@@ -8,6 +10,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.kalshi import KalshiSourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.kalshi.source import KalshiSource+_PEM = (+ ed25519.Ed25519PrivateKey.generate()+ .private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())+ .decode()+)+class TestKalshiSource:@parameterized.expand(@@ -16,6 +24,13 @@ class TestKalshiSource:("historical_trades_is_incremental", "historical_trades", True),("market_candlesticks_is_incremental", "market_candlesticks", True),("weather_index_is_incremental", "weather_index", True),+ ("fills_is_incremental", "fills", True),+ ("historical_fills_is_incremental", "historical_fills", True),+ ("settlements_is_incremental", "settlements", True),+ ("orders_is_not", "orders", False),+ ("historical_orders_is_not", "historical_orders", False),+ ("positions_is_not", "positions", False),+ ("historical_positions_is_not", "historical_positions", False),("markets_is_not", "markets", False),("historical_markets_is_not", "historical_markets", False),("events_is_not", "events", False),@@ -40,6 +55,61 @@ class TestKalshiSource:assert (message is None) is expectedassert network.requests_log[0].path == "/trade-api/v2/exchange/status"+ @parameterized.expand(+ [+ (+ "valid_key",+ "key-id",+ _PEM,+ None,+ [ScriptedResponse(status=200)],+ True,+ ["/trade-api/v2/portfolio/balance"],+ ),+ (+ "rejected_key",+ "key-id",+ _PEM,+ None,+ [ScriptedResponse(status=401)],+ False,+ ["/trade-api/v2/portfolio/balance"],+ ),+ ("key_id_without_private_key", "key-id", None, None, [], False, []),+ ("unreadable_private_key", "key-id", "not a key", None, [], False, []),+ ("portfolio_table_without_key", None, None, "fills", [], False, []),+ ]+ )+ def test_validate_credentials_with_an_api_key(+ self,+ _name: str,+ api_key_id: str | None,+ private_key: str | None,+ schema_name: str | None,+ script: list[ScriptedResponse],+ expected: bool,+ paths: list[str],+ ) -> None:+ config = KalshiSourceConfig(api_key_id=api_key_id, private_key=private_key)+ with scripted_network(script) as network:+ ok, message = KalshiSource().validate_credentials(config, 1, schema_name=schema_name)++ assert ok is expected+ assert (message is None) is expected+ assert [request.path for request in network.requests_log] == paths++ @parameterized.expand([("without_key", None, None, False), ("with_key", "key-id", _PEM, True)])+ def test_endpoint_permissions_flag_portfolio_tables_without_a_key(+ self, _name: str, api_key_id: str | None, private_key: str | None, reachable: bool+ ) -> None:+ permissions = KalshiSource().get_endpoint_permissions(+ KalshiSourceConfig(api_key_id=api_key_id, private_key=private_key), 1, ["markets", "fills", "positions"]+ )++ assert permissions["markets"] is None+ assert (permissions["fills"] is None) is reachable+ assert (permissions["positions"] is None) is reachable+def test_source_for_pipeline_drops_the_watermark_on_full_refresh(self) -> None:# A stale watermark leaking into a full refresh would filter rows the user asked to re-import.result = SourceDriver(KalshiSource(), KalshiSourceConfig()).run(