:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add kalshi credit history and weather calibration tables (#126973)
Tom Owers46a344be5e90
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex bccc52cba367..23e81aa1487a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -6771,6 +6771,8 @@ Diffed against: <https://docs.kalshi.com/openapi.yaml>- [ ] `Market orderbook (GET /trade-api/v2/markets/{ticker}/orderbook)` — measure market depth and available liquidity at each price level (medium)- [ ] `Series fee changes (GET /trade-api/v2/series/fee_changes)` — explain trading cost changes across series and time (medium)- [x] `Weather index (GET /trade-api/v2/live_data/weather/{city})` — analyze the minute-resolution city temperature index behind hourly temperature markets (medium)+- [x] `Weather index calibrations (GET /trade-api/v2/live_data/weather/{city}/calibrations)` — reproduce published weather index values from each city's station weights and offsets over time (medium)+- [x] `Credit history (GET /trade-api/v2/portfolio/credit_history)` — reconcile credits applied to the account alongside fills and settlements (medium)Note: Parsed every GET operation in the official Trade API OpenAPI specification. Historical endpoints contain archived records absent from current collections. Portfolio tables require authenticated requests; the current source only reads public market data. Order book history requires periodic snapshots; API keys, exchange status, and trading actions were excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.pyindex 7294767e7f4d..8f26f4fc9a1d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py@@ -193,6 +193,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"receipt_basis": "Set only on points from the historical backfill for the period before a city went live. These points are not settlement-eligible.",},},+ "weather_index_calibrations": {+ "description": "Each city's weather index configuration timeline, one row per version: the launch configuration plus every weekly offset calibration and methodology update.",+ "docs_url": "https://docs.kalshi.com/api-reference/live-data/get-weather-index-calibrations",+ "columns": {+ "city": "Index city ID, for example miami.",+ "config_version": "Configuration version. Weather index points report the version they were computed under in their own config_version field.",+ "published_at_ms": "When the record was published, as a Unix timestamp in milliseconds (UTC).",+ "effective_at_ms": "Start of the minutes this record governs, as a Unix timestamp in milliseconds (UTC). It applies until the next record takes effect.",+ "change_reason": "Why the configuration changed.",+ "calibration_window_start_ms": "Start of the observation window the offsets were estimated from. Empty on the launch configuration.",+ "calibration_window_end_ms": "End of the calibration window, exclusive.",+ "city_reference_c": "City reference in Celsius: the weighted sum of the station offsets.",+ "stations": "Member stations in configuration order, each with its ID, weight, offset in Celsius, and weekly calibration note.",+ },+ },"fills": {"description": "Your account's fills. A fill is one match of your order against another order.","docs_url": "https://docs.kalshi.com/api-reference/portfolio/get-fills",@@ -240,4 +255,16 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"settled_time": "When the market settled and payouts were made.",},},+ "credit_history": {+ "description": "Credits applied to your account, one row per credit.",+ "docs_url": "https://docs.kalshi.com/api-reference/portfolio/get-credit-history",+ "columns": {+ "credit_id": "Unique identifier for the credit.",+ "type": "Category of the credit.",+ "status": "Processing status of the credit: pending, applied, failed, or unknown.",+ "amount_dollars": "Credit amount, in dollars.",+ "reason": "Description of the credit, which can include a market or event ticker.",+ "created_time": "When the credit was created.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.pyindex 07e117fe49bf..24649fe97bfa 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py@@ -266,11 +266,11 @@ def _candlestick_source()-def _weather_index_rows(+def _city_rows(client: RESTClient,+ config: KalshiEndpointConfig,resumable_source_manager: ResumableSourceManager[KalshiResumeConfig],- from_ms: int,- to_ms: int,+ params: dict[str, Any],) -> Iterator[list[dict[str, Any]]]:cities = list(WEATHER_INDEX_CITIES)if resumable_source_manager.can_resume():@@ -281,10 +281,10 @@ def _weather_index_rows(for position, city in enumerate(cities):rows: list[dict[str, Any]] = []for page in client.paginate(- path=f"/live_data/weather/{city}",- params={"from": from_ms, "to": to_ms},+ path=config.path.format(city=city),+ params=params,paginator=SinglePagePaginator(),- data_selector="timeseries",+ data_selector=config.data_key,data_selector_required=True,):rows.extend({"city": city, **point} for point in page)@@ -297,20 +297,22 @@ def _weather_index_rows(resumable_source_manager.safe_point()-def _weather_index_source(+def _city_source(config: KalshiEndpointConfig,resumable_source_manager: ResumableSourceManager[KalshiResumeConfig],last_value: Optional[Any],) -> SourceResponse:client = RESTClient(base_url=KALSHI_BASE_URL, headers={"Accept": "application/json"})- from_ms = _WEATHER_INDEX_EARLIEST_MS- if last_value is not None:- from_ms = max(int(last_value) - _WEATHER_INDEX_OVERLAP_MS, _WEATHER_INDEX_EARLIEST_MS)- to_ms = int(datetime.now(UTC).timestamp() * 1000)+ params: dict[str, Any] = {}+ if config.supports_incremental:+ from_ms = _WEATHER_INDEX_EARLIEST_MS+ if last_value is not None:+ from_ms = max(int(last_value) - _WEATHER_INDEX_OVERLAP_MS, _WEATHER_INDEX_EARLIEST_MS)+ params = {"from": from_ms, "to": int(datetime.now(UTC).timestamp() * 1000)}return SourceResponse(name=config.name,- items=lambda: _weather_index_rows(client, resumable_source_manager, from_ms, to_ms),+ items=lambda: _city_rows(client, config, resumable_source_manager, params),primary_keys=config.primary_keys,# Rows arrive grouped by city, not by time, so the watermark can only advance once the# walk completes.@@ -447,7 +449,7 @@ def kalshi_source(return _candlestick_source(config, resumable_source_manager, last_value)if config.city_fan_out:- return _weather_index_source(config, resumable_source_manager, last_value)+ return _city_source(config, resumable_source_manager, last_value)if config.subaccount_fan_out and auth is not None:return _subaccount_source(config, resumable_source_manager, auth, last_value)@@ -493,7 +495,7 @@ def kalshi_source(name=endpoint,items=lambda: resource,primary_keys=config.primary_keys,- # Trades, fills, and settlements arrive strictly newest-first and the API offers no ascending option, so the+ # Trades, fills, settlements, and credits arrive strictly newest-first and the API offers no ascending option, so the# watermark can only advance once the walk completes.sort_mode="desc" if config.newest_first else "asc",partition_count=1,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.pyindex 8e33e1bebfb6..440bd37725e2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py@@ -18,13 +18,13 @@ class KalshiEndpointConfig:# candlesticks take an explicit time window. Everything else accepts the param and silently ignores it, so those# endpoints are full-refresh only.supports_incremental: bool = False- # Trades, fills, and settlements come back newest-first; every other endpoint is treated as ascending.+ # Trades, fills, settlements, and credits come back newest-first; every other endpoint is treated as ascending.newest_first: bool = False# `/series` returns the whole collection in one response with no `cursor` key.paginated: bool = True# Candlesticks are fetched per batch of markets, so rows arrive in no global time order.market_fan_out: bool = False- # The weather index is served one city at a time, with no listing endpoint for the cities.+ # The weather index and its calibrations are served one city at a time, with no listing endpoint for the cities.city_fan_out: bool = False# Portfolio endpoints read the account's own data and need a signed API key.requires_auth: bool = False@@ -192,6 +192,16 @@ KALSHI_ENDPOINTS: dict[str, KalshiEndpointConfig] = {paginated=False,city_fan_out=True,),+ # Each city's append-only index configuration timeline: the launch configuration plus every+ # weekly offset calibration. It has no filter and stays small, so it is a full refresh.+ "weather_index_calibrations": KalshiEndpointConfig(+ name="weather_index_calibrations",+ path="/live_data/weather/{city}/calibrations",+ data_key="calibrations",+ primary_keys=["city", "config_version"],+ paginated=False,+ city_fan_out=True,+ ),# The account's own executions. `min_ts` filters on the fill time, and fills come back# newest-first like public trades."fills": KalshiEndpointConfig(@@ -275,6 +285,19 @@ KALSHI_ENDPOINTS: dict[str, KalshiEndpointConfig] = {subaccount_fan_out=True,page_size=1000,),+ # Credits to the account, newest-first. A credit's status moves from pending to applied or+ # failed, but `min_ts` filters on creation time, so an incremental run would never see that+ # change on an older credit. This is a full refresh.+ "credit_history": KalshiEndpointConfig(+ name="credit_history",+ path="/portfolio/credit_history",+ data_key="credits",+ primary_keys=["credit_id"],+ partition_key="created_time",+ newest_first=True,+ requires_auth=True,+ page_size=1000,+ ),}ENDPOINTS = tuple(KALSHI_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.pyindex 51112b038912..99857d129907 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py@@ -62,9 +62,9 @@ class KalshiSource(ResumableSource[KalshiSourceConfig, KalshiResumeConfig]):releaseStatus=ReleaseStatus.BETA,caption="""Kalshi's market data endpoints are public, so this source needs no API key for them.-It syncs markets, events, series, trades, milestones, daily market candlesticks, and the per-minute weather index for each city. It also syncs the historical archive of markets and trades.+It syncs markets, events, series, trades, milestones, daily market candlesticks, and the per-minute weather index and its calibration history for each city. It also syncs the historical archive of markets and trades.-To sync your own fills, orders, positions, and settlements, add an API key. Create one in your [Kalshi account settings](https://kalshi.com/account/profile) and paste the key ID and the private key file contents below. The key only needs read access.""",+To sync your own fills, orders, positions, settlements, and credit history, add an API key. Create one in your [Kalshi account settings](https://kalshi.com/account/profile) and paste the key ID and the private key file contents below. The key only needs read access.""",iconPath="/static/services/kalshi.png",docsUrl="https://posthog.com/docs/cdp/sources/kalshi",keywords=["prediction market", "event contracts", "trading"],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.pyindex 525322e1855f..b7f0d5115235 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py@@ -131,7 +131,9 @@ class TestKalshiTransport:assert result.requests[0].param("min_ts") == "1609459200"assert result.requests[0].param("limit") == str(KALSHI_ENDPOINTS[endpoint].page_size)- @parameterized.expand(["markets", "events", "milestones", "historical_markets", "orders", "historical_orders"])+ @parameterized.expand(+ ["markets", "events", "milestones", "historical_markets", "orders", "historical_orders", "credit_history"]+ )def test_non_incremental_endpoints_never_send_min_ts(self, endpoint: str) -> None:# Kalshi answers 200 and ignores params it doesn't recognise, so a filter wired onto an# endpoint that has none would look like it worked while syncing everything every run.@@ -529,30 +531,34 @@ class TestKalshiWeatherIndex:@parameterized.expand([# `from=0` would silently fall back to the trailing 24 hours.- ("full_refresh", {"should_use_incremental_field": False}, "1"),+ ("full_refresh", "weather_index", {"should_use_incremental_field": False}, "1"),("incremental_rereads_the_last_hour",+ "weather_index",{"incremental_field": "t", "db_incremental_field_last_value": 1_790_000_000_000},str(1_790_000_000_000 - 3_600_000),),+ # The calibration timeline takes no window, so it is read whole for every city.+ ("calibrations", "weather_index_calibrations", {"should_use_incremental_field": False}, None),])def test_reads_every_city_from_the_window_start(- self, _name: str, inputs: dict[str, Any], expected_from: str+ self, _name: str, endpoint: str, inputs: dict[str, Any], expected_from: str | None) -> None:- point = {"t": 1_790_000_060_000, "v": 79.16, "status": "normal", "contributors": 5}+ config = KALSHI_ENDPOINTS[endpoint]+ row = {"t": 1_790_000_060_000, "v": 79.16, "config_version": "v1.0"}result = _driver().run(- "weather_index",- always(ScriptedResponse(json={"city": "any", "units": "fahrenheit", "timeseries": [point]})),+ endpoint,+ always(ScriptedResponse(json={"city": "any", "units": "fahrenheit", config.data_key: [row]})),**inputs,)assert result.raised is None- assert result.paths == [f"/trade-api/v2/live_data/weather/{city}" for city in WEATHER_INDEX_CITIES]+ assert result.paths == [f"/trade-api/v2{config.path.format(city=city)}" for city in WEATHER_INDEX_CITIES]assert set(result.params("from")) == {expected_from}- assert all(to is not None for to in result.params("to"))- # The city is half of the primary key; without it every city's minute collides.- assert result.rows == [{"city": city, **point} for city in WEATHER_INDEX_CITIES]+ assert all((to is not None) == (expected_from is not None) for to in result.params("to"))+ # The city is half of the primary key; without it every city's rows collide.+ assert result.rows == [{"city": city, **row} for city in WEATHER_INDEX_CITIES]assert result.response is not Noneassert result.response.sort_mode == "desc"