Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add kalshi credit history and weather calibration tables (#126973)

Tom Owers46a344be5e90
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index bccc52cba367..23e81aa1487a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6771,6 +6771,8 @@ Diffed against: <https://docs.kalshi.com/openapi.yaml>
- [ ] `Market orderbook (GET /trade-api/v2/markets/{ticker}/orderbook)` — measure market depth and available liquidity at each price level (medium)
- [ ] `Series fee changes (GET /trade-api/v2/series/fee_changes)` — explain trading cost changes across series and time (medium)
- [x] `Weather index (GET /trade-api/v2/live_data/weather/{city})` — analyze the minute-resolution city temperature index behind hourly temperature markets (medium)
+- [x] `Weather index calibrations (GET /trade-api/v2/live_data/weather/{city}/calibrations)` — reproduce published weather index values from each city's station weights and offsets over time (medium)
+- [x] `Credit history (GET /trade-api/v2/portfolio/credit_history)` — reconcile credits applied to the account alongside fills and settlements (medium)
Note: Parsed every GET operation in the official Trade API OpenAPI specification. Historical endpoints contain archived records absent from current collections. Portfolio tables require authenticated requests; the current source only reads public market data. Order book history requires periodic snapshots; API keys, exchange status, and trading actions were excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py
index 7294767e7f4d..8f26f4fc9a1d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/canonical_descriptions.py
@@ -193,6 +193,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"receipt_basis": "Set only on points from the historical backfill for the period before a city went live. These points are not settlement-eligible.",
},
},
+ "weather_index_calibrations": {
+ "description": "Each city's weather index configuration timeline, one row per version: the launch configuration plus every weekly offset calibration and methodology update.",
+ "docs_url": "https://docs.kalshi.com/api-reference/live-data/get-weather-index-calibrations",
+ "columns": {
+ "city": "Index city ID, for example miami.",
+ "config_version": "Configuration version. Weather index points report the version they were computed under in their own config_version field.",
+ "published_at_ms": "When the record was published, as a Unix timestamp in milliseconds (UTC).",
+ "effective_at_ms": "Start of the minutes this record governs, as a Unix timestamp in milliseconds (UTC). It applies until the next record takes effect.",
+ "change_reason": "Why the configuration changed.",
+ "calibration_window_start_ms": "Start of the observation window the offsets were estimated from. Empty on the launch configuration.",
+ "calibration_window_end_ms": "End of the calibration window, exclusive.",
+ "city_reference_c": "City reference in Celsius: the weighted sum of the station offsets.",
+ "stations": "Member stations in configuration order, each with its ID, weight, offset in Celsius, and weekly calibration note.",
+ },
+ },
"fills": {
"description": "Your account's fills. A fill is one match of your order against another order.",
"docs_url": "https://docs.kalshi.com/api-reference/portfolio/get-fills",
@@ -240,4 +255,16 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"settled_time": "When the market settled and payouts were made.",
},
},
+ "credit_history": {
+ "description": "Credits applied to your account, one row per credit.",
+ "docs_url": "https://docs.kalshi.com/api-reference/portfolio/get-credit-history",
+ "columns": {
+ "credit_id": "Unique identifier for the credit.",
+ "type": "Category of the credit.",
+ "status": "Processing status of the credit: pending, applied, failed, or unknown.",
+ "amount_dollars": "Credit amount, in dollars.",
+ "reason": "Description of the credit, which can include a market or event ticker.",
+ "created_time": "When the credit was created.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py
index 07e117fe49bf..24649fe97bfa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/kalshi.py
@@ -266,11 +266,11 @@ def _candlestick_source(
)
-def _weather_index_rows(
+def _city_rows(
client: RESTClient,
+ config: KalshiEndpointConfig,
resumable_source_manager: ResumableSourceManager[KalshiResumeConfig],
- from_ms: int,
- to_ms: int,
+ params: dict[str, Any],
) -> Iterator[list[dict[str, Any]]]:
cities = list(WEATHER_INDEX_CITIES)
if resumable_source_manager.can_resume():
@@ -281,10 +281,10 @@ def _weather_index_rows(
for position, city in enumerate(cities):
rows: list[dict[str, Any]] = []
for page in client.paginate(
- path=f"/live_data/weather/{city}",
- params={"from": from_ms, "to": to_ms},
+ path=config.path.format(city=city),
+ params=params,
paginator=SinglePagePaginator(),
- data_selector="timeseries",
+ data_selector=config.data_key,
data_selector_required=True,
):
rows.extend({"city": city, **point} for point in page)
@@ -297,20 +297,22 @@ def _weather_index_rows(
resumable_source_manager.safe_point()
-def _weather_index_source(
+def _city_source(
config: KalshiEndpointConfig,
resumable_source_manager: ResumableSourceManager[KalshiResumeConfig],
last_value: Optional[Any],
) -> SourceResponse:
client = RESTClient(base_url=KALSHI_BASE_URL, headers={"Accept": "application/json"})
- from_ms = _WEATHER_INDEX_EARLIEST_MS
- if last_value is not None:
- from_ms = max(int(last_value) - _WEATHER_INDEX_OVERLAP_MS, _WEATHER_INDEX_EARLIEST_MS)
- to_ms = int(datetime.now(UTC).timestamp() * 1000)
+ params: dict[str, Any] = {}
+ if config.supports_incremental:
+ from_ms = _WEATHER_INDEX_EARLIEST_MS
+ if last_value is not None:
+ from_ms = max(int(last_value) - _WEATHER_INDEX_OVERLAP_MS, _WEATHER_INDEX_EARLIEST_MS)
+ params = {"from": from_ms, "to": int(datetime.now(UTC).timestamp() * 1000)}
return SourceResponse(
name=config.name,
- items=lambda: _weather_index_rows(client, resumable_source_manager, from_ms, to_ms),
+ items=lambda: _city_rows(client, config, resumable_source_manager, params),
primary_keys=config.primary_keys,
# Rows arrive grouped by city, not by time, so the watermark can only advance once the
# walk completes.
@@ -447,7 +449,7 @@ def kalshi_source(
return _candlestick_source(config, resumable_source_manager, last_value)
if config.city_fan_out:
- return _weather_index_source(config, resumable_source_manager, last_value)
+ return _city_source(config, resumable_source_manager, last_value)
if config.subaccount_fan_out and auth is not None:
return _subaccount_source(config, resumable_source_manager, auth, last_value)
@@ -493,7 +495,7 @@ def kalshi_source(
name=endpoint,
items=lambda: resource,
primary_keys=config.primary_keys,
- # Trades, fills, and settlements arrive strictly newest-first and the API offers no ascending option, so the
+ # Trades, fills, settlements, and credits arrive strictly newest-first and the API offers no ascending option, so the
# watermark can only advance once the walk completes.
sort_mode="desc" if config.newest_first else "asc",
partition_count=1,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py
index 8e33e1bebfb6..440bd37725e2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/settings.py
@@ -18,13 +18,13 @@ class KalshiEndpointConfig:
# candlesticks take an explicit time window. Everything else accepts the param and silently ignores it, so those
# endpoints are full-refresh only.
supports_incremental: bool = False
- # Trades, fills, and settlements come back newest-first; every other endpoint is treated as ascending.
+ # Trades, fills, settlements, and credits come back newest-first; every other endpoint is treated as ascending.
newest_first: bool = False
# `/series` returns the whole collection in one response with no `cursor` key.
paginated: bool = True
# Candlesticks are fetched per batch of markets, so rows arrive in no global time order.
market_fan_out: bool = False
- # The weather index is served one city at a time, with no listing endpoint for the cities.
+ # The weather index and its calibrations are served one city at a time, with no listing endpoint for the cities.
city_fan_out: bool = False
# Portfolio endpoints read the account's own data and need a signed API key.
requires_auth: bool = False
@@ -192,6 +192,16 @@ KALSHI_ENDPOINTS: dict[str, KalshiEndpointConfig] = {
paginated=False,
city_fan_out=True,
),
+ # Each city's append-only index configuration timeline: the launch configuration plus every
+ # weekly offset calibration. It has no filter and stays small, so it is a full refresh.
+ "weather_index_calibrations": KalshiEndpointConfig(
+ name="weather_index_calibrations",
+ path="/live_data/weather/{city}/calibrations",
+ data_key="calibrations",
+ primary_keys=["city", "config_version"],
+ paginated=False,
+ city_fan_out=True,
+ ),
# The account's own executions. `min_ts` filters on the fill time, and fills come back
# newest-first like public trades.
"fills": KalshiEndpointConfig(
@@ -275,6 +285,19 @@ KALSHI_ENDPOINTS: dict[str, KalshiEndpointConfig] = {
subaccount_fan_out=True,
page_size=1000,
),
+ # Credits to the account, newest-first. A credit's status moves from pending to applied or
+ # failed, but `min_ts` filters on creation time, so an incremental run would never see that
+ # change on an older credit. This is a full refresh.
+ "credit_history": KalshiEndpointConfig(
+ name="credit_history",
+ path="/portfolio/credit_history",
+ data_key="credits",
+ primary_keys=["credit_id"],
+ partition_key="created_time",
+ newest_first=True,
+ requires_auth=True,
+ page_size=1000,
+ ),
}
ENDPOINTS = tuple(KALSHI_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py
index 51112b038912..99857d129907 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/source.py
@@ -62,9 +62,9 @@ class KalshiSource(ResumableSource[KalshiSourceConfig, KalshiResumeConfig]):
releaseStatus=ReleaseStatus.BETA,
caption="""Kalshi's market data endpoints are public, so this source needs no API key for them.
-It syncs markets, events, series, trades, milestones, daily market candlesticks, and the per-minute weather index for each city. It also syncs the historical archive of markets and trades.
+It syncs markets, events, series, trades, milestones, daily market candlesticks, and the per-minute weather index and its calibration history for each city. It also syncs the historical archive of markets and trades.
-To sync your own fills, orders, positions, and settlements, add an API key. Create one in your [Kalshi account settings](https://kalshi.com/account/profile) and paste the key ID and the private key file contents below. The key only needs read access.""",
+To sync your own fills, orders, positions, settlements, and credit history, add an API key. Create one in your [Kalshi account settings](https://kalshi.com/account/profile) and paste the key ID and the private key file contents below. The key only needs read access.""",
iconPath="/static/services/kalshi.png",
docsUrl="https://posthog.com/docs/cdp/sources/kalshi",
keywords=["prediction market", "event contracts", "trading"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py
index 525322e1855f..b7f0d5115235 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kalshi/tests/test_kalshi.py
@@ -131,7 +131,9 @@ class TestKalshiTransport:
assert result.requests[0].param("min_ts") == "1609459200"
assert result.requests[0].param("limit") == str(KALSHI_ENDPOINTS[endpoint].page_size)
- @parameterized.expand(["markets", "events", "milestones", "historical_markets", "orders", "historical_orders"])
+ @parameterized.expand(
+ ["markets", "events", "milestones", "historical_markets", "orders", "historical_orders", "credit_history"]
+ )
def test_non_incremental_endpoints_never_send_min_ts(self, endpoint: str) -> None:
# Kalshi answers 200 and ignores params it doesn't recognise, so a filter wired onto an
# endpoint that has none would look like it worked while syncing everything every run.
@@ -529,30 +531,34 @@ class TestKalshiWeatherIndex:
@parameterized.expand(
[
# `from=0` would silently fall back to the trailing 24 hours.
- ("full_refresh", {"should_use_incremental_field": False}, "1"),
+ ("full_refresh", "weather_index", {"should_use_incremental_field": False}, "1"),
(
"incremental_rereads_the_last_hour",
+ "weather_index",
{"incremental_field": "t", "db_incremental_field_last_value": 1_790_000_000_000},
str(1_790_000_000_000 - 3_600_000),
),
+ # The calibration timeline takes no window, so it is read whole for every city.
+ ("calibrations", "weather_index_calibrations", {"should_use_incremental_field": False}, None),
]
)
def test_reads_every_city_from_the_window_start(
- self, _name: str, inputs: dict[str, Any], expected_from: str
+ self, _name: str, endpoint: str, inputs: dict[str, Any], expected_from: str | None
) -> None:
- point = {"t": 1_790_000_060_000, "v": 79.16, "status": "normal", "contributors": 5}
+ config = KALSHI_ENDPOINTS[endpoint]
+ row = {"t": 1_790_000_060_000, "v": 79.16, "config_version": "v1.0"}
result = _driver().run(
- "weather_index",
- always(ScriptedResponse(json={"city": "any", "units": "fahrenheit", "timeseries": [point]})),
+ endpoint,
+ always(ScriptedResponse(json={"city": "any", "units": "fahrenheit", config.data_key: [row]})),
**inputs,
)
assert result.raised is None
- assert result.paths == [f"/trade-api/v2/live_data/weather/{city}" for city in WEATHER_INDEX_CITIES]
+ assert result.paths == [f"/trade-api/v2{config.path.format(city=city)}" for city in WEATHER_INDEX_CITIES]
assert set(result.params("from")) == {expected_from}
- assert all(to is not None for to in result.params("to"))
- # The city is half of the primary key; without it every city's minute collides.
- assert result.rows == [{"city": city, **point} for city in WEATHER_INDEX_CITIES]
+ assert all((to is not None) == (expected_from is not None) for to in result.params("to"))
+ # The city is half of the primary key; without it every city's rows collide.
+ assert result.rows == [{"city": city, **row} for city in WEATHER_INDEX_CITIES]
assert result.response is not None
assert result.response.sort_mode == "desc"