:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): store everflow amounts as floats (#122389)
posthog[bot]8d200a3ed5a7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/everflow.py b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/everflow.pyindex e0734eb9a51e..e28178d31769 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/everflow.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/everflow.py@@ -121,11 +121,18 @@ def flatten_entity_row(row: dict[str, Any], day: str, currency_code: str) -> dicreturn flat-def _without(columns: tuple[str, ...]) -> Callable[[dict[str, Any]], dict[str, Any]]:- def drop(row: dict[str, Any]) -> dict[str, Any]:- return {key: value for key, value in row.items() if key not in columns}+def _row_map(+ drop_columns: tuple[str, ...], float_columns: tuple[str, ...]+) -> Callable[[dict[str, Any]], dict[str, Any]]:+ def apply(row: dict[str, Any]) -> dict[str, Any]:+ mapped = {key: value for key, value in row.items() if key not in drop_columns}+ for column in float_columns:+ value = mapped.get(column)+ if isinstance(value, int) and not isinstance(value, bool):+ mapped[column] = float(value)+ return mapped- return drop+ return applydef _api_key_auth(api_key: str) -> APIKeyAuth:@@ -140,6 +147,7 @@ def _daily_performance(start: date,end: date,manager: ResumableSourceManager[EverflowResumeConfig],+ row_map: Callable[[dict[str, Any]], dict[str, Any]],) -> Iterator[list[dict[str, Any]]]:client = RESTClient(base_url=base_url, auth=_api_key_auth(api_key), request_timeout=REQUEST_TIMEOUT)day = start@@ -163,7 +171,8 @@ def _daily_performance(f"Everflow's daily report for {day.isoformat()} hit its 10,000-row limit")rows.extend(- flatten_entity_row(row, day.isoformat(), currency_code) for row in response_body.get("table") or []+ row_map(flatten_entity_row(row, day.isoformat(), currency_code))+ for row in response_body.get("table") or [])if next_day <= end:@@ -201,6 +210,7 @@ def everflow_source("partition_format": "month" if config.partition_key else None,"partition_keys": [config.partition_key] if config.partition_key else None,}+ row_map = _row_map(config.drop_columns, config.float_columns)if endpoint == "daily_performance":start = (@@ -210,7 +220,7 @@ def everflow_source()return SourceResponse(items=lambda: _daily_performance(- api_key, base_url, config.path, currency_code, start, today, resumable_source_manager+ api_key, base_url, config.path, currency_code, start, today, resumable_source_manager, row_map),# Days are requested in order, so rows arrive in date order.sort_mode="asc",@@ -259,7 +269,7 @@ def everflow_source("data_selector": config.data_selector,},"table_format": "delta",- **({"data_map": _without(config.drop_columns)} if config.drop_columns else {}),+ **({"data_map": row_map} if config.drop_columns or config.float_columns else {}),}],}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/settings.pyindex 455bbae0ea05..4a81a57db457 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/settings.py@@ -37,11 +37,19 @@ class EverflowEndpointConfig:default_incremental_lookback_seconds: int | None = None# Columns that carry credentials and must never reach the warehouse.drop_columns: tuple[str, ...] = ()+ # Fields Everflow documents as numbers but sends as a JSON integer when whole. Without the cast, a+ # first batch of whole values creates the column as int64 and a later fractional value fails the load.+ float_columns: tuple[str, ...] = ()EVERFLOW_ENDPOINTS: dict[str, EverflowEndpointConfig] = {"offers": EverflowEndpointConfig(- name="offers", path="/networks/offerstable", data_selector="offers", primary_keys=("network_offer_id",)+ name="offers",+ path="/networks/offerstable",+ data_selector="offers",+ primary_keys=("network_offer_id",),+ # `payout` and `revenue` hold formatted text, so only the `*_amount` fields are numbers.+ float_columns=("payout_amount", "revenue_amount", "today_payout_amount", "today_revenue_amount"),),"affiliates": EverflowEndpointConfig(name="affiliates",@@ -64,6 +72,7 @@ EVERFLOW_ENDPOINTS: dict[str, EverflowEndpointConfig] = {primary_keys=("conversion_id",),page_size=CONVERSIONS_PAGE_SIZE,partition_key="conversion_unix_timestamp",+ float_columns=("payout", "revenue", "sale_amount"),incremental_fields=({"label": "conversion_unix_timestamp",@@ -79,6 +88,31 @@ EVERFLOW_ENDPOINTS: dict[str, EverflowEndpointConfig] = {data_selector="table",primary_keys=("date", "network_offer_id", "network_affiliate_id"),page_size=None,+ # Every amount and rate in the report's `reporting` object. Its counts (clicks, conversions, events)+ # are documented as integers and stay int64.+ float_columns=(+ "avg_sale_value",+ "cpa",+ "cpc",+ "cpm",+ "ctr",+ "cvr",+ "epc",+ "event_revenue",+ "evr",+ "gross_sales",+ "margin",+ "media_buying_cost",+ "on_hold_potential_payout",+ "on_hold_potential_revenue",+ "payout",+ "profit",+ "revenue",+ "roas",+ "rpa",+ "rpc",+ "rpm",+ ),incremental_fields=({"label": "date",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/tests/test_everflow.py b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/tests/test_everflow.pyindex a843ae0df106..8ad38914b23c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/tests/test_everflow.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/tests/test_everflow.py@@ -80,6 +80,42 @@ class TestTables:assert result.raised is Noneassert result.rows == [{"network_advertiser_id": 135, "name": "Brand"}]+ @pytest.mark.parametrize(+ ("endpoint", "row", "expected_types"),+ [+ (+ "conversions",+ {"conversion_id": "c1", "payout": 30, "revenue": None, "sale_amount": 0},+ {"conversion_id": str, "payout": float, "revenue": type(None), "sale_amount": float},+ ),+ (+ "offers",+ {+ "network_offer_id": 1,+ "payout_amount": 30,+ "revenue_amount": 4.5,+ "today_payout_amount": 0,+ "today_revenue_amount": None,+ },+ {+ "network_offer_id": int,+ "payout_amount": float,+ "revenue_amount": float,+ "today_payout_amount": float,+ "today_revenue_amount": type(None),+ },+ ),+ ],+ )+ def test_whole_amounts_arrive_as_floats(+ self, endpoint: str, row: dict[str, Any], expected_types: dict[str, type]+ ) -> None:+ result = _run(endpoint, [_table_page(endpoint, [row], total=1)])++ assert result.raised is None+ assert result.rows == [row]+ assert [{column: type(value) for column, value in mapped.items()} for mapped in result.rows] == [expected_types]+def test_a_rejected_key_fails_the_run_with_the_status_the_error_map_matches(self) -> None:result = _run("affiliates", always(ScriptedResponse(status=401, json={"Error": "Unable to authenticate request"}))@@ -192,6 +228,24 @@ class TestDailyPerformance:assert result.raised is Noneassert [request.json()["from"] for request in result.requests] == ["2026-10-15"]+ @time_machine.travel(NOW, tick=False)+ def test_whole_amounts_arrive_as_floats(self) -> None:+ row = _entity_row("882", "102", 7) | {+ "reporting": {"total_click": 7, "cv": 1, "payout": 30, "cpm": 2, "rpm": None}+ }+ result = _run(+ "daily_performance",+ [ScriptedResponse(json={"table": [row]})],+ incremental_field="date",+ db_incremental_field_last_value=date(2026, 10, 15),+ )++ assert result.raised is None+ assert [+ {column: type(value) for column, value in flat.items() if column in row["reporting"]}+ for flat in result.rows+ ] == [{"total_click": int, "cv": int, "payout": float, "cpm": float, "rpm": type(None)}]+class TestFlattenEntityRow:@pytest.mark.parametrize(