Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): store everflow amounts as floats (#122389)

posthog[bot]8d200a3ed5a7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/everflow.py b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/everflow.py
index e0734eb9a51e..e28178d31769 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/everflow.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/everflow.py
@@ -121,11 +121,18 @@ def flatten_entity_row(row: dict[str, Any], day: str, currency_code: str) -> dic
return flat
-def _without(columns: tuple[str, ...]) -> Callable[[dict[str, Any]], dict[str, Any]]:
- def drop(row: dict[str, Any]) -> dict[str, Any]:
- return {key: value for key, value in row.items() if key not in columns}
+def _row_map(
+ drop_columns: tuple[str, ...], float_columns: tuple[str, ...]
+) -> Callable[[dict[str, Any]], dict[str, Any]]:
+ def apply(row: dict[str, Any]) -> dict[str, Any]:
+ mapped = {key: value for key, value in row.items() if key not in drop_columns}
+ for column in float_columns:
+ value = mapped.get(column)
+ if isinstance(value, int) and not isinstance(value, bool):
+ mapped[column] = float(value)
+ return mapped
- return drop
+ return apply
def _api_key_auth(api_key: str) -> APIKeyAuth:
@@ -140,6 +147,7 @@ def _daily_performance(
start: date,
end: date,
manager: ResumableSourceManager[EverflowResumeConfig],
+ row_map: Callable[[dict[str, Any]], dict[str, Any]],
) -> Iterator[list[dict[str, Any]]]:
client = RESTClient(base_url=base_url, auth=_api_key_auth(api_key), request_timeout=REQUEST_TIMEOUT)
day = start
@@ -163,7 +171,8 @@ def _daily_performance(
f"Everflow's daily report for {day.isoformat()} hit its 10,000-row limit"
)
rows.extend(
- flatten_entity_row(row, day.isoformat(), currency_code) for row in response_body.get("table") or []
+ row_map(flatten_entity_row(row, day.isoformat(), currency_code))
+ for row in response_body.get("table") or []
)
if next_day <= end:
@@ -201,6 +210,7 @@ def everflow_source(
"partition_format": "month" if config.partition_key else None,
"partition_keys": [config.partition_key] if config.partition_key else None,
}
+ row_map = _row_map(config.drop_columns, config.float_columns)
if endpoint == "daily_performance":
start = (
@@ -210,7 +220,7 @@ def everflow_source(
)
return SourceResponse(
items=lambda: _daily_performance(
- api_key, base_url, config.path, currency_code, start, today, resumable_source_manager
+ api_key, base_url, config.path, currency_code, start, today, resumable_source_manager, row_map
),
# Days are requested in order, so rows arrive in date order.
sort_mode="asc",
@@ -259,7 +269,7 @@ def everflow_source(
"data_selector": config.data_selector,
},
"table_format": "delta",
- **({"data_map": _without(config.drop_columns)} if config.drop_columns else {}),
+ **({"data_map": row_map} if config.drop_columns or config.float_columns else {}),
}
],
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/settings.py
index 455bbae0ea05..4a81a57db457 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/settings.py
@@ -37,11 +37,19 @@ class EverflowEndpointConfig:
default_incremental_lookback_seconds: int | None = None
# Columns that carry credentials and must never reach the warehouse.
drop_columns: tuple[str, ...] = ()
+ # Fields Everflow documents as numbers but sends as a JSON integer when whole. Without the cast, a
+ # first batch of whole values creates the column as int64 and a later fractional value fails the load.
+ float_columns: tuple[str, ...] = ()
EVERFLOW_ENDPOINTS: dict[str, EverflowEndpointConfig] = {
"offers": EverflowEndpointConfig(
- name="offers", path="/networks/offerstable", data_selector="offers", primary_keys=("network_offer_id",)
+ name="offers",
+ path="/networks/offerstable",
+ data_selector="offers",
+ primary_keys=("network_offer_id",),
+ # `payout` and `revenue` hold formatted text, so only the `*_amount` fields are numbers.
+ float_columns=("payout_amount", "revenue_amount", "today_payout_amount", "today_revenue_amount"),
),
"affiliates": EverflowEndpointConfig(
name="affiliates",
@@ -64,6 +72,7 @@ EVERFLOW_ENDPOINTS: dict[str, EverflowEndpointConfig] = {
primary_keys=("conversion_id",),
page_size=CONVERSIONS_PAGE_SIZE,
partition_key="conversion_unix_timestamp",
+ float_columns=("payout", "revenue", "sale_amount"),
incremental_fields=(
{
"label": "conversion_unix_timestamp",
@@ -79,6 +88,31 @@ EVERFLOW_ENDPOINTS: dict[str, EverflowEndpointConfig] = {
data_selector="table",
primary_keys=("date", "network_offer_id", "network_affiliate_id"),
page_size=None,
+ # Every amount and rate in the report's `reporting` object. Its counts (clicks, conversions, events)
+ # are documented as integers and stay int64.
+ float_columns=(
+ "avg_sale_value",
+ "cpa",
+ "cpc",
+ "cpm",
+ "ctr",
+ "cvr",
+ "epc",
+ "event_revenue",
+ "evr",
+ "gross_sales",
+ "margin",
+ "media_buying_cost",
+ "on_hold_potential_payout",
+ "on_hold_potential_revenue",
+ "payout",
+ "profit",
+ "revenue",
+ "roas",
+ "rpa",
+ "rpc",
+ "rpm",
+ ),
incremental_fields=(
{
"label": "date",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/tests/test_everflow.py b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/tests/test_everflow.py
index a843ae0df106..8ad38914b23c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/tests/test_everflow.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/everflow/tests/test_everflow.py
@@ -80,6 +80,42 @@ class TestTables:
assert result.raised is None
assert result.rows == [{"network_advertiser_id": 135, "name": "Brand"}]
+ @pytest.mark.parametrize(
+ ("endpoint", "row", "expected_types"),
+ [
+ (
+ "conversions",
+ {"conversion_id": "c1", "payout": 30, "revenue": None, "sale_amount": 0},
+ {"conversion_id": str, "payout": float, "revenue": type(None), "sale_amount": float},
+ ),
+ (
+ "offers",
+ {
+ "network_offer_id": 1,
+ "payout_amount": 30,
+ "revenue_amount": 4.5,
+ "today_payout_amount": 0,
+ "today_revenue_amount": None,
+ },
+ {
+ "network_offer_id": int,
+ "payout_amount": float,
+ "revenue_amount": float,
+ "today_payout_amount": float,
+ "today_revenue_amount": type(None),
+ },
+ ),
+ ],
+ )
+ def test_whole_amounts_arrive_as_floats(
+ self, endpoint: str, row: dict[str, Any], expected_types: dict[str, type]
+ ) -> None:
+ result = _run(endpoint, [_table_page(endpoint, [row], total=1)])
+
+ assert result.raised is None
+ assert result.rows == [row]
+ assert [{column: type(value) for column, value in mapped.items()} for mapped in result.rows] == [expected_types]
+
def test_a_rejected_key_fails_the_run_with_the_status_the_error_map_matches(self) -> None:
result = _run(
"affiliates", always(ScriptedResponse(status=401, json={"Error": "Unable to authenticate request"}))
@@ -192,6 +228,24 @@ class TestDailyPerformance:
assert result.raised is None
assert [request.json()["from"] for request in result.requests] == ["2026-10-15"]
+ @time_machine.travel(NOW, tick=False)
+ def test_whole_amounts_arrive_as_floats(self) -> None:
+ row = _entity_row("882", "102", 7) | {
+ "reporting": {"total_click": 7, "cv": 1, "payout": 30, "cpm": 2, "rpm": None}
+ }
+ result = _run(
+ "daily_performance",
+ [ScriptedResponse(json={"table": [row]})],
+ incremental_field="date",
+ db_incremental_field_last_value=date(2026, 10, 15),
+ )
+
+ assert result.raised is None
+ assert [
+ {column: type(value) for column, value in flat.items() if column in row["reporting"]}
+ for flat in result.rows
+ ] == [{"total_click": int, "cv": int, "payout": float, "cpm": float, "rpm": type(None)}]
+
class TestFlattenEntityRow:
@pytest.mark.parametrize(