:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add rollbar daily count tables (#128416)
Tom Owers085616afccde
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 3198eb341a5a..f2892542da13 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10412,13 +10412,13 @@ Note: Both official reporting references were fetched with curl and their operat## Rollbar — gaps-Today (4): `deploys`, `environments`, `items`, `occurrences`+Today (6): `activated_item_counts`, `deploys`, `environments`, `items`, `occurrence_counts`, `occurrences`Diffed against: <https://docs.rollbar.com/reference>- [ ] `projects (list-all-projects)` — lookup table resolving the project IDs carried on items and deploys we already sync (high)-- [ ] `item occurrence counts (get-occurrence-counts)` — the per-item error-volume time series, Rollbar's headline metric (high)-- [ ] `activated item counts (get-activated-item-counts)` — new-error-rate trend over time, the standard regression signal (high)+- [x] `item occurrence counts (get-occurrence-counts)` — the per-item error-volume time series, Rollbar's headline metric (high)+- [x] `activated item counts (get-activated-item-counts)` — new-error-rate trend over time, the standard regression signal (high)- [ ] `users (list-all-users)` — lookup resolving assignee and resolver IDs on items (medium)- [ ] `teams (list-all-teams)` — lookup for ownership attribution of items and projects (medium)- [ ] `top active items (get-top-active-items)` — ranked noisiest errors per period without recomputing from occurrences (medium)diff --git a/products/warehouse_sources/sources/rollbar/canonical_descriptions.py b/products/warehouse_sources/sources/rollbar/canonical_descriptions.pyindex c5b444e41ff3..c0419aad9bec 100644--- a/products/warehouse_sources/sources/rollbar/canonical_descriptions.py+++ b/products/warehouse_sources/sources/rollbar/canonical_descriptions.py@@ -70,4 +70,20 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"name": "The environment's name.",},},+ "occurrence_counts": {+ "description": "Daily count of active occurrences in the Rollbar project, at all levels. Each row is one day bucket in GMT.",+ "docs_url": "https://docs.rollbar.com/reference/get-occurrence-counts",+ "columns": {+ "timestamp": "Start of the day bucket (Unix seconds).",+ "count": "Number of active occurrences from the bucket start to the end of that day.",+ },+ },+ "activated_item_counts": {+ "description": "Daily count of items that were first seen or reactivated in the Rollbar project. Each row is one day bucket in the project timezone.",+ "docs_url": "https://docs.rollbar.com/reference/get-activated-item-counts",+ "columns": {+ "timestamp": "Start of the day bucket (Unix seconds).",+ "count": "Number of items first seen or reactivated in that day.",+ },+ },}diff --git a/products/warehouse_sources/sources/rollbar/rollbar.py b/products/warehouse_sources/sources/rollbar/rollbar.pyindex 63d8a47e0db3..1c84dba86bcc 100644--- a/products/warehouse_sources/sources/rollbar/rollbar.py+++ b/products/warehouse_sources/sources/rollbar/rollbar.py@@ -59,6 +59,20 @@ def _extract_items(body: Any, data_key: Optional[str]) -> list[dict[str, Any]]:return []+def _extract_count_rows(body: Any) -> list[dict[str, Any]]:+ """Turn the `[timestamp, count]` pairs of a count report into rows, oldest bucket first."""+ result = body.get("result") if isinstance(body, dict) else None+ rows: list[dict[str, Any]] = []+ for pair in result if isinstance(result, list) else []:+ if not isinstance(pair, list) or len(pair) < 2:+ continue+ timestamp = _to_int(pair[0])+ if timestamp is None:+ continue+ rows.append({"timestamp": timestamp, "count": _to_int(pair[1])})+ return sorted(rows, key=lambda row: row["timestamp"])++def _build_url(path: str, params: dict[str, Any]) -> str:if not params:return f"{ROLLBAR_BASE_URL}{path}"@@ -118,6 +132,12 @@ def get_rows(return response.json()+ if config.pagination == "time_series":+ rows = _extract_count_rows(fetch(_build_url(config.path, config.params)))+ if rows:+ yield rows+ return+resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Noneif config.pagination == "keyset":diff --git a/products/warehouse_sources/sources/rollbar/settings.py b/products/warehouse_sources/sources/rollbar/settings.pyindex d41d603e1a21..54903973bf4d 100644--- a/products/warehouse_sources/sources/rollbar/settings.py+++ b/products/warehouse_sources/sources/rollbar/settings.py@@ -1,5 +1,5 @@from dataclasses import dataclass, field-from typing import Literal, Optional+from typing import Any, Literal, Optionalfrom products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType@@ -13,17 +13,28 @@ class RollbarEndpointConfig:# is only for the defensive case where `result` is itself a bare list.data_key: Optional[str]primary_key: str = "id"- pagination: Literal["page", "keyset"] = "page"+ pagination: Literal["page", "keyset", "time_series"] = "page"incremental_fields: list[IncrementalField] = field(default_factory=list)# Stable event-time field used for datetime partitioning.partition_key: Optional[str] = None+ params: dict[str, Any] = field(default_factory=dict)+_TIMESTAMP_INCREMENTAL_FIELD: IncrementalField = {+ "label": "timestamp",+ "type": IncrementalFieldType.Integer,+ "field": "timestamp",+ "field_type": IncrementalFieldType.Integer,+}+# Rollbar's project-scoped API (api.rollbar.com/api/1, X-Rollbar-Access-Token# header with a read-scope project token). No list endpoint has a server-side# timestamp filter; occurrences are strictly descending by id, so they get an# id high-water-mark incremental (walk from the head until crossing the# watermark). Items mutate in place (counters), so they stay full refresh.+# The count reports return one fixed window of recent daily buckets in a single+# request, so an incremental sync re-reads that window and merges on+# `timestamp`, which keeps the buckets that later fall out of the window.ROLLBAR_ENDPOINTS: dict[str, RollbarEndpointConfig] = {"items": RollbarEndpointConfig(name="items",@@ -55,6 +66,25 @@ ROLLBAR_ENDPOINTS: dict[str, RollbarEndpointConfig] = {path="/environments",data_key="environments",),+ "occurrence_counts": RollbarEndpointConfig(+ name="occurrence_counts",+ path="/reports/occurrence_counts",+ data_key=None,+ primary_key="timestamp",+ pagination="time_series",+ # The API counts only error and critical occurrences unless `min_level` is lower.+ params={"bucket_size": 86400, "min_level": "debug"},+ incremental_fields=[_TIMESTAMP_INCREMENTAL_FIELD],+ ),+ "activated_item_counts": RollbarEndpointConfig(+ name="activated_item_counts",+ path="/reports/activated_counts",+ data_key=None,+ primary_key="timestamp",+ pagination="time_series",+ params={"bucket_size": 86400, "buckets": 366},+ incremental_fields=[_TIMESTAMP_INCREMENTAL_FIELD],+ ),}ENDPOINTS = tuple(ROLLBAR_ENDPOINTS.keys())diff --git a/products/warehouse_sources/sources/rollbar/tests/test_rollbar.py b/products/warehouse_sources/sources/rollbar/tests/test_rollbar.pyindex d88df4bd9fd0..c17e536b9b30 100644--- a/products/warehouse_sources/sources/rollbar/tests/test_rollbar.py+++ b/products/warehouse_sources/sources/rollbar/tests/test_rollbar.py@@ -166,6 +166,50 @@ class TestGetRowsKeyset:assert result.params("lastId") == [None]+class TestGetRowsTimeSeries:+ @pytest.mark.parametrize(+ "endpoint, expected_path, expected_params",+ [+ (+ "occurrence_counts",+ "/api/1/reports/occurrence_counts",+ {"bucket_size": ("86400",), "min_level": ("debug",)},+ ),+ (+ "activated_item_counts",+ "/api/1/reports/activated_counts",+ {"bucket_size": ("86400",), "buckets": ("366",)},+ ),+ ],+ )+ @pytest.mark.parametrize(+ "incremental_inputs",+ [+ {},+ # The window is one request, so an incremental run reads and merges all of it.+ {"incremental_field": "timestamp", "db_incremental_field_last_value": 1728561200},+ ],+ )+ def test_reads_one_window_as_sorted_rows(self, endpoint, expected_path, expected_params, incremental_inputs):+ body = ScriptedResponse(+ json={+ "err": 0,+ "result": [[1728647600, "4"], [1728561200, 0], "bad", [None, 3], [1728604400], [1728604400, 6]],+ }+ )++ result = _driver().run(endpoint, always(body), **incremental_inputs)++ assert result.raised is None+ assert result.paths == [expected_path]+ assert result.queries == [expected_params]+ assert result.rows == [+ {"timestamp": 1728561200, "count": 0},+ {"timestamp": 1728604400, "count": 6},+ {"timestamp": 1728647600, "count": 4},+ ]++class TestRollbarSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint):