Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add rollbar daily count tables (#128416)

Tom Owers085616afccde
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 3198eb341a5a..f2892542da13 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10412,13 +10412,13 @@ Note: Both official reporting references were fetched with curl and their operat
## Rollbar — gaps
-Today (4): `deploys`, `environments`, `items`, `occurrences`
+Today (6): `activated_item_counts`, `deploys`, `environments`, `items`, `occurrence_counts`, `occurrences`
Diffed against: <https://docs.rollbar.com/reference>
- [ ] `projects (list-all-projects)` — lookup table resolving the project IDs carried on items and deploys we already sync (high)
-- [ ] `item occurrence counts (get-occurrence-counts)` — the per-item error-volume time series, Rollbar's headline metric (high)
-- [ ] `activated item counts (get-activated-item-counts)` — new-error-rate trend over time, the standard regression signal (high)
+- [x] `item occurrence counts (get-occurrence-counts)` — the per-item error-volume time series, Rollbar's headline metric (high)
+- [x] `activated item counts (get-activated-item-counts)` — new-error-rate trend over time, the standard regression signal (high)
- [ ] `users (list-all-users)` — lookup resolving assignee and resolver IDs on items (medium)
- [ ] `teams (list-all-teams)` — lookup for ownership attribution of items and projects (medium)
- [ ] `top active items (get-top-active-items)` — ranked noisiest errors per period without recomputing from occurrences (medium)
diff --git a/products/warehouse_sources/sources/rollbar/canonical_descriptions.py b/products/warehouse_sources/sources/rollbar/canonical_descriptions.py
index c5b444e41ff3..c0419aad9bec 100644
--- a/products/warehouse_sources/sources/rollbar/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/rollbar/canonical_descriptions.py
@@ -70,4 +70,20 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"name": "The environment's name.",
},
},
+ "occurrence_counts": {
+ "description": "Daily count of active occurrences in the Rollbar project, at all levels. Each row is one day bucket in GMT.",
+ "docs_url": "https://docs.rollbar.com/reference/get-occurrence-counts",
+ "columns": {
+ "timestamp": "Start of the day bucket (Unix seconds).",
+ "count": "Number of active occurrences from the bucket start to the end of that day.",
+ },
+ },
+ "activated_item_counts": {
+ "description": "Daily count of items that were first seen or reactivated in the Rollbar project. Each row is one day bucket in the project timezone.",
+ "docs_url": "https://docs.rollbar.com/reference/get-activated-item-counts",
+ "columns": {
+ "timestamp": "Start of the day bucket (Unix seconds).",
+ "count": "Number of items first seen or reactivated in that day.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/rollbar/rollbar.py b/products/warehouse_sources/sources/rollbar/rollbar.py
index 63d8a47e0db3..1c84dba86bcc 100644
--- a/products/warehouse_sources/sources/rollbar/rollbar.py
+++ b/products/warehouse_sources/sources/rollbar/rollbar.py
@@ -59,6 +59,20 @@ def _extract_items(body: Any, data_key: Optional[str]) -> list[dict[str, Any]]:
return []
+def _extract_count_rows(body: Any) -> list[dict[str, Any]]:
+ """Turn the `[timestamp, count]` pairs of a count report into rows, oldest bucket first."""
+ result = body.get("result") if isinstance(body, dict) else None
+ rows: list[dict[str, Any]] = []
+ for pair in result if isinstance(result, list) else []:
+ if not isinstance(pair, list) or len(pair) < 2:
+ continue
+ timestamp = _to_int(pair[0])
+ if timestamp is None:
+ continue
+ rows.append({"timestamp": timestamp, "count": _to_int(pair[1])})
+ return sorted(rows, key=lambda row: row["timestamp"])
+
+
def _build_url(path: str, params: dict[str, Any]) -> str:
if not params:
return f"{ROLLBAR_BASE_URL}{path}"
@@ -118,6 +132,12 @@ def get_rows(
return response.json()
+ if config.pagination == "time_series":
+ rows = _extract_count_rows(fetch(_build_url(config.path, config.params)))
+ if rows:
+ yield rows
+ return
+
resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
if config.pagination == "keyset":
diff --git a/products/warehouse_sources/sources/rollbar/settings.py b/products/warehouse_sources/sources/rollbar/settings.py
index d41d603e1a21..54903973bf4d 100644
--- a/products/warehouse_sources/sources/rollbar/settings.py
+++ b/products/warehouse_sources/sources/rollbar/settings.py
@@ -1,5 +1,5 @@
from dataclasses import dataclass, field
-from typing import Literal, Optional
+from typing import Any, Literal, Optional
from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType
@@ -13,17 +13,28 @@ class RollbarEndpointConfig:
# is only for the defensive case where `result` is itself a bare list.
data_key: Optional[str]
primary_key: str = "id"
- pagination: Literal["page", "keyset"] = "page"
+ pagination: Literal["page", "keyset", "time_series"] = "page"
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Stable event-time field used for datetime partitioning.
partition_key: Optional[str] = None
+ params: dict[str, Any] = field(default_factory=dict)
+_TIMESTAMP_INCREMENTAL_FIELD: IncrementalField = {
+ "label": "timestamp",
+ "type": IncrementalFieldType.Integer,
+ "field": "timestamp",
+ "field_type": IncrementalFieldType.Integer,
+}
+
# Rollbar's project-scoped API (api.rollbar.com/api/1, X-Rollbar-Access-Token
# header with a read-scope project token). No list endpoint has a server-side
# timestamp filter; occurrences are strictly descending by id, so they get an
# id high-water-mark incremental (walk from the head until crossing the
# watermark). Items mutate in place (counters), so they stay full refresh.
+# The count reports return one fixed window of recent daily buckets in a single
+# request, so an incremental sync re-reads that window and merges on
+# `timestamp`, which keeps the buckets that later fall out of the window.
ROLLBAR_ENDPOINTS: dict[str, RollbarEndpointConfig] = {
"items": RollbarEndpointConfig(
name="items",
@@ -55,6 +66,25 @@ ROLLBAR_ENDPOINTS: dict[str, RollbarEndpointConfig] = {
path="/environments",
data_key="environments",
),
+ "occurrence_counts": RollbarEndpointConfig(
+ name="occurrence_counts",
+ path="/reports/occurrence_counts",
+ data_key=None,
+ primary_key="timestamp",
+ pagination="time_series",
+ # The API counts only error and critical occurrences unless `min_level` is lower.
+ params={"bucket_size": 86400, "min_level": "debug"},
+ incremental_fields=[_TIMESTAMP_INCREMENTAL_FIELD],
+ ),
+ "activated_item_counts": RollbarEndpointConfig(
+ name="activated_item_counts",
+ path="/reports/activated_counts",
+ data_key=None,
+ primary_key="timestamp",
+ pagination="time_series",
+ params={"bucket_size": 86400, "buckets": 366},
+ incremental_fields=[_TIMESTAMP_INCREMENTAL_FIELD],
+ ),
}
ENDPOINTS = tuple(ROLLBAR_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/sources/rollbar/tests/test_rollbar.py b/products/warehouse_sources/sources/rollbar/tests/test_rollbar.py
index d88df4bd9fd0..c17e536b9b30 100644
--- a/products/warehouse_sources/sources/rollbar/tests/test_rollbar.py
+++ b/products/warehouse_sources/sources/rollbar/tests/test_rollbar.py
@@ -166,6 +166,50 @@ class TestGetRowsKeyset:
assert result.params("lastId") == [None]
+class TestGetRowsTimeSeries:
+ @pytest.mark.parametrize(
+ "endpoint, expected_path, expected_params",
+ [
+ (
+ "occurrence_counts",
+ "/api/1/reports/occurrence_counts",
+ {"bucket_size": ("86400",), "min_level": ("debug",)},
+ ),
+ (
+ "activated_item_counts",
+ "/api/1/reports/activated_counts",
+ {"bucket_size": ("86400",), "buckets": ("366",)},
+ ),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "incremental_inputs",
+ [
+ {},
+ # The window is one request, so an incremental run reads and merges all of it.
+ {"incremental_field": "timestamp", "db_incremental_field_last_value": 1728561200},
+ ],
+ )
+ def test_reads_one_window_as_sorted_rows(self, endpoint, expected_path, expected_params, incremental_inputs):
+ body = ScriptedResponse(
+ json={
+ "err": 0,
+ "result": [[1728647600, "4"], [1728561200, 0], "bad", [None, 3], [1728604400], [1728604400, 6]],
+ }
+ )
+
+ result = _driver().run(endpoint, always(body), **incremental_inputs)
+
+ assert result.raised is None
+ assert result.paths == [expected_path]
+ assert result.queries == [expected_params]
+ assert result.rows == [
+ {"timestamp": 1728561200, "count": 0},
+ {"timestamp": 1728604400, "count": 6},
+ {"timestamp": 1728647600, "count": 4},
+ ]
+
+
class TestRollbarSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_response_metadata_per_endpoint(self, endpoint):