Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add pingdom results, outages and performance tables (#125437)

Tom Owers0f2fb6431a3a
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 5cf9684e6a82..3fc947f24e55 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -849,8 +849,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/pingdom.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/pinterest_ads.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/pipedrive.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index dc1165705723..34ae7da36879 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9431,10 +9431,10 @@ Today (4): `alerts`, `checks`, `maintenance`, `probes`
Diffed against: <https://docs.pingdom.com/api/API_3.1.yaml>
-- [ ] `GET /results/{checkid}` — raw per-probe check results (status, response time, probe) - the underlying monitoring data, none of which is synced today (high)
-- [ ] `GET /summary.outage/{checkid}` — up/down state-transition history per check; the only way to compute uptime or incident duration (high)
-- [ ] `GET /summary.performance/{checkid}` — bucketed response-time series - Pingdom's headline metric (high)
-- [ ] `GET /tms/check` — transaction (synthetic scripted) checks are a separate check type entirely and are invisible in the synced checks table (high)
+- [x] `GET /results/{checkid}` — raw per-probe check results (status, response time, probe) - the underlying monitoring data, none of which is synced today (high)
+- [x] `GET /summary.outage/{checkid}` — up/down state-transition history per check; the only way to compute uptime or incident duration (high)
+- [x] `GET /summary.performance/{checkid}` — bucketed response-time series - Pingdom's headline metric (high)
+- [x] `GET /tms/check` — transaction (synthetic scripted) checks are a separate check type entirely and are invisible in the synced checks table (high)
- [ ] `GET /summary.average/{checkid}` — average response time over a period, optionally broken down by country/probe (medium)
- [ ] `GET /tms/check/{check_id}/report/status` — status/uptime history for transaction checks (medium)
- [ ] `GET /tms/check/{check_id}/report/performance` — response-time history for transaction checks (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/canonical_descriptions.py
index ba61078f7d28..f78f386f64cc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/canonical_descriptions.py
@@ -73,4 +73,57 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"messagefull": "Full text of the alert message.",
},
},
+ "transaction_checks": {
+ "description": "A transaction check that runs a scripted or recorded user flow (TMS) against a site.",
+ "docs_url": "https://docs.pingdom.com/api/#tag/TMS-Checks",
+ "columns": {
+ "id": "Unique identifier for the transaction check.",
+ "name": "The transaction check's display name.",
+ "type": "Type of transaction check: script for regular TMS checks, recording for Transaction Recorder checks.",
+ "active": "Whether the transaction check is active.",
+ "status": "Whether the check is currently passing or failing (successful, failing, unknown).",
+ "interval": "How often the check runs, in minutes.",
+ "region": "Region where the check runs (us-east, us-west, eu, au).",
+ "tags": "Tags applied to the transaction check.",
+ "created_at": "Time at which the check was created, as a Unix timestamp.",
+ "modified_at": "Time at which the check was last modified, as a Unix timestamp.",
+ "last_downtime_start": "Time the last downtime started, as a Unix timestamp.",
+ "last_downtime_end": "Time the last downtime ended, as a Unix timestamp.",
+ },
+ },
+ "results": {
+ "description": "A raw test result for an uptime check, recorded by one probe at one point in time.",
+ "docs_url": "https://docs.pingdom.com/api/#tag/Results",
+ "columns": {
+ "checkid": "ID of the check the result belongs to.",
+ "probeid": "ID of the probe that ran the test.",
+ "time": "Time of the test, as a Unix timestamp.",
+ "status": "Result of the test (up, down, unconfirmed, unknown).",
+ "responsetime": "Response time of the test, in milliseconds.",
+ "statusdesc": "Short description of the test status.",
+ "statusdesclong": "Long description of the test status.",
+ },
+ },
+ "outages": {
+ "description": "A period during which an uptime check stayed in one status. Consecutive rows form the check's up/down history; the first row of each check starts at the start of the synced window.",
+ "docs_url": "https://docs.pingdom.com/api/#tag/Summary.outage",
+ "columns": {
+ "checkid": "ID of the check the status period belongs to.",
+ "status": "Status of the check during the period (up, down, unknown).",
+ "timefrom": "Start of the period, as a Unix timestamp.",
+ "timeto": "End of the period, as a Unix timestamp.",
+ },
+ },
+ "performance": {
+ "description": "Hourly response-time and uptime summary for an uptime check.",
+ "docs_url": "https://docs.pingdom.com/api/#tag/Summary.performance",
+ "columns": {
+ "checkid": "ID of the check the summary belongs to.",
+ "starttime": "Start of the hour, as a Unix timestamp.",
+ "avgresponse": "Average response time during the hour, in milliseconds.",
+ "uptime": "Total uptime during the hour, in seconds.",
+ "downtime": "Total downtime during the hour, in seconds.",
+ "unmonitored": "Total unmonitored time during the hour, in seconds.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/pingdom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/pingdom.py
index 45ab6adbba56..989f37c61496 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/pingdom.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/pingdom.py
@@ -1,16 +1,30 @@
+import time
import dataclasses
+from collections.abc import Iterable, Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
OffsetPaginator,
+ SinglePagePaginator,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ Endpoint,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -22,12 +36,17 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.pingdom.se
PINGDOM_BASE_URL = "https://api.pingdom.com/api/3.1"
[email protected](frozen=True)
class PingdomResumeConfig:
# Pingdom paginates with limit/offset; the static query params (limit,
# incremental `from` filter) are deterministically rebuilt from the endpoint
# config and job inputs on resume.
offset: int = 0
+ # Per-check tables checkpoint the shape `build_dependent_resource` emits for the current time
+ # window, plus the window bounds, so a resumed run keeps the interrupted run's windows.
+ fanout_state: Optional[dict[str, Any]] = None
+ window_start: Optional[int] = None
+ window_end: Optional[int] = None
def _to_epoch(value: Any) -> Optional[int]:
@@ -49,6 +68,12 @@ def _to_epoch(value: Any) -> Optional[int]:
return None
+def _paginator(config: PingdomEndpointConfig) -> BasePaginator:
+ if not config.paginated:
+ return SinglePagePaginator()
+ return OffsetPaginator(limit=config.page_size, total_path=None, maximum_offset=config.maximum_offset)
+
+
def _build_endpoint(config: PingdomEndpointConfig, should_use_incremental_field: bool) -> Endpoint:
is_incremental = should_use_incremental_field and bool(config.incremental_fields)
@@ -70,10 +95,94 @@ def _build_endpoint(config: PingdomEndpointConfig, should_use_incremental_field:
# lacks the key yields zero rows and stops (no data_selector_required — Pingdom's small
# dimension endpoints legitimately return an empty/absent list).
"data_selector": config.data_key,
- "paginator": OffsetPaginator(limit=config.page_size, total_path=None),
+ "paginator": _paginator(config),
}
+@frozen
+class TimeWindow:
+ start: Optional[int]
+ end: int
+
+
+def _windows(window_start: Optional[int], window_end: int, max_window_seconds: Optional[int]) -> list[TimeWindow]:
+ if window_start is None or not max_window_seconds:
+ return [TimeWindow(start=window_start, end=window_end)]
+ windows = []
+ start = window_start
+ while True:
+ end = min(start + max_window_seconds, window_end)
+ windows.append(TimeWindow(start=start, end=end))
+ if end >= window_end:
+ return windows
+ start = end
+
+
+def _fanout_resource(
+ config: PingdomEndpointConfig,
+ fanout: DependentEndpointConfig,
+ client_config: ClientConfig,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[PingdomResumeConfig],
+ db_incremental_field_last_value: Optional[Any],
+ should_use_incremental_field: bool,
+) -> Resource:
+ parent_config = PINGDOM_ENDPOINTS[fanout.parent_name]
+
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume is not None and resume.window_end is not None:
+ window_end = resume.window_end
+ window_start = resume.window_start
+ else:
+ window_end = int(time.time())
+ window_start = window_end - config.lookback_seconds if config.lookback_seconds else None
+ is_incremental = should_use_incremental_field and bool(config.incremental_fields)
+ watermark = _to_epoch(db_incremental_field_last_value) if is_incremental else None
+ if watermark is not None:
+ # A catch-up after a long pause walks every window from the watermark, so no rows are skipped.
+ window_start = min(watermark, window_end)
+
+ def build_window(window: TimeWindow, fanout_state: Optional[dict[str, Any]]) -> Iterable[Any]:
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(
+ PingdomResumeConfig(fanout_state=state, window_start=window.start, window_end=window_end)
+ )
+
+ child_params: dict[str, Any] = {**config.params, "to": window.end}
+ if window.start is not None:
+ child_params["from"] = window.start
+
+ return build_dependent_resource(
+ endpoint_configs=PINGDOM_ENDPOINTS,
+ child_endpoint=config.name,
+ fanout=fanout,
+ client_config=client_config,
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra={"data_selector": parent_config.data_key, "paginator": _paginator(parent_config)},
+ child_endpoint_extra={"data_selector": config.data_key, "paginator": _paginator(config)},
+ child_params_extra=child_params,
+ page_size_param=None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=fanout_state,
+ )
+
+ windows = _windows(window_start, window_end, config.max_window_seconds)
+ first = build_window(windows[0], resume.fanout_state if resume else None)
+
+ def walk_windows() -> Iterator[Any]:
+ # Each window's resource hands its pages on unchanged and holds no rows.
+ yield from first
+ for window in windows[1:]:
+ yield from build_window(window, None)
+
+ return Resource(walk_windows, name=config.name, hints=getattr(first, "_hints", {}))
+
+
def pingdom_source(
api_token: str,
endpoint: str,
@@ -85,41 +194,56 @@ def pingdom_source(
) -> SourceResponse:
config = PINGDOM_ENDPOINTS[endpoint]
- rest_config: RESTAPIConfig = {
- "client": {
- "base_url": PINGDOM_BASE_URL,
- # Bearer auth via the framework so the token is redacted from logs and error messages.
- "auth": {"type": "bearer", "token": api_token},
- },
- "resource_defaults": {},
- "resources": [
- {
- "name": endpoint,
- "endpoint": _build_endpoint(config, should_use_incremental_field),
- }
- ],
+ client_config: ClientConfig = {
+ "base_url": PINGDOM_BASE_URL,
+ # Bearer auth via the framework so the token is redacted from logs and error messages.
+ "auth": {"type": "bearer", "token": api_token},
}
- initial_paginator_state: Optional[dict[str, Any]] = None
- if resumable_source_manager.can_resume():
- resume = resumable_source_manager.load_state()
- if resume is not None:
- initial_paginator_state = {"offset": resume.offset}
-
- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields
- # the last page (merge dedupes on primary key) rather than skipping it.
- if state and state.get("offset") is not None:
- resumable_source_manager.save_state(PingdomResumeConfig(offset=int(state["offset"])))
-
- resource = rest_api_resource(
- rest_config,
- team_id,
- job_id,
- db_incremental_field_last_value,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_paginator_state,
- )
+ resource: Iterable[Any]
+ if config.fanout is not None:
+ resource = _fanout_resource(
+ config,
+ config.fanout,
+ client_config,
+ team_id,
+ job_id,
+ resumable_source_manager,
+ db_incremental_field_last_value,
+ should_use_incremental_field,
+ )
+ else:
+ rest_config: RESTAPIConfig = {
+ "client": client_config,
+ "resource_defaults": {},
+ "resources": [
+ {
+ "name": endpoint,
+ "endpoint": _build_endpoint(config, should_use_incremental_field),
+ }
+ ],
+ }
+
+ initial_paginator_state: Optional[dict[str, Any]] = None
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None:
+ initial_paginator_state = {"offset": resume.offset}
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ # Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields
+ # the last page (merge dedupes on primary key) rather than skipping it.
+ if state and state.get("offset") is not None:
+ resumable_source_manager.save_state(PingdomResumeConfig(offset=int(state["offset"])))
+
+ resource = rest_api_resource(
+ rest_config,
+ team_id,
+ job_id,
+ db_incremental_field_last_value,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
return SourceResponse(
name=endpoint,
@@ -130,7 +254,8 @@ def pingdom_source(
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
- sort_mode="asc",
+ # Per-check rows arrive one check at a time, so they are not one ordered series.
+ sort_mode=None if config.fanout else "asc",
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/settings.py
index 1ba7cf358d9d..fec2a7e94646 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/settings.py
@@ -1,10 +1,26 @@
from dataclasses import dataclass, field
-from typing import Optional
+from typing import Any, Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+DAY_SECONDS = 24 * 60 * 60
-@dataclass
+# Every per-check child is fetched from the check listing. A check deleted between the listing and
+# its child request 404s, which must not fail the rest of the fan-out.
+CHECK_FANOUT = DependentEndpointConfig(
+ parent_name="checks",
+ resolve_param="checkid",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "checkid"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+)
+
+
+@dataclass(frozen=True)
class PingdomEndpointConfig:
name: str
path: str
@@ -18,11 +34,25 @@ class PingdomEndpointConfig:
partition_key: Optional[str] = None
# Max page size the endpoint accepts (checks allow 25000, actions 1000).
page_size: int = 1000
+ # Per-check endpoints only take a `from`/`to` time window and are fanned out over /checks.
+ fanout: Optional[DependentEndpointConfig] = None
+ paginated: bool = True
+ # Pagination stops before an offset the endpoint rejects (/results caps it at 43200).
+ maximum_offset: Optional[int] = None
+ params: dict[str, Any] = field(default_factory=dict)
+ # Windowed endpoints: how far back a full refresh or a first incremental sync reads, and the
+ # widest `from`/`to` span one request may ask for. Longer ranges are split into windows.
+ lookback_seconds: Optional[int] = None
+ max_window_seconds: Optional[int] = None
+
+ @property
+ def default_incremental_field(self) -> Optional[str]:
+ return self.incremental_fields[0]["field"] if self.incremental_fields else None
-# Pingdom API 3.1 timestamps are UNIX epoch seconds. Only /actions exposes a
-# server-side `from`/`to` time filter among the endpoints we sync; /checks,
-# /probes and /maintenance are small full-refresh dimension tables.
+# Pingdom API 3.1 timestamps are UNIX epoch seconds. /actions and the per-check endpoints expose a
+# server-side `from`/`to` time filter; /checks, /probes, /maintenance and /tms/check are small
+# full-refresh dimension tables.
PINGDOM_ENDPOINTS: dict[str, PingdomEndpointConfig] = {
"checks": PingdomEndpointConfig(
name="checks",
@@ -60,6 +90,68 @@ PINGDOM_ENDPOINTS: dict[str, PingdomEndpointConfig] = {
},
],
),
+ "transaction_checks": PingdomEndpointConfig(
+ name="transaction_checks",
+ path="/tms/check",
+ data_key="checks",
+ partition_key="created_at",
+ ),
+ "results": PingdomEndpointConfig(
+ name="results",
+ path="/results/{checkid}",
+ data_key="results",
+ fanout=CHECK_FANOUT,
+ primary_keys=["checkid", "probeid", "time"],
+ partition_key="time",
+ page_size=1000,
+ maximum_offset=43200,
+ # A one-minute check writes ~1440 raw results a day, so a 7-day window stays under the offset cap.
+ lookback_seconds=7 * DAY_SECONDS,
+ max_window_seconds=7 * DAY_SECONDS,
+ incremental_fields=[
+ {
+ "label": "time",
+ "type": IncrementalFieldType.DateTime,
+ "field": "time",
+ "field_type": IncrementalFieldType.Integer,
+ },
+ ],
+ ),
+ # The states cover the whole requested window, so the first one is clipped to `from` and the
+ # last one is still open. A `from` taken from the global watermark would clip a different state
+ # per check, so this table is full refresh only.
+ "outages": PingdomEndpointConfig(
+ name="outages",
+ path="/summary.outage/{checkid}",
+ data_key="summary.states",
+ fanout=CHECK_FANOUT,
+ primary_keys=["checkid", "timefrom"],
+ partition_key="timefrom",
+ paginated=False,
+ params={"order": "asc"},
+ lookback_seconds=365 * DAY_SECONDS,
+ ),
+ "performance": PingdomEndpointConfig(
+ name="performance",
+ path="/summary.performance/{checkid}",
+ data_key="summary.hours",
+ fanout=CHECK_FANOUT,
+ primary_keys=["checkid", "starttime"],
+ partition_key="starttime",
+ paginated=False,
+ params={"resolution": "hour", "includeuptime": "true", "order": "asc"},
+ # Pingdom limits hourly report windows to about one week per request.
+ lookback_seconds=7 * DAY_SECONDS,
+ max_window_seconds=7 * DAY_SECONDS,
+ incremental_fields=[
+ {
+ "label": "starttime",
+ "type": IncrementalFieldType.DateTime,
+ "field": "starttime",
+ "field_type": IncrementalFieldType.Integer,
+ },
+ ],
+ ),
}
ENDPOINTS = tuple(PINGDOM_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/tests/test_pingdom.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/tests/test_pingdom.py
index 56e0ffc2d2c7..b6068ca90250 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/tests/test_pingdom.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/tests/test_pingdom.py
@@ -2,12 +2,14 @@ from datetime import UTC, date, datetime
from typing import Any
import pytest
+import time_machine
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.extract import validate_incremental_sync
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.pingdom import (
@@ -18,7 +20,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.pingdom.pi
_to_epoch,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.pingdom.settings import PINGDOM_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.pingdom.settings import (
+ DAY_SECONDS,
+ PINGDOM_ENDPOINTS,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.pingdom.source import PingdomSource
@@ -88,6 +93,131 @@ class TestPagination:
assert result.params("from") == ["1700000000"]
+NOW = 1_800_000_000
+
+
+def _checks(*ids: int) -> ScriptedResponse:
+ return _response("checks", [{"id": check_id, "name": f"check {check_id}"} for check_id in ids])
+
+
+class TestPerCheckFanout:
+ @time_machine.travel(NOW, tick=False)
+ def test_results_walk_every_check_and_page_in_one_window(self) -> None:
+ page_size = PINGDOM_ENDPOINTS["results"].page_size
+ full_page = [{"probeid": 1, "time": NOW - i} for i in range(page_size)]
+ result = _driver().run(
+ "results",
+ route(
+ {
+ "/checks": [_checks(1, 2)],
+ "/results/1": [_response("results", full_page), _response("results", [{"probeid": 2, "time": 5}])],
+ "/results/2": [_response("results", [{"probeid": 3, "time": 7}])],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/api/3.1/checks", "/api/3.1/results/1", "/api/3.1/results/1", "/api/3.1/results/2"]
+ assert result.params("offset") == ["0", "0", str(page_size), "0"]
+ assert result.params("to") == [None, str(NOW), str(NOW), str(NOW)]
+ assert result.params("from") == [None] + [str(NOW - 7 * DAY_SECONDS)] * 3
+ assert len(result.rows) == page_size + 2
+ assert [row["checkid"] for row in result.rows[-2:]] == [1, 2]
+ assert all("_checks_id" not in row for row in result.rows)
+ assert result.saved_states and all(state.window_end == NOW for state in result.saved_states)
+
+ @pytest.mark.parametrize(
+ "endpoint, watermark, expected_windows",
+ [
+ ("results", NOW - DAY_SECONDS, [(NOW - DAY_SECONDS, NOW)]),
+ ("results", None, [(NOW - 7 * DAY_SECONDS, NOW)]),
+ (
+ "results",
+ NOW - 10 * DAY_SECONDS,
+ [(NOW - 10 * DAY_SECONDS, NOW - 3 * DAY_SECONDS), (NOW - 3 * DAY_SECONDS, NOW)],
+ ),
+ ("performance", NOW - 3600, [(NOW - 3600, NOW)]),
+ (
+ "performance",
+ NOW - 15 * DAY_SECONDS,
+ [
+ (NOW - 15 * DAY_SECONDS, NOW - 8 * DAY_SECONDS),
+ (NOW - 8 * DAY_SECONDS, NOW - DAY_SECONDS),
+ (NOW - DAY_SECONDS, NOW),
+ ],
+ ),
+ ],
+ )
+ @time_machine.travel(NOW, tick=False)
+ def test_incremental_sync_walks_bounded_windows_from_the_watermark(
+ self, endpoint: str, watermark: int | None, expected_windows: list[tuple[int, int]]
+ ) -> None:
+ path = PINGDOM_ENDPOINTS[endpoint].path.replace("{checkid}", "1")
+ result = _driver().run(
+ endpoint,
+ route(
+ {
+ "/checks": [_checks(1)] * len(expected_windows),
+ path: [_response(endpoint, [])] * len(expected_windows),
+ }
+ ),
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=watermark,
+ )
+
+ assert result.raised is None
+ child_requests = [request for request in result.requests if request.path.endswith(path)]
+ assert [(r.param("from"), r.param("to")) for r in child_requests] == [
+ (str(start), str(end)) for start, end in expected_windows
+ ]
+
+ def test_results_pagination_stops_within_the_offset_cap(self) -> None:
+ full_page = _response("results", [{"probeid": 1, "time": i} for i in range(1000)])
+ result = _driver().run("results", route({"/checks": [_checks(1)], "/results/1": [full_page] * 50}))
+
+ assert result.raised is None
+ assert max(int(offset) for offset in result.params("offset") if offset is not None) <= 43200
+
+ def test_performance_and_outages_request_one_page_per_check(self) -> None:
+ result = _driver().run(
+ "performance",
+ route(
+ {
+ "/checks": [_checks(1)],
+ "/summary.performance/1": [
+ _response("performance", [{"starttime": 100, "avgresponse": 200, "uptime": 3600}])
+ ],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ request = result.requests[-1]
+ assert (request.param("resolution"), request.param("includeuptime"), request.param("offset")) == (
+ "hour",
+ "true",
+ None,
+ )
+ assert result.rows == [{"starttime": 100, "avgresponse": 200, "uptime": 3600, "checkid": 1}]
+
+ def test_resume_skips_finished_checks_and_reuses_the_window(self) -> None:
+ window_end = NOW - 600
+ result = _driver().run(
+ "outages",
+ route({"/checks": [_checks(1, 2)], "/summary.outage/2": [_response("outages", [])]}),
+ resume_state=PingdomResumeConfig(
+ fanout_state={"completed": ["/summary.outage/1"], "current": None, "child_state": None},
+ window_start=window_end - 365 * DAY_SECONDS,
+ window_end=window_end,
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/api/3.1/checks", "/api/3.1/summary.outage/2"]
+ assert result.params("to") == [None, str(window_end)]
+ assert result.params("from") == [None, str(window_end - 365 * DAY_SECONDS)]
+
+
class TestValidateCredentials:
@pytest.mark.parametrize(
"status_code, expected",
@@ -119,5 +249,5 @@ class TestPingdomSourceResponse:
@pytest.mark.parametrize("config", list(PINGDOM_ENDPOINTS.values()))
def test_partition_keys_are_stable_fields(self, config):
if config.partition_key:
- # Alert timestamps are immutable event times.
- assert config.partition_key == "time"
+ # Event, interval-start and creation timestamps never change once written.
+ assert config.partition_key in {"time", "timefrom", "starttime", "created_at"}