Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): request tenjin history in 30-day windows (#128955)

Chris Shine7dc32046d112
diff --git a/products/warehouse_sources/sources/tenjin/source.py b/products/warehouse_sources/sources/tenjin/source.py
index e14de0d9e088..21ecb2d1fa90 100644
--- a/products/warehouse_sources/sources/tenjin/source.py
+++ b/products/warehouse_sources/sources/tenjin/source.py
@@ -23,6 +23,7 @@ from products.warehouse_sources.sources.sdk import (
from products.warehouse_sources.sources.tenjin._config import TenjinSourceConfig
from products.warehouse_sources.sources.tenjin.settings import DESCRIPTIONS, ENDPOINTS, INCREMENTAL_FIELDS
from products.warehouse_sources.sources.tenjin.tenjin import (
+ REJECTED_REQUEST_MESSAGE,
TenjinCredentialsError,
TenjinResumeConfig,
TenjinRetryableError,
@@ -61,7 +62,7 @@ class TenjinSource(ResumableSource[TenjinSourceConfig, TenjinResumeConfig]):
"403 Client Error: Forbidden for url: https://api.tenjin.com": "Tenjin denied access. Check that your access token is valid and has the Reporting Metrics API permission.",
# A 400 means the report request itself is invalid (unknown metric or group_by). The
# request shape is fixed per table, so retrying it identically can never succeed.
- "400 Client Error: Bad Request for url: https://api.tenjin.com": "Tenjin rejected the report request. Check that your account has access to the requested report and metrics.",
+ REJECTED_REQUEST_MESSAGE: None,
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
diff --git a/products/warehouse_sources/sources/tenjin/tenjin.py b/products/warehouse_sources/sources/tenjin/tenjin.py
index f90107724382..0760f6ae4171 100644
--- a/products/warehouse_sources/sources/tenjin/tenjin.py
+++ b/products/warehouse_sources/sources/tenjin/tenjin.py
@@ -1,24 +1,34 @@
+from collections.abc import Iterator
from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
from urllib.parse import urlencode
+import requests
+
from products.warehouse_sources.sources.sdk import (
JSONResponsePaginator,
+ Resource,
RESTAPIConfig,
ResumableSourceManager,
SourceResponse,
+ SyncWindow,
frozen,
make_tracked_session,
+ redact_literal_values,
rest_api_resource,
)
-from products.warehouse_sources.sources.tenjin.settings import DEFAULT_LOOKBACK_DAYS, TENJIN_REPORTS
+from products.warehouse_sources.sources.tenjin.settings import DEFAULT_LOOKBACK_DAYS, TENJIN_REPORTS, TenjinReportConfig
TENJIN_BASE_URL = "https://api.tenjin.com/v2"
# First sync backfill depth. Attribution platforms rarely keep report data useful beyond this.
MAX_HISTORY_DAYS = 730
+# Tenjin's own default range when start_date is omitted. A backfill requests one window at a time.
+WINDOW_DAYS = 30
# The documented maximum, to keep the request count low on long backfills.
PER_PAGE = 1000
VALIDATE_TIMEOUT_SECONDS = 30
+REJECTED_REQUEST_MESSAGE = "Tenjin rejected the report request"
+ERROR_DETAIL_LIMIT = 300
class TenjinRetryableError(Exception):
@@ -38,6 +48,8 @@ class TenjinResumeConfig:
# Absolute next-page URL from the response's `links.next`. It carries the full original query
# (dates, group_by, metrics), so resuming from it continues the exact same report pull.
next_url: Optional[str] = None
+ # Last day the resume point covers: the window `next_url` pages through, else the last finished one.
+ window_end: Optional[str] = None
def _today() -> date:
@@ -74,25 +86,33 @@ def resolve_start_date(
return earliest
-def tenjin_source(
- api_key: str,
- endpoint: str,
- team_id: int,
- job_id: str,
- resumable_source_manager: ResumableSourceManager[TenjinResumeConfig],
- db_incremental_field_last_value: Optional[Any],
- should_use_incremental_field: bool = False,
-) -> SourceResponse:
- report = TENJIN_REPORTS.get(endpoint)
- if report is None:
- raise ValueError(f"Unknown Tenjin report: {endpoint}")
+def date_windows(start: date, end: date) -> Iterator[SyncWindow[date]]:
+ while start <= end:
+ window_end = min(start + timedelta(days=WINDOW_DAYS - 1), end)
+ yield SyncWindow(start=start, end=window_end)
+ start = window_end + timedelta(days=1)
+
+
+def rejected_request_message(response: requests.Response, api_key: str) -> str:
+ """The error for a 400, with Tenjin's reason from its documented `{"error": "..."}` body and the token removed."""
+ try:
+ body = response.json()
+ except ValueError:
+ body = None
+ detail = body.get("error") if isinstance(body, dict) else None
+ if not isinstance(detail, str) or not detail.strip():
+ return f"{REJECTED_REQUEST_MESSAGE}. Check that your account has access to the requested report and metrics."
+ return f"{REJECTED_REQUEST_MESSAGE}: {redact_literal_values(detail, (api_key,)).strip()[:ERROR_DETAIL_LIMIT]}"
- today = _today()
- start = resolve_start_date(
- should_use_incremental_field, db_incremental_field_last_value, today, lookback_days=report.lookback_days
- )
- config: RESTAPIConfig = {
+def _report_config(
+ api_key: str,
+ endpoint: str,
+ report: TenjinReportConfig,
+ dates: dict[str, str],
+ should_use_incremental_field: bool,
+) -> RESTAPIConfig:
+ return {
"client": {
"base_url": TENJIN_BASE_URL,
"auth": {
@@ -105,6 +125,8 @@ def tenjin_source(
# bearer token off-host.
"allowed_hosts": [],
"allow_redirects": False,
+ # The client ignores Tenjin's X-Retry-After, so the backoff must outlast its one-minute limit.
+ "max_retries": 7,
},
"resource_defaults": {
"write_disposition": {
@@ -124,8 +146,7 @@ def tenjin_source(
"data_selector": "data[*].attributes",
"path": report.path,
"params": {
- "start_date": start.isoformat(),
- "end_date": today.isoformat(),
+ **dates,
# Always daily: rows carry no `date` at totals-daily granularity, which
# would break both the primary key and the incremental cursor.
"granularity": "daily",
@@ -139,30 +160,74 @@ def tenjin_source(
],
}
- initial_paginator_state: Optional[dict[str, Any]] = None
- if resumable_source_manager.can_resume():
- resume = resumable_source_manager.load_state()
- if resume is not None and resume.next_url:
- initial_paginator_state = {"next_url": resume.next_url}
- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+def tenjin_source(
+ api_key: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[TenjinResumeConfig],
+ db_incremental_field_last_value: Optional[Any],
+ should_use_incremental_field: bool = False,
+) -> SourceResponse:
+ report = TENJIN_REPORTS.get(endpoint)
+ if report is None:
+ raise ValueError(f"Unknown Tenjin report: {endpoint}")
+
+ today = _today()
+ start = resolve_start_date(
+ should_use_incremental_field, db_incremental_field_last_value, today, lookback_days=report.lookback_days
+ )
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume is not None and resume.window_end:
+ start = date.fromisoformat(resume.window_end) + timedelta(days=1)
+
+ def save_checkpoint(state: Optional[dict[str, Any]], window_end: date) -> None:
# Only persist when there's a next page to resume to; the Redis TTL handles cleanup on
# completion.
if state and state.get("next_url"):
- resumable_source_manager.save_state(TenjinResumeConfig(next_url=str(state["next_url"])))
+ resumable_source_manager.save_state(
+ TenjinResumeConfig(next_url=str(state["next_url"]), window_end=window_end.isoformat())
+ )
- resource = rest_api_resource(
- config,
+ def window_pages(window_end: date, dates: dict[str, str], next_url: Optional[str] = None) -> Iterator[Any]:
+ yield from rest_api_resource(
+ _report_config(api_key, endpoint, report, dates, should_use_incremental_field),
+ team_id,
+ job_id,
+ db_incremental_field_last_value,
+ resume_hook=lambda state: save_checkpoint(state, window_end),
+ initial_paginator_state={"next_url": next_url} if next_url else None,
+ )
+ if window_end < today:
+ resumable_source_manager.save_state(TenjinResumeConfig(window_end=window_end.isoformat()))
+ resumable_source_manager.safe_point()
+
+ def pages() -> Iterator[Any]:
+ try:
+ if resume is not None and resume.window_end and resume.next_url:
+ yield from window_pages(date.fromisoformat(resume.window_end), {}, resume.next_url)
+ for window in date_windows(start, today):
+ yield from window_pages(
+ window.end, {"start_date": window.start.isoformat(), "end_date": window.end.isoformat()}
+ )
+ except requests.HTTPError as e:
+ if e.response is None or e.response.status_code != 400:
+ raise
+ raise requests.HTTPError(rejected_request_message(e.response, api_key), response=e.response) from e
+
+ # As a `Resource` the walk keeps framework checkpoints, so a hand-off never reads a page twice.
+ hints = rest_api_resource(
+ _report_config(api_key, endpoint, report, {}, should_use_incremental_field),
team_id,
job_id,
db_incremental_field_last_value,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_paginator_state,
- )
+ )._hints
+ walk = Resource(pages, name=endpoint, hints=hints)
return SourceResponse(
name=endpoint,
- items=lambda: resource,
+ items=lambda: walk,
primary_keys=list(report.primary_keys),
partition_count=1,
partition_size=1,
diff --git a/products/warehouse_sources/sources/tenjin/tests/test_tenjin.py b/products/warehouse_sources/sources/tenjin/tests/test_tenjin.py
index 221f465d155c..9c7e1b8c9ed7 100644
--- a/products/warehouse_sources/sources/tenjin/tests/test_tenjin.py
+++ b/products/warehouse_sources/sources/tenjin/tests/test_tenjin.py
@@ -11,6 +11,7 @@ from parameterized import parameterized
from products.warehouse_sources.sources.sdk.internals import validate_incremental_sync
from products.warehouse_sources.sources.sdk.testing import (
DriveResult,
+ RecordedRequest,
Script,
ScriptedResponse,
SourceDriver,
@@ -24,6 +25,7 @@ from products.warehouse_sources.sources.tenjin.source import TenjinSource
from products.warehouse_sources.sources.tenjin.tenjin import (
MAX_HISTORY_DAYS,
TENJIN_BASE_URL,
+ WINDOW_DAYS,
TenjinCredentialsError,
TenjinResumeConfig,
TenjinRetryableError,
@@ -65,29 +67,44 @@ class TestResolveStartDate:
def test_unparseable_watermark_falls_back_to_full_history(self) -> None:
assert resolve_start_date(True, "not-a-date", TODAY) == TODAY - timedelta(days=MAX_HISTORY_DAYS)
+ def test_lookback_is_clamped_to_full_history(self) -> None:
+ assert resolve_start_date(True, "2024-07-01", TODAY, lookback_days=37) == TODAY - timedelta(
+ days=MAX_HISTORY_DAYS
+ )
+
class TestRequestShaping:
@parameterized.expand(
[
- ("spend_report", "app_report", "2026-06-28", "2026-06-21"),
- ("ad_revenue_report", "ad_placement_report", "2026-06-28", "2026-06-21"),
+ ("spend_report", "app_report", [("2026-06-21", "2026-06-30")]),
+ ("ad_revenue_report", "ad_placement_report", [("2026-06-21", "2026-06-30")]),
# Day-30 cohort metrics keep changing for 30 days after the install date.
- ("cohort_report", "campaign_cohort_report", "2026-06-28", "2026-05-22"),
- ("cohort_window_clamped_to_history", "campaign_cohort_report", "2024-07-01", "2024-06-30"),
+ ("cohort_report", "campaign_cohort_report", [("2026-05-22", "2026-06-20"), ("2026-06-21", "2026-06-30")]),
]
)
def test_incremental_requests_only_the_watermark_range(
- self, _name: str, endpoint: str, watermark: str, expected_start: str
+ self, _name: str, endpoint: str, expected_windows: list[tuple[str, str]]
) -> None:
result = _drive(
- [_report_page([{"date": watermark}])],
+ always(_report_page([{"date": "2026-06-28"}])),
endpoint=endpoint,
should_use_incremental_field=True,
- db_incremental_field_last_value=watermark,
+ db_incremental_field_last_value="2026-06-28",
)
- assert result.requests[0].param("start_date") == expected_start
- assert result.requests[0].param("end_date") == "2026-06-30"
+ assert list(zip(result.params("start_date"), result.params("end_date"))) == expected_windows
+
+ def test_first_sync_walks_the_history_in_contiguous_windows(self) -> None:
+ result = _drive(always(_report_page([])))
+
+ assert result.raised is None
+ starts = [date.fromisoformat(str(value)) for value in result.params("start_date")]
+ ends = [date.fromisoformat(str(value)) for value in result.params("end_date")]
+ assert starts[0] == TODAY - timedelta(days=MAX_HISTORY_DAYS)
+ assert ends[-1] == TODAY
+ assert all((end - start).days < WINDOW_DAYS for start, end in zip(starts, ends))
+ assert all(start == end + timedelta(days=1) for end, start in zip(ends, starts[1:]))
+ assert result.final_state == TenjinResumeConfig(window_end=ends[-2].isoformat())
@parameterized.expand([(name,) for name in TENJIN_REPORTS])
def test_every_report_is_requested_at_daily_granularity(self, endpoint: str) -> None:
@@ -105,9 +122,9 @@ class TestToday:
driver = SourceDriver(TenjinSource(), TenjinSourceConfig(api_key="test-key"))
with time_machine.travel("2026-07-09T23:30:00Z", tick=False):
- result = driver.run("app_report", [_report_page([])])
+ result = driver.run("app_report", always(_report_page([])))
- assert result.requests[0].param("end_date") == "2026-07-09"
+ assert result.requests[-1].param("end_date") == "2026-07-09"
class TestPaginationAndResume:
@@ -117,23 +134,70 @@ class TestPaginationAndResume:
[
_report_page([{"date": "2026-06-29", "app_id": "a1"}], next_url=next_url),
_report_page([{"date": "2026-06-30", "app_id": "a1"}]),
- ]
+ ],
+ should_use_incremental_field=True,
+ db_incremental_field_last_value="2026-06-28",
)
assert len(result.items) == 2
assert len(result.requests) == 2
assert result.requests[1].param("page") == "2"
- def test_resume_starts_at_the_saved_next_url(self) -> None:
+ def test_failed_page_leaves_a_resume_point_at_that_page(self) -> None:
+ next_url = f"{TENJIN_BASE_URL}/reports/spend?group_by=app&page=2&per_page=1000"
+ result = _drive(
+ [
+ _report_page([{"date": "2026-06-29", "app_id": "a1"}], next_url=next_url),
+ ScriptedResponse(status=400, json={"error": "Invalid request"}),
+ ],
+ should_use_incremental_field=True,
+ db_incremental_field_last_value="2026-06-28",
+ )
+
+ assert result.final_state == TenjinResumeConfig(next_url=next_url, window_end="2026-06-30")
+
+ def test_resume_finishes_the_saved_window_then_requests_only_later_windows(self) -> None:
saved_url = f"{TENJIN_BASE_URL}/reports/spend?group_by=app&page=7&per_page=1000"
result = _drive(
- [_report_page([{"date": "2026-06-30", "app_id": "a1"}])],
- resume_state=TenjinResumeConfig(next_url=saved_url),
+ always(_report_page([{"date": "2026-06-10", "app_id": "a1"}])),
+ resume_state=TenjinResumeConfig(next_url=saved_url, window_end="2026-06-10"),
)
- assert len(result.requests) == 1
- assert result.requests[0].param("page") == "7"
+ assert result.params("page") == ["7", None]
+ assert result.params("start_date") == [None, "2026-06-11"]
+ assert result.params("end_date") == [None, "2026-06-30"]
+
+ def test_failed_window_leaves_a_resume_point_after_the_finished_windows(self) -> None:
+ def answer(request: RecordedRequest) -> ScriptedResponse:
+ if request.param("start_date") == "2024-07-30":
+ return ScriptedResponse(status=400, json={"error": "Invalid request"})
+ return _report_page([{"date": request.param("start_date"), "app_id": "a1"}])
+
+ result = _drive(answer)
+
+ assert result.final_state == TenjinResumeConfig(window_end="2024-07-29")
+
+ @parameterized.expand(
+ [
+ (
+ "reason",
+ {"error": "start_date test-key is invalid"},
+ "Tenjin rejected the report request: start_date REDACTED is invalid",
+ ),
+ (
+ "no_reason",
+ {"message": "nope"},
+ "Tenjin rejected the report request. Check that your account has access to the requested report and metrics.",
+ ),
+ ]
+ )
+ def test_rejected_request_error_carries_tenjins_reason(
+ self, _name: str, body: dict[str, Any], expected: str
+ ) -> None:
+ result = _drive([ScriptedResponse(status=400, json=body)])
+
+ assert str(result.raised) == expected
class TestTenjinSourceResponse: