:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): request tenjin history in 30-day windows (#128955)
Chris Shine7dc32046d112
diff --git a/products/warehouse_sources/sources/tenjin/source.py b/products/warehouse_sources/sources/tenjin/source.pyindex e14de0d9e088..21ecb2d1fa90 100644--- a/products/warehouse_sources/sources/tenjin/source.py+++ b/products/warehouse_sources/sources/tenjin/source.py@@ -23,6 +23,7 @@ from products.warehouse_sources.sources.sdk import (from products.warehouse_sources.sources.tenjin._config import TenjinSourceConfigfrom products.warehouse_sources.sources.tenjin.settings import DESCRIPTIONS, ENDPOINTS, INCREMENTAL_FIELDSfrom products.warehouse_sources.sources.tenjin.tenjin import (+ REJECTED_REQUEST_MESSAGE,TenjinCredentialsError,TenjinResumeConfig,TenjinRetryableError,@@ -61,7 +62,7 @@ class TenjinSource(ResumableSource[TenjinSourceConfig, TenjinResumeConfig]):"403 Client Error: Forbidden for url: https://api.tenjin.com": "Tenjin denied access. Check that your access token is valid and has the Reporting Metrics API permission.",# A 400 means the report request itself is invalid (unknown metric or group_by). The# request shape is fixed per table, so retrying it identically can never succeed.- "400 Client Error: Bad Request for url: https://api.tenjin.com": "Tenjin rejected the report request. Check that your account has access to the requested report and metrics.",+ REJECTED_REQUEST_MESSAGE: None,}def get_canonical_descriptions(self) -> CanonicalDescriptions:diff --git a/products/warehouse_sources/sources/tenjin/tenjin.py b/products/warehouse_sources/sources/tenjin/tenjin.pyindex f90107724382..0760f6ae4171 100644--- a/products/warehouse_sources/sources/tenjin/tenjin.py+++ b/products/warehouse_sources/sources/tenjin/tenjin.py@@ -1,24 +1,34 @@+from collections.abc import Iteratorfrom datetime import UTC, date, datetime, timedeltafrom typing import Any, Optionalfrom urllib.parse import urlencode+import requests+from products.warehouse_sources.sources.sdk import (JSONResponsePaginator,+ Resource,RESTAPIConfig,ResumableSourceManager,SourceResponse,+ SyncWindow,frozen,make_tracked_session,+ redact_literal_values,rest_api_resource,)-from products.warehouse_sources.sources.tenjin.settings import DEFAULT_LOOKBACK_DAYS, TENJIN_REPORTS+from products.warehouse_sources.sources.tenjin.settings import DEFAULT_LOOKBACK_DAYS, TENJIN_REPORTS, TenjinReportConfigTENJIN_BASE_URL = "https://api.tenjin.com/v2"# First sync backfill depth. Attribution platforms rarely keep report data useful beyond this.MAX_HISTORY_DAYS = 730+# Tenjin's own default range when start_date is omitted. A backfill requests one window at a time.+WINDOW_DAYS = 30# The documented maximum, to keep the request count low on long backfills.PER_PAGE = 1000VALIDATE_TIMEOUT_SECONDS = 30+REJECTED_REQUEST_MESSAGE = "Tenjin rejected the report request"+ERROR_DETAIL_LIMIT = 300class TenjinRetryableError(Exception):@@ -38,6 +48,8 @@ class TenjinResumeConfig:# Absolute next-page URL from the response's `links.next`. It carries the full original query# (dates, group_by, metrics), so resuming from it continues the exact same report pull.next_url: Optional[str] = None+ # Last day the resume point covers: the window `next_url` pages through, else the last finished one.+ window_end: Optional[str] = Nonedef _today() -> date:@@ -74,25 +86,33 @@ def resolve_start_date(return earliest-def tenjin_source(- api_key: str,- endpoint: str,- team_id: int,- job_id: str,- resumable_source_manager: ResumableSourceManager[TenjinResumeConfig],- db_incremental_field_last_value: Optional[Any],- should_use_incremental_field: bool = False,-) -> SourceResponse:- report = TENJIN_REPORTS.get(endpoint)- if report is None:- raise ValueError(f"Unknown Tenjin report: {endpoint}")+def date_windows(start: date, end: date) -> Iterator[SyncWindow[date]]:+ while start <= end:+ window_end = min(start + timedelta(days=WINDOW_DAYS - 1), end)+ yield SyncWindow(start=start, end=window_end)+ start = window_end + timedelta(days=1)+++def rejected_request_message(response: requests.Response, api_key: str) -> str:+ """The error for a 400, with Tenjin's reason from its documented `{"error": "..."}` body and the token removed."""+ try:+ body = response.json()+ except ValueError:+ body = None+ detail = body.get("error") if isinstance(body, dict) else None+ if not isinstance(detail, str) or not detail.strip():+ return f"{REJECTED_REQUEST_MESSAGE}. Check that your account has access to the requested report and metrics."+ return f"{REJECTED_REQUEST_MESSAGE}: {redact_literal_values(detail, (api_key,)).strip()[:ERROR_DETAIL_LIMIT]}"- today = _today()- start = resolve_start_date(- should_use_incremental_field, db_incremental_field_last_value, today, lookback_days=report.lookback_days- )- config: RESTAPIConfig = {+def _report_config(+ api_key: str,+ endpoint: str,+ report: TenjinReportConfig,+ dates: dict[str, str],+ should_use_incremental_field: bool,+) -> RESTAPIConfig:+ return {"client": {"base_url": TENJIN_BASE_URL,"auth": {@@ -105,6 +125,8 @@ def tenjin_source(# bearer token off-host."allowed_hosts": [],"allow_redirects": False,+ # The client ignores Tenjin's X-Retry-After, so the backoff must outlast its one-minute limit.+ "max_retries": 7,},"resource_defaults": {"write_disposition": {@@ -124,8 +146,7 @@ def tenjin_source("data_selector": "data[*].attributes","path": report.path,"params": {- "start_date": start.isoformat(),- "end_date": today.isoformat(),+ **dates,# Always daily: rows carry no `date` at totals-daily granularity, which# would break both the primary key and the incremental cursor."granularity": "daily",@@ -139,30 +160,74 @@ def tenjin_source(],}- initial_paginator_state: Optional[dict[str, Any]] = None- if resumable_source_manager.can_resume():- resume = resumable_source_manager.load_state()- if resume is not None and resume.next_url:- initial_paginator_state = {"next_url": resume.next_url}- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+def tenjin_source(+ api_key: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[TenjinResumeConfig],+ db_incremental_field_last_value: Optional[Any],+ should_use_incremental_field: bool = False,+) -> SourceResponse:+ report = TENJIN_REPORTS.get(endpoint)+ if report is None:+ raise ValueError(f"Unknown Tenjin report: {endpoint}")++ today = _today()+ start = resolve_start_date(+ should_use_incremental_field, db_incremental_field_last_value, today, lookback_days=report.lookback_days+ )+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ if resume is not None and resume.window_end:+ start = date.fromisoformat(resume.window_end) + timedelta(days=1)++ def save_checkpoint(state: Optional[dict[str, Any]], window_end: date) -> None:# Only persist when there's a next page to resume to; the Redis TTL handles cleanup on# completion.if state and state.get("next_url"):- resumable_source_manager.save_state(TenjinResumeConfig(next_url=str(state["next_url"])))+ resumable_source_manager.save_state(+ TenjinResumeConfig(next_url=str(state["next_url"]), window_end=window_end.isoformat())+ )- resource = rest_api_resource(- config,+ def window_pages(window_end: date, dates: dict[str, str], next_url: Optional[str] = None) -> Iterator[Any]:+ yield from rest_api_resource(+ _report_config(api_key, endpoint, report, dates, should_use_incremental_field),+ team_id,+ job_id,+ db_incremental_field_last_value,+ resume_hook=lambda state: save_checkpoint(state, window_end),+ initial_paginator_state={"next_url": next_url} if next_url else None,+ )+ if window_end < today:+ resumable_source_manager.save_state(TenjinResumeConfig(window_end=window_end.isoformat()))+ resumable_source_manager.safe_point()++ def pages() -> Iterator[Any]:+ try:+ if resume is not None and resume.window_end and resume.next_url:+ yield from window_pages(date.fromisoformat(resume.window_end), {}, resume.next_url)+ for window in date_windows(start, today):+ yield from window_pages(+ window.end, {"start_date": window.start.isoformat(), "end_date": window.end.isoformat()}+ )+ except requests.HTTPError as e:+ if e.response is None or e.response.status_code != 400:+ raise+ raise requests.HTTPError(rejected_request_message(e.response, api_key), response=e.response) from e++ # As a `Resource` the walk keeps framework checkpoints, so a hand-off never reads a page twice.+ hints = rest_api_resource(+ _report_config(api_key, endpoint, report, {}, should_use_incremental_field),team_id,job_id,db_incremental_field_last_value,- resume_hook=save_checkpoint,- initial_paginator_state=initial_paginator_state,- )+ )._hints+ walk = Resource(pages, name=endpoint, hints=hints)return SourceResponse(name=endpoint,- items=lambda: resource,+ items=lambda: walk,primary_keys=list(report.primary_keys),partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/sources/tenjin/tests/test_tenjin.py b/products/warehouse_sources/sources/tenjin/tests/test_tenjin.pyindex 221f465d155c..9c7e1b8c9ed7 100644--- a/products/warehouse_sources/sources/tenjin/tests/test_tenjin.py+++ b/products/warehouse_sources/sources/tenjin/tests/test_tenjin.py@@ -11,6 +11,7 @@ from parameterized import parameterizedfrom products.warehouse_sources.sources.sdk.internals import validate_incremental_syncfrom products.warehouse_sources.sources.sdk.testing import (DriveResult,+ RecordedRequest,Script,ScriptedResponse,SourceDriver,@@ -24,6 +25,7 @@ from products.warehouse_sources.sources.tenjin.source import TenjinSourcefrom products.warehouse_sources.sources.tenjin.tenjin import (MAX_HISTORY_DAYS,TENJIN_BASE_URL,+ WINDOW_DAYS,TenjinCredentialsError,TenjinResumeConfig,TenjinRetryableError,@@ -65,29 +67,44 @@ class TestResolveStartDate:def test_unparseable_watermark_falls_back_to_full_history(self) -> None:assert resolve_start_date(True, "not-a-date", TODAY) == TODAY - timedelta(days=MAX_HISTORY_DAYS)+ def test_lookback_is_clamped_to_full_history(self) -> None:+ assert resolve_start_date(True, "2024-07-01", TODAY, lookback_days=37) == TODAY - timedelta(+ days=MAX_HISTORY_DAYS+ )+class TestRequestShaping:@parameterized.expand([- ("spend_report", "app_report", "2026-06-28", "2026-06-21"),- ("ad_revenue_report", "ad_placement_report", "2026-06-28", "2026-06-21"),+ ("spend_report", "app_report", [("2026-06-21", "2026-06-30")]),+ ("ad_revenue_report", "ad_placement_report", [("2026-06-21", "2026-06-30")]),# Day-30 cohort metrics keep changing for 30 days after the install date.- ("cohort_report", "campaign_cohort_report", "2026-06-28", "2026-05-22"),- ("cohort_window_clamped_to_history", "campaign_cohort_report", "2024-07-01", "2024-06-30"),+ ("cohort_report", "campaign_cohort_report", [("2026-05-22", "2026-06-20"), ("2026-06-21", "2026-06-30")]),])def test_incremental_requests_only_the_watermark_range(- self, _name: str, endpoint: str, watermark: str, expected_start: str+ self, _name: str, endpoint: str, expected_windows: list[tuple[str, str]]) -> None:result = _drive(- [_report_page([{"date": watermark}])],+ always(_report_page([{"date": "2026-06-28"}])),endpoint=endpoint,should_use_incremental_field=True,- db_incremental_field_last_value=watermark,+ db_incremental_field_last_value="2026-06-28",)- assert result.requests[0].param("start_date") == expected_start- assert result.requests[0].param("end_date") == "2026-06-30"+ assert list(zip(result.params("start_date"), result.params("end_date"))) == expected_windows++ def test_first_sync_walks_the_history_in_contiguous_windows(self) -> None:+ result = _drive(always(_report_page([])))++ assert result.raised is None+ starts = [date.fromisoformat(str(value)) for value in result.params("start_date")]+ ends = [date.fromisoformat(str(value)) for value in result.params("end_date")]+ assert starts[0] == TODAY - timedelta(days=MAX_HISTORY_DAYS)+ assert ends[-1] == TODAY+ assert all((end - start).days < WINDOW_DAYS for start, end in zip(starts, ends))+ assert all(start == end + timedelta(days=1) for end, start in zip(ends, starts[1:]))+ assert result.final_state == TenjinResumeConfig(window_end=ends[-2].isoformat())@parameterized.expand([(name,) for name in TENJIN_REPORTS])def test_every_report_is_requested_at_daily_granularity(self, endpoint: str) -> None:@@ -105,9 +122,9 @@ class TestToday:driver = SourceDriver(TenjinSource(), TenjinSourceConfig(api_key="test-key"))with time_machine.travel("2026-07-09T23:30:00Z", tick=False):- result = driver.run("app_report", [_report_page([])])+ result = driver.run("app_report", always(_report_page([])))- assert result.requests[0].param("end_date") == "2026-07-09"+ assert result.requests[-1].param("end_date") == "2026-07-09"class TestPaginationAndResume:@@ -117,23 +134,70 @@ class TestPaginationAndResume:[_report_page([{"date": "2026-06-29", "app_id": "a1"}], next_url=next_url),_report_page([{"date": "2026-06-30", "app_id": "a1"}]),- ]+ ],+ should_use_incremental_field=True,+ db_incremental_field_last_value="2026-06-28",)assert len(result.items) == 2assert len(result.requests) == 2assert result.requests[1].param("page") == "2"- def test_resume_starts_at_the_saved_next_url(self) -> None:+ def test_failed_page_leaves_a_resume_point_at_that_page(self) -> None:+ next_url = f"{TENJIN_BASE_URL}/reports/spend?group_by=app&page=2&per_page=1000"+ result = _drive(+ [+ _report_page([{"date": "2026-06-29", "app_id": "a1"}], next_url=next_url),+ ScriptedResponse(status=400, json={"error": "Invalid request"}),+ ],+ should_use_incremental_field=True,+ db_incremental_field_last_value="2026-06-28",+ )++ assert result.final_state == TenjinResumeConfig(next_url=next_url, window_end="2026-06-30")++ def test_resume_finishes_the_saved_window_then_requests_only_later_windows(self) -> None:saved_url = f"{TENJIN_BASE_URL}/reports/spend?group_by=app&page=7&per_page=1000"result = _drive(- [_report_page([{"date": "2026-06-30", "app_id": "a1"}])],- resume_state=TenjinResumeConfig(next_url=saved_url),+ always(_report_page([{"date": "2026-06-10", "app_id": "a1"}])),+ resume_state=TenjinResumeConfig(next_url=saved_url, window_end="2026-06-10"),)- assert len(result.requests) == 1- assert result.requests[0].param("page") == "7"+ assert result.params("page") == ["7", None]+ assert result.params("start_date") == [None, "2026-06-11"]+ assert result.params("end_date") == [None, "2026-06-30"]++ def test_failed_window_leaves_a_resume_point_after_the_finished_windows(self) -> None:+ def answer(request: RecordedRequest) -> ScriptedResponse:+ if request.param("start_date") == "2024-07-30":+ return ScriptedResponse(status=400, json={"error": "Invalid request"})+ return _report_page([{"date": request.param("start_date"), "app_id": "a1"}])++ result = _drive(answer)++ assert result.final_state == TenjinResumeConfig(window_end="2024-07-29")++ @parameterized.expand(+ [+ (+ "reason",+ {"error": "start_date test-key is invalid"},+ "Tenjin rejected the report request: start_date REDACTED is invalid",+ ),+ (+ "no_reason",+ {"message": "nope"},+ "Tenjin rejected the report request. Check that your account has access to the requested report and metrics.",+ ),+ ]+ )+ def test_rejected_request_error_carries_tenjins_reason(+ self, _name: str, body: dict[str, Any], expected: str+ ) -> None:+ result = _drive([ScriptedResponse(status=400, json=body)])++ assert str(result.raised) == expectedclass TestTenjinSourceResponse: