:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(mailgun): narrow the logs window to the account's log retention (#123654)
posthog[bot]9b5f142be1ab
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.pyindex 537be3892177..0ca757a6128f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py@@ -1,3 +1,4 @@+import reimport timeimport dataclassesfrom collections import Counter@@ -84,6 +85,12 @@ LOGS_PAGE_SIZE = 100# Mailgun keeps logs for its plan's event retention, at most 30 days on standard plans, so a# full refresh asks for the longest window any account can still hold.LOGS_LOOKBACK_DAYS = 30+# Plans with shorter retention reject a window starting before it with a 400 naming the earliest+# time the account still holds, so the window is narrowed to that time instead of failing the sync.+# The time uses Go's default format, which adds fractional seconds only when they are non-zero.+LOGS_RETENTION_ERROR_PATTERN = re.compile(+ r"before log retention time (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})(\.\d+)? ([+-]\d{4})"+)class MailgunRetryableError(Exception):@@ -668,6 +675,22 @@ def _logs_request_body(start: datetime, end: datetime, token: Optional[str]) ->}+def _logs_retention_start(error: requests.HTTPError) -> Optional[datetime]:+ response = error.response+ if response is None or response.status_code != 400:+ return None+ match = LOGS_RETENTION_ERROR_PATTERN.search(response.text)+ if match is None:+ return None+ whole_seconds, fraction, offset = match.groups()+ retention_start = datetime.strptime(f"{whole_seconds} {offset}", "%Y-%m-%d %H:%M:%S %z").astimezone(UTC)+ # The request sends whole seconds, so a fractional retention time rounds up. Rounding down would+ # send a start just before the retention time, and Mailgun would reject it again.+ if fraction:+ retention_start += timedelta(seconds=1)+ return retention_start++def get_logs_rows(api_key: str,region: str,@@ -699,7 +722,20 @@ def get_logs_rows(seen_tokens: set[str] = set()while start < end:- data = _fetch_page(url, api_key, logger, json_body=_logs_request_body(start, end, token))+ try:+ data = _fetch_page(url, api_key, logger, json_body=_logs_request_body(start, end, token))+ except requests.HTTPError as error:+ retention_start = _logs_retention_start(error)+ # Requiring the start to move forward means a repeated rejection can't loop forever.+ if retention_start is None or retention_start <= start:+ raise+ logger.info(+ f"Mailgun: narrowing {LOGS_ENDPOINT} to the account's log retention, starting at {retention_start}"+ )+ # A page token only belongs to the window that issued it.+ start, token = min(retention_start, end), None+ seen_tokens.clear()+ continueitems = data.get("items") or []next_token = (data.get("pagination") or {}).get("next")if token is not None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.pyindex e94e92af2f81..89b4313bad9f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py@@ -548,6 +548,49 @@ class TestLogsRows:start = parsedate_to_datetime(result.requests[0].json()["start"])assert abs(start - (now - timedelta(days=expected_start_days_ago))) < timedelta(minutes=1)+ @pytest.mark.parametrize(+ "fraction, expected_start_delay",+ [+ ("", timedelta(0)),+ # The request carries whole seconds, so it can only start inside retention one second later.+ (".25", timedelta(seconds=1)),+ ],+ )+ def test_narrows_the_window_to_the_accounts_log_retention(self, fraction, expected_start_delay):+ retention_start = (datetime.now(UTC) - timedelta(days=4)).replace(microsecond=0)+ retention_time = retention_start + timedelta(seconds=float(fraction or 0))+ retention_message = (+ "start time 2024-07-01 00:00:00 +0000 UTC with log retention days 5 is before log retention time "+ f"{retention_start.strftime('%Y-%m-%d %H:%M:%S')}{fraction} +0000 UTC"+ )++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if parsedate_to_datetime(request.json()["start"]) < retention_time:+ return ScriptedResponse(status=400, json={"message": retention_message})+ return ScriptedResponse(json={"items": [_log_item("e1")], "pagination": {}})++ result = _driver().run("logs", respond)++ assert result.raised is None+ starts = [parsedate_to_datetime(request.json()["start"]) for request in result.requests]+ assert len(starts) == 2+ assert starts[1] == retention_start + expected_start_delay+ assert [row["id"] for row in result.rows] == ["e1"]++ @pytest.mark.parametrize(+ "message",+ [+ "invalid filter",+ # A retention time that doesn't move the window forward would only be rejected again.+ "start time is before log retention time 2000-01-01 00:00:00 +0000 UTC",+ ],+ )+ def test_other_bad_requests_still_fail_the_sync(self, message):+ result = _driver().run("logs", lambda _request: ScriptedResponse(status=400, json={"message": message}))++ assert isinstance(result.raised, requests.HTTPError)+ assert len(result.requests) == 1+@pytest.mark.parametrize("resume_token, saved_start, expected_tokens",[