Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(mailgun): narrow the logs window to the account's log retention (#123654)

posthog[bot]9b5f142be1ab
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py
index 537be3892177..0ca757a6128f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py
@@ -1,3 +1,4 @@
+import re
import time
import dataclasses
from collections import Counter
@@ -84,6 +85,12 @@ LOGS_PAGE_SIZE = 100
# Mailgun keeps logs for its plan's event retention, at most 30 days on standard plans, so a
# full refresh asks for the longest window any account can still hold.
LOGS_LOOKBACK_DAYS = 30
+# Plans with shorter retention reject a window starting before it with a 400 naming the earliest
+# time the account still holds, so the window is narrowed to that time instead of failing the sync.
+# The time uses Go's default format, which adds fractional seconds only when they are non-zero.
+LOGS_RETENTION_ERROR_PATTERN = re.compile(
+ r"before log retention time (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})(\.\d+)? ([+-]\d{4})"
+)
class MailgunRetryableError(Exception):
@@ -668,6 +675,22 @@ def _logs_request_body(start: datetime, end: datetime, token: Optional[str]) ->
}
+def _logs_retention_start(error: requests.HTTPError) -> Optional[datetime]:
+ response = error.response
+ if response is None or response.status_code != 400:
+ return None
+ match = LOGS_RETENTION_ERROR_PATTERN.search(response.text)
+ if match is None:
+ return None
+ whole_seconds, fraction, offset = match.groups()
+ retention_start = datetime.strptime(f"{whole_seconds} {offset}", "%Y-%m-%d %H:%M:%S %z").astimezone(UTC)
+ # The request sends whole seconds, so a fractional retention time rounds up. Rounding down would
+ # send a start just before the retention time, and Mailgun would reject it again.
+ if fraction:
+ retention_start += timedelta(seconds=1)
+ return retention_start
+
+
def get_logs_rows(
api_key: str,
region: str,
@@ -699,7 +722,20 @@ def get_logs_rows(
seen_tokens: set[str] = set()
while start < end:
- data = _fetch_page(url, api_key, logger, json_body=_logs_request_body(start, end, token))
+ try:
+ data = _fetch_page(url, api_key, logger, json_body=_logs_request_body(start, end, token))
+ except requests.HTTPError as error:
+ retention_start = _logs_retention_start(error)
+ # Requiring the start to move forward means a repeated rejection can't loop forever.
+ if retention_start is None or retention_start <= start:
+ raise
+ logger.info(
+ f"Mailgun: narrowing {LOGS_ENDPOINT} to the account's log retention, starting at {retention_start}"
+ )
+ # A page token only belongs to the window that issued it.
+ start, token = min(retention_start, end), None
+ seen_tokens.clear()
+ continue
items = data.get("items") or []
next_token = (data.get("pagination") or {}).get("next")
if token is not None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py
index e94e92af2f81..89b4313bad9f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py
@@ -548,6 +548,49 @@ class TestLogsRows:
start = parsedate_to_datetime(result.requests[0].json()["start"])
assert abs(start - (now - timedelta(days=expected_start_days_ago))) < timedelta(minutes=1)
+ @pytest.mark.parametrize(
+ "fraction, expected_start_delay",
+ [
+ ("", timedelta(0)),
+ # The request carries whole seconds, so it can only start inside retention one second later.
+ (".25", timedelta(seconds=1)),
+ ],
+ )
+ def test_narrows_the_window_to_the_accounts_log_retention(self, fraction, expected_start_delay):
+ retention_start = (datetime.now(UTC) - timedelta(days=4)).replace(microsecond=0)
+ retention_time = retention_start + timedelta(seconds=float(fraction or 0))
+ retention_message = (
+ "start time 2024-07-01 00:00:00 +0000 UTC with log retention days 5 is before log retention time "
+ f"{retention_start.strftime('%Y-%m-%d %H:%M:%S')}{fraction} +0000 UTC"
+ )
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if parsedate_to_datetime(request.json()["start"]) < retention_time:
+ return ScriptedResponse(status=400, json={"message": retention_message})
+ return ScriptedResponse(json={"items": [_log_item("e1")], "pagination": {}})
+
+ result = _driver().run("logs", respond)
+
+ assert result.raised is None
+ starts = [parsedate_to_datetime(request.json()["start"]) for request in result.requests]
+ assert len(starts) == 2
+ assert starts[1] == retention_start + expected_start_delay
+ assert [row["id"] for row in result.rows] == ["e1"]
+
+ @pytest.mark.parametrize(
+ "message",
+ [
+ "invalid filter",
+ # A retention time that doesn't move the window forward would only be rejected again.
+ "start time is before log retention time 2000-01-01 00:00:00 +0000 UTC",
+ ],
+ )
+ def test_other_bad_requests_still_fail_the_sync(self, message):
+ result = _driver().run("logs", lambda _request: ScriptedResponse(status=400, json={"message": message}))
+
+ assert isinstance(result.raised, requests.HTTPError)
+ assert len(result.requests) == 1
+
@pytest.mark.parametrize(
"resume_token, saved_start, expected_tokens",
[