:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(warehouse): log team_id when person-property update goes to DLQ (#122293)
Jovan Sakovicdd8446b09e8d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.pyindex 73c2489e41ba..2296155f3214 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer.py@@ -16,6 +16,7 @@ import jsonimport timeimport signalfrom collections.abc import Callable+from concurrent.futures import ThreadPoolExecutorfrom typing import Anyfrom django.conf import settings@@ -33,6 +34,7 @@ from posthog.kafka_client.topics import (KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES_DLQ,)from posthog.models.instance_setting import get_instance_setting+from posthog.models.team import Teamfrom posthog.settings import CONSTANCE_CONFIGlogger = structlog.get_logger(__name__)@@ -188,6 +190,26 @@ def _is_permanent_client_error(status_code: object) -> bool:return isinstance(status_code, int) and 400 <= status_code < 500 and status_code not in _RETRYABLE_CLIENT_STATUSES+# One worker, so a hung cache or database call blocks later lookups instead of piling up threads.+# Each caller still waits at most the timeout, then logs without a team.+_TEAM_LOOKUP_TIMEOUT_SECONDS = 2.0+_team_lookup_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="dlq-team-lookup")+++def _team_id_for_token(token: object) -> int | None:+ """Best-effort team lookup so a DLQ log line can name the affected team. Never raises and never+ waits past the timeout: a slow or failed lookup must not stall or retry a DLQ write."""+ if not isinstance(token, str):+ return None+ try:+ team = _team_lookup_executor.submit(Team.objects.get_team_from_cache_or_token, token).result(+ timeout=_TEAM_LOOKUP_TIMEOUT_SECONDS+ )+ except Exception:+ return None+ return team.id if team else None++class PersonPropertyUpdateConsumer:def __init__(self,@@ -225,11 +247,11 @@ class PersonPropertyUpdateConsumer:self._dlq_producer = get_producer(topic=KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES_DLQ)return self._dlq_producer- def _dlq(self, value: bytes, reason: str) -> str:+ def _dlq(self, value: bytes, reason: str, token: object = None) -> str:"""Route a poison message to the DLQ. Returns ``DLQ`` only once the write is confirmeddelivered; a failed DLQ write returns ``RETRY`` so the source offset stays uncommitted andthe message is redelivered rather than silently dropped from both topics."""- logger.warning("person_property_update.dlq", reason=reason)+ logger.warning("person_property_update.dlq", reason=reason, team_id=_team_id_for_token(token))producer = self._get_dlq_producer()result = producer.produce(topic=KAFKA_WAREHOUSE_PERSON_PROPERTY_UPDATES_DLQ,@@ -309,7 +331,7 @@ class PersonPropertyUpdateConsumer:# 0 (transport/client-side), retryable 4xx (408/429), and 5xx stay transient -> uncommitted.status_code = getattr(exc, "status_code", None)if _is_permanent_client_error(status_code):- return self._dlq(value, f"capture_client_error:{status_code}")+ return self._dlq(value, f"capture_client_error:{status_code}", kwargs["token"])RETRY_TOTAL.inc()logger.exception("person_property_update.capture_error")return RETRY@@ -319,12 +341,12 @@ class PersonPropertyUpdateConsumer:# A drop is terminal: capture rejected the event permanently (e.g. invalid/stale token,# validation failure), so it's poison. DLQ it rather than redelivering forever.if result.dropped:- return self._dlq(value, "capture_dropped")+ return self._dlq(value, "capture_dropped", kwargs["token"])# A permanent whole-request 4xx is likewise poison (bad/stale token, validation), so DLQ it# too. Everything else (exhausted retries, unaccounted, retryable 4xx, 5xx, transport) is# transient -> retry.if result.error and _is_permanent_client_error(result.status_code):- return self._dlq(value, f"capture_error:{result.status_code}")+ return self._dlq(value, f"capture_error:{result.status_code}", kwargs["token"])RETRY_TOTAL.inc()return RETRYdiff --git a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.pyindex d2558b7f153c..1524796a3e23 100644--- a/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py+++ b/products/warehouse_sources/backend/temporal/data_imports/person_property_update_consumer_test.py@@ -1,4 +1,5 @@import json+import threadingfrom types import SimpleNamespaceimport pytest@@ -7,6 +8,7 @@ from unittest.mock import MagicMock, patchfrom django.test import override_settingsfrom parameterized import parameterized+from structlog.testing import capture_logsfrom products.warehouse_sources.backend.temporal.data_imports.person_property_update_consumer import (_DEFAULT_RATE_PER_SEC,@@ -20,6 +22,8 @@ from products.warehouse_sources.backend.temporal.data_imports.person_property_upbuild_capture_kwargs,)+hang = threading.Event()+_MODULE = "products.warehouse_sources.backend.temporal.data_imports.person_property_update_consumer"_SETTING = ("products.warehouse_sources.backend.temporal.data_imports.person_property_update_consumer.get_instance_setting")@@ -180,6 +184,32 @@ class TestProcessRecord:assert c.process_record(value) == DLQdlq.produce.assert_called_once()+ @parameterized.expand(+ [+ ("team_found", {"return_value": SimpleNamespace(id=42)}, 42),+ ("team_missing", {"return_value": None}, None),+ ("lookup_error", {"side_effect": RuntimeError("db down")}, None),+ ("lookup_hangs", {"side_effect": lambda _token: hang.wait(5)}, None),+ ]+ )+ def test_capture_rejection_dlq_log_names_team(self, _name, lookup_kwargs, expected_team_id):+ hang.clear()+ capture = MagicMock(return_value=_capture_result(succeeded=False, error={"error": "x"}, status_code=402))+ c = self._consumer(capture)+ value = json.dumps({"token": "t", "distinct_id": "a", "properties": {"p": 1}}).encode()++ with (+ patch(f"{_MODULE}.Team.objects.get_team_from_cache_or_token", **lookup_kwargs),+ patch(f"{_MODULE}._TEAM_LOOKUP_TIMEOUT_SECONDS", 0.05),+ capture_logs() as logs,+ ):+ assert c.process_record(value) == DLQ+ hang.set()++ [entry] = [e for e in logs if e["event"] == "person_property_update.dlq"]+ assert entry["reason"] == "capture_error:402"+ assert entry["team_id"] == expected_team_id+@parameterized.expand([("invalid_json", b"not json"),