:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(credentials-service): retry dropped connections in the client (#124931)
Tom Owers5879b087f4bf
diff --git a/posthog/credentials_service/client.py b/posthog/credentials_service/client.pyindex c62aecdc389c..79c6ebf02297 100644--- a/posthog/credentials_service/client.py+++ b/posthog/credentials_service/client.py@@ -15,7 +15,9 @@ Three behaviours are load-bearing:2. **A failure is a failure.** With no cache there is no last known good, so anunreachable service raises rather than quietly serving something stale. The service- needs an availability SLO before the environment variables come out.+ needs an availability SLO before the environment variables come out. The one exception+ is a transport blip: a dropped connection or a gateway status gets a fresh attempt+ first, because the request is a read and repeating it changes nothing.3. **Environment fallback only when the client is off.** Flag disabled, a flag check thaterrored, or the deployment not fully configured means "read `os.environ` as before". That@@ -61,6 +63,16 @@ RESOLVE_PATH = "/v1/secrets/resolve"TOKEN_TTL = timedelta(minutes=5)REQUEST_TIMEOUT_SECONDS = 5+# The service is a Node server with the default 5 second keep-alive timeout, and the pooled+# `requests` connection has no idle limit of its own. A request sent on a connection the server+# is closing at that moment fails with `RemoteDisconnected`. urllib3 drops the broken connection,+# so the next attempt opens a new one. The request is a read, so a repeat is safe even when the+# service saw the first attempt. A read timeout is not retried: the service answered nothing in+# 5 seconds, and two more waits of the same length would only delay the failure.+REQUEST_ATTEMPTS = 3+RETRY_BACKOFF_SECONDS = 0.1+RETRYABLE_STATUS_CODES = frozenset({502, 503, 504})+# Rollout gate. Evaluated per call, so it can be turned off without a deploy if the# service misbehaves. posthoganalytics evaluates locally when a personal API key is# configured; if that ever stops being true this becomes a network hop on the credential@@ -74,6 +86,12 @@ CREDENTIALS_SERVICE_FETCH_COUNTER = Counter(labelnames=["outcome"],)+CREDENTIALS_SERVICE_RETRY_COUNTER = Counter(+ "posthog_credentials_service_retry_total",+ "Credentials service requests sent again after a dropped connection or a gateway status",+ labelnames=["reason"],+)+CREDENTIALS_SERVICE_ENV_FALLBACK_COUNTER = Counter("posthog_credentials_service_env_fallback_total","Credential reads served from the local environment instead of the credentials service",@@ -177,6 +195,16 @@ def credentials_service_enabled() -> bool:return _disabled_reason() is None+def _retry_reason(error: BaseException) -> str | None:+ """The retry counter label for a failure worth another attempt, or None to fail now."""+ if isinstance(error, requests.ConnectionError):+ return "connection_error"+ status_code = getattr(getattr(error, "response", None), "status_code", None)+ if status_code in RETRYABLE_STATUS_CODES:+ return str(status_code)+ return None++def _log_resolve(url: str,keys: list[str],@@ -186,6 +214,7 @@ def _log_resolve(response: Any = None,body: dict[str, Any] | None = None,error: BaseException | None = None,+ attempt: int = 1,) -> None:"""One structured line per credential request. Never raises.@@ -227,6 +256,7 @@ def _log_resolve("states": states,"version_ids": versions,"missing": missing,+ "attempt": attempt,}if error is not None or (status_code is not None and status_code >= 400):@@ -330,25 +360,37 @@ class CredentialsServiceClient:started = time.monotonic()headers = self._auth_headers(keys, caller)url = f"{settings.CREDENTIALS_SERVICE_URL.rstrip('/')}{RESOLVE_PATH}"- try:- response = self._session.post(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)- response.raise_for_status()- # ValueError covers a body that is not JSON. Modern `requests` raises its own- # JSONDecodeError (a RequestException), but the stdlib/simplejson ValueError still- # surfaces on older paths, and nothing else in this block raises one.- body: dict[str, Any] = response.json()- except (requests.RequestException, ValueError) as e:- CREDENTIALS_SERVICE_FETCH_COUNTER.labels(outcome="unreachable").inc()- CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="error").observe(+ attempt = 1+ while True:+ try:+ response = self._session.post(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)+ response.raise_for_status()+ # ValueError covers a body that is not JSON. Modern `requests` raises its own+ # JSONDecodeError (a RequestException), but the stdlib/simplejson ValueError still+ # surfaces on older paths, and nothing else in this block raises one.+ body: dict[str, Any] = response.json()+ except (requests.RequestException, ValueError) as e:+ error_response = getattr(e, "response", None)+ retry_reason = _retry_reason(e)+ if retry_reason is not None and attempt < REQUEST_ATTEMPTS:+ CREDENTIALS_SERVICE_RETRY_COUNTER.labels(reason=retry_reason).inc()+ _log_resolve(url, keys, caller, started, response=error_response, error=e, attempt=attempt)+ time.sleep(RETRY_BACKOFF_SECONDS * attempt)+ attempt += 1+ continue+ CREDENTIALS_SERVICE_FETCH_COUNTER.labels(outcome="unreachable").inc()+ CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="error").observe(+ time.monotonic() - started+ )+ _log_resolve(url, keys, caller, started, response=error_response, error=e, attempt=attempt)+ raise CredentialsServiceUnreachableError(+ f"The credentials service did not answer a credential request after {attempt} attempt(s): {e}"+ ) from e+ CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="ok").observe(time.monotonic() - started)- _log_resolve(url, keys, caller, started, response=getattr(e, "response", None), error=e)- raise CredentialsServiceUnreachableError(- f"The credentials service did not answer a credential request: {e}"- ) from e- CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="ok").observe(time.monotonic() - started)- _log_resolve(url, keys, caller, started, response=response, body=body)- return body+ _log_resolve(url, keys, caller, started, response=response, body=body, attempt=attempt)+ return bodydef _auth_headers(self, keys: list[str], caller: CredentialsCaller) -> dict[str, str]:token = encode_jwt(diff --git a/posthog/credentials_service/test/test_client.py b/posthog/credentials_service/test/test_client.pyindex 2aefeac41e8e..87eb6ca015d8 100644--- a/posthog/credentials_service/test/test_client.py+++ b/posthog/credentials_service/test/test_client.py@@ -37,6 +37,7 @@ CALLER = CredentialsCaller.WAREHOUSE_SOURCESFLAG = "posthog.credentials_service.client.posthoganalytics.feature_enabled"POST = "posthog.credentials_service.client.internal_requests.post"+SLEEP = "posthog.credentials_service.client.time.sleep"class FakeResponse:@@ -54,6 +55,23 @@ def body(secrets: dict[str, Any] | None = None, missing: list[str] | None = Nonereturn {"secrets": secrets or {}, "missing": missing or []}+def http_error(status: int) -> requests.HTTPError:+ response = requests.Response()+ response.status_code = status+ return requests.HTTPError(f"{status} Error", response=response)+++class ErrorResponse:+ def __init__(self, raised: Exception) -> None:+ self._raised = raised++ def raise_for_status(self) -> None:+ raise self._raised++ def json(self) -> dict[str, Any]:+ return {}++def steady(value: str) -> dict[str, Any]:return {"state": "steady", "value": value, "version_id": "v1", "fetched_at": "now"}@@ -65,6 +83,9 @@ class TestCredentialsServiceClient(SimpleTestCase):flag = patch(FLAG, return_value=True)flag.start()self.addCleanup(flag.stop)+ sleep = patch(SLEEP)+ sleep.start()+ self.addCleanup(sleep.stop)def test_returns_the_value_the_service_resolved(self) -> None:with patch(POST, return_value=FakeResponse(body({KEY: steady("sec")}))):@@ -136,31 +157,53 @@ class TestCredentialsServiceClient(SimpleTestCase):# raised — and callers act on that difference. The 404 is the case that bites: a misrouted# CREDENTIALS_SERVICE_URL is our deploy error, but a caller seeing a raw 404 reads it as the# user's endpoint being gone and can stop their work over it.+ #+ # The attempt count is the retry contract. A dropped connection or a gateway status is sent+ # again, and anything else fails on the first answer: a 4xx will not change on a repeat, and a+ # repeated read timeout only multiplies the wait in front of the same failure.@parameterized.expand([- ("404 misrouted url", requests.HTTPError("404 Client Error", response=requests.Response())),- ("401 unaccepted signing key", requests.HTTPError("401 Unauthorized", response=requests.Response())),- ("connection refused", requests.ConnectionError("connection refused")),- ("read timeout", requests.Timeout("timed out")),+ ("404 misrouted url", http_error(404), 1),+ ("401 unaccepted signing key", http_error(401), 1),+ ("read timeout", requests.Timeout("timed out"), 1),+ ("connection refused", requests.ConnectionError("connection refused"), 3),+ ("503 from a pod without secrets", http_error(503), 3),])- def test_transport_failure_wears_this_clients_type_for(self, _name: str, raised: Exception) -> None:- class ErrorResponse:- def raise_for_status(self) -> None:- raise raised-- def json(self) -> dict[str, Any]:- return {}-+ def test_transport_failure_wears_this_clients_type_for(self, _name: str, raised: Exception, attempts: int) -> None:# Raised from the call itself for a connection failure, from raise_for_status for a status.side_effect = raised if isinstance(raised, requests.ConnectionError | requests.Timeout) else None- patched = patch(POST, side_effect=side_effect) if side_effect else patch(POST, return_value=ErrorResponse())- with patched:+ patched = (+ patch(POST, side_effect=side_effect) if side_effect else patch(POST, return_value=ErrorResponse(raised))+ )+ with patched as post:with pytest.raises(CredentialsServiceUnreachableError) as exc_info:self.secrets.get(KEY, CALLER)# The cause is kept so error tracking and logs still show what actually went wrong.assert exc_info.value.__cause__ is raisedassert not isinstance(exc_info.value, requests.RequestException)+ assert post.call_count == attempts++ # The keep-alive race: the server closes an idle pooled connection just as a request goes out+ # on it. The read never reached the service, so the next attempt on a new connection succeeds.+ @parameterized.expand(+ [+ (+ "a closed keep-alive connection",+ requests.ConnectionError("('Connection aborted.', RemoteDisconnected('Remote end closed connection'))"),+ "connection_error",+ ),+ ("a 503", ErrorResponse(http_error(503)), "503"),+ ("a 502", ErrorResponse(http_error(502)), "502"),+ ]+ )+ def test_answers_after_a_retry_following(self, _name: str, first: Any, reason: str) -> None:+ labels = {"reason": reason}+ before = REGISTRY.get_sample_value("posthog_credentials_service_retry_total", labels) or 0+ with patch(POST, side_effect=[first, FakeResponse(body({KEY: steady("sec")}))]) as post:+ assert self.secrets.get(KEY, CALLER) == "sec"+ assert post.call_count == 2+ assert REGISTRY.get_sample_value("posthog_credentials_service_retry_total", labels) == before + 1# A body that isn't JSON is the same class of failure as no answer at all: something is# between us and the service, or the service is broken. It must not surface as a missing key.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/credentials_service.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/credentials_service.pyindex d465afa5f22e..888bc065a70a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/credentials_service.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/credentials_service.py@@ -54,11 +54,11 @@ def _build_session() -> requests.Session:it. This is the same reason `posthog.security.outbound_proxy.internal_requests` exists, andthe reason we cannot simply reuse a plain tracked session.- `Retry(total=0)` — the client holds no cache, so there is no last known good and a retry here- would only add latency in front of a failure Temporal already retries with backoff. It matches- the shared internal session, whose adapter also does not retry. (`DEFAULT_RETRY` would be a- no-op anyway, since it retries only GET/HEAD/OPTIONS and this is a POST — stated explicitly so- it stays true if that policy changes.)+ `Retry(total=0)` — the client already retries a dropped connection or a gateway status itself,+ for every session it is given. A second retry layer here would multiply those attempts. It+ matches the shared internal session, whose adapter also does not retry. (`DEFAULT_RETRY` would+ be a no-op anyway, since it retries only GET/HEAD/OPTIONS and this is a POST — stated+ explicitly so it stays true if that policy changes.)"""session = make_tracked_session(retry=Retry(total=0), capture=False)session.trust_env = False