Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(credentials-service): retry dropped connections in the client (#124931)

Tom Owers5879b087f4bf
diff --git a/posthog/credentials_service/client.py b/posthog/credentials_service/client.py
index c62aecdc389c..79c6ebf02297 100644
--- a/posthog/credentials_service/client.py
+++ b/posthog/credentials_service/client.py
@@ -15,7 +15,9 @@ Three behaviours are load-bearing:
2. **A failure is a failure.** With no cache there is no last known good, so an
unreachable service raises rather than quietly serving something stale. The service
- needs an availability SLO before the environment variables come out.
+ needs an availability SLO before the environment variables come out. The one exception
+ is a transport blip: a dropped connection or a gateway status gets a fresh attempt
+ first, because the request is a read and repeating it changes nothing.
3. **Environment fallback only when the client is off.** Flag disabled, a flag check that
errored, or the deployment not fully configured means "read `os.environ` as before". That
@@ -61,6 +63,16 @@ RESOLVE_PATH = "/v1/secrets/resolve"
TOKEN_TTL = timedelta(minutes=5)
REQUEST_TIMEOUT_SECONDS = 5
+# The service is a Node server with the default 5 second keep-alive timeout, and the pooled
+# `requests` connection has no idle limit of its own. A request sent on a connection the server
+# is closing at that moment fails with `RemoteDisconnected`. urllib3 drops the broken connection,
+# so the next attempt opens a new one. The request is a read, so a repeat is safe even when the
+# service saw the first attempt. A read timeout is not retried: the service answered nothing in
+# 5 seconds, and two more waits of the same length would only delay the failure.
+REQUEST_ATTEMPTS = 3
+RETRY_BACKOFF_SECONDS = 0.1
+RETRYABLE_STATUS_CODES = frozenset({502, 503, 504})
+
# Rollout gate. Evaluated per call, so it can be turned off without a deploy if the
# service misbehaves. posthoganalytics evaluates locally when a personal API key is
# configured; if that ever stops being true this becomes a network hop on the credential
@@ -74,6 +86,12 @@ CREDENTIALS_SERVICE_FETCH_COUNTER = Counter(
labelnames=["outcome"],
)
+CREDENTIALS_SERVICE_RETRY_COUNTER = Counter(
+ "posthog_credentials_service_retry_total",
+ "Credentials service requests sent again after a dropped connection or a gateway status",
+ labelnames=["reason"],
+)
+
CREDENTIALS_SERVICE_ENV_FALLBACK_COUNTER = Counter(
"posthog_credentials_service_env_fallback_total",
"Credential reads served from the local environment instead of the credentials service",
@@ -177,6 +195,16 @@ def credentials_service_enabled() -> bool:
return _disabled_reason() is None
+def _retry_reason(error: BaseException) -> str | None:
+ """The retry counter label for a failure worth another attempt, or None to fail now."""
+ if isinstance(error, requests.ConnectionError):
+ return "connection_error"
+ status_code = getattr(getattr(error, "response", None), "status_code", None)
+ if status_code in RETRYABLE_STATUS_CODES:
+ return str(status_code)
+ return None
+
+
def _log_resolve(
url: str,
keys: list[str],
@@ -186,6 +214,7 @@ def _log_resolve(
response: Any = None,
body: dict[str, Any] | None = None,
error: BaseException | None = None,
+ attempt: int = 1,
) -> None:
"""One structured line per credential request. Never raises.
@@ -227,6 +256,7 @@ def _log_resolve(
"states": states,
"version_ids": versions,
"missing": missing,
+ "attempt": attempt,
}
if error is not None or (status_code is not None and status_code >= 400):
@@ -330,25 +360,37 @@ class CredentialsServiceClient:
started = time.monotonic()
headers = self._auth_headers(keys, caller)
url = f"{settings.CREDENTIALS_SERVICE_URL.rstrip('/')}{RESOLVE_PATH}"
- try:
- response = self._session.post(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)
- response.raise_for_status()
- # ValueError covers a body that is not JSON. Modern `requests` raises its own
- # JSONDecodeError (a RequestException), but the stdlib/simplejson ValueError still
- # surfaces on older paths, and nothing else in this block raises one.
- body: dict[str, Any] = response.json()
- except (requests.RequestException, ValueError) as e:
- CREDENTIALS_SERVICE_FETCH_COUNTER.labels(outcome="unreachable").inc()
- CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="error").observe(
+ attempt = 1
+ while True:
+ try:
+ response = self._session.post(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)
+ response.raise_for_status()
+ # ValueError covers a body that is not JSON. Modern `requests` raises its own
+ # JSONDecodeError (a RequestException), but the stdlib/simplejson ValueError still
+ # surfaces on older paths, and nothing else in this block raises one.
+ body: dict[str, Any] = response.json()
+ except (requests.RequestException, ValueError) as e:
+ error_response = getattr(e, "response", None)
+ retry_reason = _retry_reason(e)
+ if retry_reason is not None and attempt < REQUEST_ATTEMPTS:
+ CREDENTIALS_SERVICE_RETRY_COUNTER.labels(reason=retry_reason).inc()
+ _log_resolve(url, keys, caller, started, response=error_response, error=e, attempt=attempt)
+ time.sleep(RETRY_BACKOFF_SECONDS * attempt)
+ attempt += 1
+ continue
+ CREDENTIALS_SERVICE_FETCH_COUNTER.labels(outcome="unreachable").inc()
+ CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="error").observe(
+ time.monotonic() - started
+ )
+ _log_resolve(url, keys, caller, started, response=error_response, error=e, attempt=attempt)
+ raise CredentialsServiceUnreachableError(
+ f"The credentials service did not answer a credential request after {attempt} attempt(s): {e}"
+ ) from e
+ CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="ok").observe(
time.monotonic() - started
)
- _log_resolve(url, keys, caller, started, response=getattr(e, "response", None), error=e)
- raise CredentialsServiceUnreachableError(
- f"The credentials service did not answer a credential request: {e}"
- ) from e
- CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="ok").observe(time.monotonic() - started)
- _log_resolve(url, keys, caller, started, response=response, body=body)
- return body
+ _log_resolve(url, keys, caller, started, response=response, body=body, attempt=attempt)
+ return body
def _auth_headers(self, keys: list[str], caller: CredentialsCaller) -> dict[str, str]:
token = encode_jwt(
diff --git a/posthog/credentials_service/test/test_client.py b/posthog/credentials_service/test/test_client.py
index 2aefeac41e8e..87eb6ca015d8 100644
--- a/posthog/credentials_service/test/test_client.py
+++ b/posthog/credentials_service/test/test_client.py
@@ -37,6 +37,7 @@ CALLER = CredentialsCaller.WAREHOUSE_SOURCES
FLAG = "posthog.credentials_service.client.posthoganalytics.feature_enabled"
POST = "posthog.credentials_service.client.internal_requests.post"
+SLEEP = "posthog.credentials_service.client.time.sleep"
class FakeResponse:
@@ -54,6 +55,23 @@ def body(secrets: dict[str, Any] | None = None, missing: list[str] | None = None
return {"secrets": secrets or {}, "missing": missing or []}
+def http_error(status: int) -> requests.HTTPError:
+ response = requests.Response()
+ response.status_code = status
+ return requests.HTTPError(f"{status} Error", response=response)
+
+
+class ErrorResponse:
+ def __init__(self, raised: Exception) -> None:
+ self._raised = raised
+
+ def raise_for_status(self) -> None:
+ raise self._raised
+
+ def json(self) -> dict[str, Any]:
+ return {}
+
+
def steady(value: str) -> dict[str, Any]:
return {"state": "steady", "value": value, "version_id": "v1", "fetched_at": "now"}
@@ -65,6 +83,9 @@ class TestCredentialsServiceClient(SimpleTestCase):
flag = patch(FLAG, return_value=True)
flag.start()
self.addCleanup(flag.stop)
+ sleep = patch(SLEEP)
+ sleep.start()
+ self.addCleanup(sleep.stop)
def test_returns_the_value_the_service_resolved(self) -> None:
with patch(POST, return_value=FakeResponse(body({KEY: steady("sec")}))):
@@ -136,31 +157,53 @@ class TestCredentialsServiceClient(SimpleTestCase):
# raised — and callers act on that difference. The 404 is the case that bites: a misrouted
# CREDENTIALS_SERVICE_URL is our deploy error, but a caller seeing a raw 404 reads it as the
# user's endpoint being gone and can stop their work over it.
+ #
+ # The attempt count is the retry contract. A dropped connection or a gateway status is sent
+ # again, and anything else fails on the first answer: a 4xx will not change on a repeat, and a
+ # repeated read timeout only multiplies the wait in front of the same failure.
@parameterized.expand(
[
- ("404 misrouted url", requests.HTTPError("404 Client Error", response=requests.Response())),
- ("401 unaccepted signing key", requests.HTTPError("401 Unauthorized", response=requests.Response())),
- ("connection refused", requests.ConnectionError("connection refused")),
- ("read timeout", requests.Timeout("timed out")),
+ ("404 misrouted url", http_error(404), 1),
+ ("401 unaccepted signing key", http_error(401), 1),
+ ("read timeout", requests.Timeout("timed out"), 1),
+ ("connection refused", requests.ConnectionError("connection refused"), 3),
+ ("503 from a pod without secrets", http_error(503), 3),
]
)
- def test_transport_failure_wears_this_clients_type_for(self, _name: str, raised: Exception) -> None:
- class ErrorResponse:
- def raise_for_status(self) -> None:
- raise raised
-
- def json(self) -> dict[str, Any]:
- return {}
-
+ def test_transport_failure_wears_this_clients_type_for(self, _name: str, raised: Exception, attempts: int) -> None:
# Raised from the call itself for a connection failure, from raise_for_status for a status.
side_effect = raised if isinstance(raised, requests.ConnectionError | requests.Timeout) else None
- patched = patch(POST, side_effect=side_effect) if side_effect else patch(POST, return_value=ErrorResponse())
- with patched:
+ patched = (
+ patch(POST, side_effect=side_effect) if side_effect else patch(POST, return_value=ErrorResponse(raised))
+ )
+ with patched as post:
with pytest.raises(CredentialsServiceUnreachableError) as exc_info:
self.secrets.get(KEY, CALLER)
# The cause is kept so error tracking and logs still show what actually went wrong.
assert exc_info.value.__cause__ is raised
assert not isinstance(exc_info.value, requests.RequestException)
+ assert post.call_count == attempts
+
+ # The keep-alive race: the server closes an idle pooled connection just as a request goes out
+ # on it. The read never reached the service, so the next attempt on a new connection succeeds.
+ @parameterized.expand(
+ [
+ (
+ "a closed keep-alive connection",
+ requests.ConnectionError("('Connection aborted.', RemoteDisconnected('Remote end closed connection'))"),
+ "connection_error",
+ ),
+ ("a 503", ErrorResponse(http_error(503)), "503"),
+ ("a 502", ErrorResponse(http_error(502)), "502"),
+ ]
+ )
+ def test_answers_after_a_retry_following(self, _name: str, first: Any, reason: str) -> None:
+ labels = {"reason": reason}
+ before = REGISTRY.get_sample_value("posthog_credentials_service_retry_total", labels) or 0
+ with patch(POST, side_effect=[first, FakeResponse(body({KEY: steady("sec")}))]) as post:
+ assert self.secrets.get(KEY, CALLER) == "sec"
+ assert post.call_count == 2
+ assert REGISTRY.get_sample_value("posthog_credentials_service_retry_total", labels) == before + 1
# A body that isn't JSON is the same class of failure as no answer at all: something is
# between us and the service, or the service is broken. It must not surface as a missing key.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/credentials_service.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/credentials_service.py
index d465afa5f22e..888bc065a70a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/credentials_service.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/credentials_service.py
@@ -54,11 +54,11 @@ def _build_session() -> requests.Session:
it. This is the same reason `posthog.security.outbound_proxy.internal_requests` exists, and
the reason we cannot simply reuse a plain tracked session.
- `Retry(total=0)` — the client holds no cache, so there is no last known good and a retry here
- would only add latency in front of a failure Temporal already retries with backoff. It matches
- the shared internal session, whose adapter also does not retry. (`DEFAULT_RETRY` would be a
- no-op anyway, since it retries only GET/HEAD/OPTIONS and this is a POST — stated explicitly so
- it stays true if that policy changes.)
+ `Retry(total=0)` — the client already retries a dropped connection or a gateway status itself,
+ for every session it is given. A second retry layer here would multiply those attempts. It
+ matches the shared internal session, whose adapter also does not retry. (`DEFAULT_RETRY` would
+ be a no-op anyway, since it retries only GET/HEAD/OPTIONS and this is a POST — stated
+ explicitly so it stays true if that policy changes.)
"""
session = make_tracked_session(retry=Retry(total=0), capture=False)
session.trust_env = False