:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(credentials-service): record caller-side request latency (#122178)
Tom Owers714f0f2b5132
diff --git a/posthog/credentials_service/client.py b/posthog/credentials_service/client.pyindex de9f82b809a8..b27591dc9a13 100644--- a/posthog/credentials_service/client.py+++ b/posthog/credentials_service/client.py@@ -45,7 +45,7 @@ from django.conf import settingsimport requestsimport structlogimport posthoganalytics-from prometheus_client import Counter+from prometheus_client import Counter, Histogramfrom posthog.dataclasses import frozenfrom posthog.jwt import PosthogJwtAudience, encode_jwt@@ -85,6 +85,16 @@ CREDENTIALS_SERVICE_ENV_FALLBACK_COUNTER = Counter(labelnames=["reason"],)+# End to end from the caller's side: signing the token, the round trip, and parsing the body.+# The service's own histogram starts when the request arrives, so it misses connection setup and+# anything slow inside the calling pod.+CREDENTIALS_SERVICE_REQUEST_DURATION = Histogram(+ "posthog_credentials_service_request_duration_seconds",+ "Time a caller spends on one credentials service request, from signing the token to a parsed body",+ labelnames=["caller", "result"],+ buckets=(0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0),+)+@frozenclass SecretValue:@@ -312,9 +322,9 @@ class CredentialsServiceClient:# Minted outside the try: signing is local work, and a failure there is a bug in this# process, not the service being unreachable. Catching it here would file it under the# one label nobody investigates.+ started = time.monotonic()headers = self._auth_headers(keys, caller)url = f"{settings.CREDENTIALS_SERVICE_URL.rstrip('/')}{RESOLVE_PATH}"- started = time.monotonic()try:response = self._session.post(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)response.raise_for_status()@@ -324,10 +334,14 @@ class CredentialsServiceClient:body: dict[str, Any] = response.json()except (requests.RequestException, ValueError) as e:CREDENTIALS_SERVICE_FETCH_COUNTER.labels(outcome="unreachable").inc()+ CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="error").observe(+ time.monotonic() - started+ )_log_resolve(url, keys, caller, started, response=getattr(e, "response", None), error=e)raise CredentialsServiceUnreachableError(f"The credentials service did not answer a credential request: {e}") from e+ CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="ok").observe(time.monotonic() - started)_log_resolve(url, keys, caller, started, response=response, body=body)return bodydiff --git a/posthog/credentials_service/test/test_client.py b/posthog/credentials_service/test/test_client.pyindex f30d2b597cd0..be779d3175a7 100644--- a/posthog/credentials_service/test/test_client.py+++ b/posthog/credentials_service/test/test_client.py@@ -10,6 +10,7 @@ from django.test import SimpleTestCase, override_settingsimport jwtimport requestsfrom parameterized import parameterized+from prometheus_client import REGISTRYfrom posthog.credentials_service.callers import CredentialsCallerfrom posthog.credentials_service.client import (@@ -194,6 +195,28 @@ class TestCredentialsServiceClient(SimpleTestCase):with pytest.raises(CredentialsServiceUnreachableError):self.secrets.get(KEY, CALLER)+ @parameterized.expand(+ [+ ("answered", None, "ok"),+ ("unreachable", requests.ConnectionError("credentials service is down"), "error"),+ ]+ )+ def test_records_request_latency_by_result(self, _name: str, raised: Exception | None, result: str) -> None:+ labels = {"caller": str(CALLER), "result": result}+ before = REGISTRY.get_sample_value("posthog_credentials_service_request_duration_seconds_count", labels) or 0+ patched = (+ patch(POST, side_effect=raised)+ if raised+ else patch(POST, return_value=FakeResponse(body({KEY: steady("sec")})))+ )+ with patched:+ try:+ self.secrets.get(KEY, CALLER)+ except CredentialsServiceUnreachableError:+ pass+ after = REGISTRY.get_sample_value("posthog_credentials_service_request_duration_seconds_count", labels)+ assert after == before + 1+# The base type is the whole contract a caller depends on: catch one thing, and a subclass# added later is covered without every call site being revisited.@parameterized.expand(