Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(credentials-service): record caller-side request latency (#122178)

Tom Owers714f0f2b5132
diff --git a/posthog/credentials_service/client.py b/posthog/credentials_service/client.py
index de9f82b809a8..b27591dc9a13 100644
--- a/posthog/credentials_service/client.py
+++ b/posthog/credentials_service/client.py
@@ -45,7 +45,7 @@ from django.conf import settings
import requests
import structlog
import posthoganalytics
-from prometheus_client import Counter
+from prometheus_client import Counter, Histogram
from posthog.dataclasses import frozen
from posthog.jwt import PosthogJwtAudience, encode_jwt
@@ -85,6 +85,16 @@ CREDENTIALS_SERVICE_ENV_FALLBACK_COUNTER = Counter(
labelnames=["reason"],
)
+# End to end from the caller's side: signing the token, the round trip, and parsing the body.
+# The service's own histogram starts when the request arrives, so it misses connection setup and
+# anything slow inside the calling pod.
+CREDENTIALS_SERVICE_REQUEST_DURATION = Histogram(
+ "posthog_credentials_service_request_duration_seconds",
+ "Time a caller spends on one credentials service request, from signing the token to a parsed body",
+ labelnames=["caller", "result"],
+ buckets=(0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0),
+)
+
@frozen
class SecretValue:
@@ -312,9 +322,9 @@ class CredentialsServiceClient:
# Minted outside the try: signing is local work, and a failure there is a bug in this
# process, not the service being unreachable. Catching it here would file it under the
# one label nobody investigates.
+ started = time.monotonic()
headers = self._auth_headers(keys, caller)
url = f"{settings.CREDENTIALS_SERVICE_URL.rstrip('/')}{RESOLVE_PATH}"
- started = time.monotonic()
try:
response = self._session.post(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)
response.raise_for_status()
@@ -324,10 +334,14 @@ class CredentialsServiceClient:
body: dict[str, Any] = response.json()
except (requests.RequestException, ValueError) as e:
CREDENTIALS_SERVICE_FETCH_COUNTER.labels(outcome="unreachable").inc()
+ CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="error").observe(
+ time.monotonic() - started
+ )
_log_resolve(url, keys, caller, started, response=getattr(e, "response", None), error=e)
raise CredentialsServiceUnreachableError(
f"The credentials service did not answer a credential request: {e}"
) from e
+ CREDENTIALS_SERVICE_REQUEST_DURATION.labels(caller=str(caller), result="ok").observe(time.monotonic() - started)
_log_resolve(url, keys, caller, started, response=response, body=body)
return body
diff --git a/posthog/credentials_service/test/test_client.py b/posthog/credentials_service/test/test_client.py
index f30d2b597cd0..be779d3175a7 100644
--- a/posthog/credentials_service/test/test_client.py
+++ b/posthog/credentials_service/test/test_client.py
@@ -10,6 +10,7 @@ from django.test import SimpleTestCase, override_settings
import jwt
import requests
from parameterized import parameterized
+from prometheus_client import REGISTRY
from posthog.credentials_service.callers import CredentialsCaller
from posthog.credentials_service.client import (
@@ -194,6 +195,28 @@ class TestCredentialsServiceClient(SimpleTestCase):
with pytest.raises(CredentialsServiceUnreachableError):
self.secrets.get(KEY, CALLER)
+ @parameterized.expand(
+ [
+ ("answered", None, "ok"),
+ ("unreachable", requests.ConnectionError("credentials service is down"), "error"),
+ ]
+ )
+ def test_records_request_latency_by_result(self, _name: str, raised: Exception | None, result: str) -> None:
+ labels = {"caller": str(CALLER), "result": result}
+ before = REGISTRY.get_sample_value("posthog_credentials_service_request_duration_seconds_count", labels) or 0
+ patched = (
+ patch(POST, side_effect=raised)
+ if raised
+ else patch(POST, return_value=FakeResponse(body({KEY: steady("sec")})))
+ )
+ with patched:
+ try:
+ self.secrets.get(KEY, CALLER)
+ except CredentialsServiceUnreachableError:
+ pass
+ after = REGISTRY.get_sample_value("posthog_credentials_service_request_duration_seconds_count", labels)
+ assert after == before + 1
+
# The base type is the whole contract a caller depends on: catch one thing, and a subclass
# added later is covered without every call site being revisited.
@parameterized.expand(