Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(platform): every PostHog event carries the revision that sent it (#15609)

Reinier van der Leer8afd5f3b5f4b
diff --git a/autogpt_platform/backend/backend/util/feature_flag/posthog.py b/autogpt_platform/backend/backend/util/feature_flag/posthog.py
index 14e785ca9e..a95b2d443d 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/posthog.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/posthog.py
@@ -14,6 +14,7 @@ from typing import Any
from posthog import Posthog
+from backend.util.posthog_client import revision_properties
from backend.util.settings import Settings
from .definition_cache import (
@@ -222,6 +223,7 @@ def _build_client() -> Posthog | None:
enable_local_evaluation=bool(secret_key),
poll_interval=refresh_interval_seconds(),
flag_definition_cache_provider=definition_cache,
+ super_properties=revision_properties(settings),
)
_init_attempted = True
logger.info(
diff --git a/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py b/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py
index 60e9f7d34a..b536542dfb 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py
@@ -54,6 +54,22 @@ class TestClientConstruction:
assert kwargs["secret_key"] is None
assert kwargs["enable_local_evaluation"] is False
+ @pytest.mark.parametrize(
+ ("sentry_release", "expected"),
+ [("0123abcd", {"revision": "0123abcd"}), ("", {})],
+ ids=["revision set", "no revision"],
+ )
+ def test_its_events_carry_the_revision(self, mocker, sentry_release, expected):
+ """``$feature_flag_called`` comes from this client, not ``capture``."""
+ configure(mocker)
+ mocker.patch.object(ph.settings.config, "sentry_release", sentry_release)
+ posthog = mocker.patch.object(ph, "Posthog")
+
+ ph.get_flag_client()
+
+ _, kwargs = posthog.call_args
+ assert kwargs["super_properties"] == expected
+
def test_the_client_is_built_once(self, mocker):
configure(mocker)
posthog = mocker.patch.object(ph, "Posthog")
diff --git a/autogpt_platform/backend/backend/util/posthog_client.py b/autogpt_platform/backend/backend/util/posthog_client.py
index 4cebbaf15e..0777222b4a 100644
--- a/autogpt_platform/backend/backend/util/posthog_client.py
+++ b/autogpt_platform/backend/backend/util/posthog_client.py
@@ -5,7 +5,9 @@ process is both cheaper and safer than one per module. Emitters send through
:func:`capture`, which adds the base properties every event carries
(``environment`` and ``source``, see ``docs/engineering/tracking-plan.md``),
treats a missing client as "analytics disabled" and never lets tracking
-raise into the request or execution that produced the event.
+raise into the request or execution that produced the event. Every client
+built here also stamps ``revision`` on everything it sends (see
+:func:`revision_properties`).
"""
import asyncio
@@ -55,11 +57,26 @@ def new_posthog_client(**options: Any) -> Posthog | None:
settings = Settings()
if not settings.secrets.posthog_api_key:
return None
+ super_properties = {
+ **(options.pop("super_properties", None) or {}),
+ **revision_properties(settings),
+ }
return Posthog(
- settings.secrets.posthog_api_key, host=settings.secrets.posthog_host, **options
+ settings.secrets.posthog_api_key,
+ host=settings.secrets.posthog_host,
+ super_properties=super_properties,
+ **options,
)
+def revision_properties(settings: Settings) -> dict[str, str]:
+ """``revision`` for a client to stamp on every event it sends: the deployed
+ build's git sha, which Sentry reports as its release. Empty when the
+ deploy sets none, so local events carry no revision rather than an empty one."""
+ revision = settings.config.sentry_release.strip()
+ return {"revision": revision} if revision else {}
+
+
# The client drops events once its in-memory queue is full.
FLUSH_EVERY = 500
diff --git a/autogpt_platform/backend/backend/util/posthog_client_test.py b/autogpt_platform/backend/backend/util/posthog_client_test.py
index 38b7b22969..047a4a0a87 100644
--- a/autogpt_platform/backend/backend/util/posthog_client_test.py
+++ b/autogpt_platform/backend/backend/util/posthog_client_test.py
@@ -33,6 +33,7 @@ def test_concurrent_initialization_reuses_one_client(
settings = Mock()
settings.secrets.posthog_api_key = "test-key"
settings.secrets.posthog_host = "https://example.com"
+ settings.config.sentry_release = ""
factory = Mock(side_effect=create_client)
monkeypatch.setattr(posthog_client, "_client", None)
monkeypatch.setattr(posthog_client, "Settings", Mock(return_value=settings))
@@ -52,7 +53,9 @@ def test_concurrent_initialization_reuses_one_client(
assert first.result(timeout=5) is client
assert second.result(timeout=5) is client
- factory.assert_called_once_with("test-key", host="https://example.com")
+ factory.assert_called_once_with(
+ "test-key", host="https://example.com", super_properties={}
+ )
def test_disabled_analytics_does_not_create_a_client(
@@ -231,6 +234,7 @@ def test_a_client_of_its_own_gets_the_options_the_shared_one_lacks(
settings = Mock()
settings.secrets.posthog_api_key = "test-key"
settings.secrets.posthog_host = "https://example.com"
+ settings.config.sentry_release = ""
factory = Mock()
on_error = Mock()
monkeypatch.setattr(posthog_client, "Settings", Mock(return_value=settings))
@@ -238,12 +242,42 @@ def test_a_client_of_its_own_gets_the_options_the_shared_one_lacks(
assert posthog_client.new_posthog_client(on_error=on_error) is factory.return_value
factory.assert_called_once_with(
- "test-key", host="https://example.com", on_error=on_error
+ "test-key", host="https://example.com", super_properties={}, on_error=on_error
)
# The shared client is not created or replaced on the way.
assert posthog_client._client is None
+ ("sentry_release", "expected"),
+ [("0123abcd\n", {"revision": "0123abcd"}), ("", {})],
+ ids=["revision set", "no revision"],
+)
+def test_every_event_a_client_sends_carries_the_revision(
+ monkeypatch: pytest.MonkeyPatch, sentry_release: str, expected: dict[str, str]
+) -> None:
+ settings = Mock()
+ settings.secrets.posthog_api_key = "phc_test"
+ settings.secrets.posthog_host = "http://127.0.0.1:9"
+ settings.config.sentry_release = sentry_release
+ monkeypatch.setattr(posthog_client, "Settings", Mock(return_value=settings))
+ sent: list[dict] = []
+ # Returning None drops each event after the client has built it.
+ client = posthog_client.new_posthog_client(
+ super_properties={"cohort": "beta"}, before_send=sent.append, sync_mode=True
+ )
+ assert client is not None
+
+ posthog_client.capture("user-1", PostHogEvent.AI_METRIC, client=client)
+ client.set(distinct_id="user-1", properties={"plan": "pro"})
+ client.shutdown()
+
+ assert [
+ {k: v for k, v in event["properties"].items() if k in ("revision", "cohort")}
+ for event in sent
+ ] == [{"cohort": "beta", **expected}] * 2
+
+
def test_no_client_of_its_own_when_analytics_is_off(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/autogpt_platform/frontend/next.config.mjs b/autogpt_platform/frontend/next.config.mjs
index 62f2b76711..27b1306327 100644
--- a/autogpt_platform/frontend/next.config.mjs
+++ b/autogpt_platform/frontend/next.config.mjs
@@ -19,6 +19,10 @@ const nextConfig = {
NEXT_PUBLIC_VERCEL_TARGET_ENV: process.env.VERCEL_TARGET_ENV ?? "",
// Sentry initialises in the browser too, so its kill switch must reach it.
DISABLE_SENTRY: process.env.DISABLE_SENTRY ?? "",
+ // The build's commit, which Sentry's plugin also names its release, so
+ // PostHog events can be split by deploy.
+ NEXT_PUBLIC_REVISION:
+ process.env.SENTRY_RELEASE || process.env.VERCEL_GIT_COMMIT_SHA || "",
},
// Suppress the "X-Powered-By: Next.js" header (framework fingerprinting).
poweredByHeader: false,
diff --git a/autogpt_platform/frontend/src/services/analytics/__tests__/posthog-base-properties.test.ts b/autogpt_platform/frontend/src/services/analytics/__tests__/posthog-base-properties.test.ts
index 0d91eacca7..73b9ea2ad1 100644
--- a/autogpt_platform/frontend/src/services/analytics/__tests__/posthog-base-properties.test.ts
+++ b/autogpt_platform/frontend/src/services/analytics/__tests__/posthog-base-properties.test.ts
@@ -34,6 +34,24 @@ describe("getPostHogBaseProperties", () => {
expect(getPostHogBaseProperties().environment).toBe("preview");
});
+ it("carries the build's revision", () => {
+ stubDeployment("prod", "production", "production");
+ vi.stubEnv("NEXT_PUBLIC_REVISION", "0123abcd");
+
+ expect(getPostHogBaseProperties()).toEqual({
+ source: "web",
+ environment: "prod",
+ revision: "0123abcd",
+ });
+ });
+
+ it("leaves the revision out of a build that has none", () => {
+ stubDeployment("prod", "production", "production");
+ vi.stubEnv("NEXT_PUBLIC_REVISION", "");
+
+ expect(getPostHogBaseProperties()).not.toHaveProperty("revision");
+ });
+
it("reports the dev deployment as dev although Vercel calls its staging environment a preview", () => {
stubDeployment("dev", "preview", "staging");
diff --git a/autogpt_platform/frontend/src/services/analytics/posthog-base-properties.ts b/autogpt_platform/frontend/src/services/analytics/posthog-base-properties.ts
index d9ad4fdbf6..5e31cf5ca9 100644
--- a/autogpt_platform/frontend/src/services/analytics/posthog-base-properties.ts
+++ b/autogpt_platform/frontend/src/services/analytics/posthog-base-properties.ts
@@ -7,11 +7,14 @@ import { environment } from "@/services/environment";
// production NEXT_PUBLIC_APP_ENV, so they are told apart here, as in
// `environment.getEnvironmentStr`. A custom Vercel environment, such as the dev
// deployment's `staging`, is not a preview and reports its own app env.
+// `revision` is the build's commit, left out of a build that has none.
export function getPostHogBaseProperties() {
+ const revision = environment.getRevision();
return {
source: "web",
environment: environment.isVercelPreview()
? "preview"
: environment.getAppEnv(),
+ ...(revision ? { revision } : {}),
};
}
diff --git a/autogpt_platform/frontend/src/services/environment/index.ts b/autogpt_platform/frontend/src/services/environment/index.ts
index cb99ef07df..3203003000 100644
--- a/autogpt_platform/frontend/src/services/environment/index.ts
+++ b/autogpt_platform/frontend/src/services/environment/index.ts
@@ -88,6 +88,10 @@ function getPreviewStealingDev() {
return branch;
}
+function getRevision() {
+ return process.env.NEXT_PUBLIC_REVISION?.trim() || "";
+}
+
function getPostHogCredentials() {
return {
key: process.env.NEXT_PUBLIC_POSTHOG_KEY,
@@ -205,6 +209,7 @@ export const environment = {
getAGPTServerBaseUrl,
getAGPTWsServerUrl,
getPreviewStealingDev,
+ getRevision,
getPostHogCredentials,
getLaunchDarklyClientId,
getGoogleAdsID,
diff --git a/docs/engineering/tracking-plan.md b/docs/engineering/tracking-plan.md
index 88ae429d36..c0e4112eac 100644
--- a/docs/engineering/tracking-plan.md
+++ b/docs/engineering/tracking-plan.md
@@ -59,10 +59,18 @@ Every event must carry:
| --- | --- |
| `environment` | `settings.config.app_env` on the backend (`local`, `dev`, `prod`), the matching app environment in the browser. |
| `source` | Which emitter sent it: `chat_copilot` (the events `copilot/tracking.py` captures itself; `chat_message_sent` goes through `product_analytics` and is `platform`), `platform` (every other backend emitter), `web` (the browser). |
+| `revision` | The deployed build's git sha, which Sentry reports as its release: `SENTRY_RELEASE` on the backend, the build's commit in the browser (`SENTRY_RELEASE`, else Vercel's `VERCEL_GIT_COMMIT_SHA`). Left out of a build that has none, such as a local one, and from `$ai_generation` (below). |
Every backend event goes through `capture()` in
-`backend/util/posthog_client.py`, which applies both last, so a caller cannot
-overwrite them. The browser registers them once as PostHog super properties
+`backend/util/posthog_client.py`, which applies `environment` and `source`
+last, so a caller cannot overwrite them. `revision` is a super property of
+every backend PostHog client instead (`revision_properties()`), so the flag
+client's `$feature_flag_called` carries it too. AutoPilot's `$ai_generation`
+is the exception: OpenRouter builds it from the request rather than through
+our client, so it carries no `revision` until #15604 adds `trace.revision`,
+which arrives as `metadata_revision`. #15605's `$ai_trace` and `$ai_span` go
+through `capture()`, so they carry `revision` as soon as they ship. The
+browser registers all three once as PostHog super properties
(`src/services/analytics/posthog-base-properties.ts`), again after `reset()`
at logout; a Vercel preview reports `environment: preview`, while the dev
deployment, which is Vercel's `staging` custom environment rather than a