Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-vision): report the entry point on the scanner created event (#128386)

Kim Svatos Dugan9ba036dcecfb
diff --git a/products/replay_vision/backend/api/scanners.py b/products/replay_vision/backend/api/scanners.py
index 1890c9f56608..8cbf4ecd9fe0 100644
--- a/products/replay_vision/backend/api/scanners.py
+++ b/products/replay_vision/backend/api/scanners.py
@@ -222,6 +222,11 @@ class ScannerCreationMethod(models.TextChoices):
SCRATCH = "scratch", "From scratch"
+def _event_source(context: dict[str, Any]) -> EventSource | None:
+ request = context.get("request")
+ return get_event_source(request) if request is not None else context.get("event_source")
+
+
def _reported_creation_method(context: dict[str, Any], claimed: str | None) -> str | None:
"""What `creation_method` says on the created event.
@@ -238,13 +243,24 @@ def _reported_creation_method(context: dict[str, Any], claimed: str | None) -> s
context the same way it passes `user`. Without that it would fall through as unattributed, which
is the one gap a request-derived source cannot close.
"""
- request = context.get("request")
- source = get_event_source(request) if request is not None else context.get("event_source")
+ source = _event_source(context)
if source is None:
return claimed
return claimed if source == EventSource.WEB else source.value
+def _reported_entry_point(context: dict[str, Any], claimed: str | None) -> str | None:
+ """What `entry_point` says on the created event: the in-app surface, or the caller for anything else.
+
+ Only the app has in-app surfaces, so any other caller reports its own source, whatever it
+ sends. Otherwise an agent could claim `replay_filters` and count toward that surface.
+ """
+ source = _event_source(context)
+ if source is None or source == EventSource.WEB:
+ return claimed
+ return source.value
+
+
# Size caps enforced at the write boundary; scanner_config and query are copied into every observation's snapshot.
_MAX_DESCRIPTION_LENGTH = 1_000
@@ -523,6 +539,18 @@ class ReplayScannerSerializer(TaggedItemSerializerMixin, UserAccessControlSerial
"calling surface instead of whatever it sends here. Ignored on update."
),
)
+ entry_point = serializers.RegexField(
+ r"^[a-z0-9_]{1,64}$",
+ required=False,
+ allow_null=True,
+ write_only=True,
+ help_text=(
+ "The app surface the creator started from, such as `replay_filters`. Reported to product "
+ "analytics at creation so a scanner can be attributed to its entry point, and not stored "
+ "on the scanner. Only the app's value is kept: any other caller is reported as its own "
+ "source, such as `mcp`. Ignored on update."
+ ),
+ )
scanner_config = serializers.JSONField(
help_text=(
"Type-specific configuration. All scanner types require `prompt`; monitors add optional `allow_inconclusive`, "
@@ -703,6 +731,7 @@ class ReplayScannerSerializer(TaggedItemSerializerMixin, UserAccessControlSerial
"scanner_type",
"goal",
"creation_method",
+ "entry_point",
"take_over_as_creator",
"scanner_config",
"prompt_question",
@@ -1189,6 +1218,7 @@ class ReplayScannerSerializer(TaggedItemSerializerMixin, UserAccessControlSerial
tags = validated_data.pop("tags", None)
# Telemetry only, so it must not reach the model constructor.
creation_method = validated_data.pop("creation_method", None)
+ entry_point = validated_data.pop("entry_point", None)
# A model call, so it runs before the transaction opens.
validated_data.update(
question_fields_for_save(
@@ -1215,6 +1245,7 @@ class ReplayScannerSerializer(TaggedItemSerializerMixin, UserAccessControlSerial
{
**scanner_lifecycle_properties(scanner),
"creation_method": _reported_creation_method(self.context, creation_method),
+ "entry_point": _reported_entry_point(self.context, entry_point),
},
team=team,
**analytics_source_kwargs(self.context),
@@ -1229,6 +1260,7 @@ class ReplayScannerSerializer(TaggedItemSerializerMixin, UserAccessControlSerial
# A scanner is built once, so this says nothing about an edit. Dropped here because the UI
# PATCHes the whole form back and would otherwise send it into the getattr diff below.
validated_data.pop("creation_method", None)
+ validated_data.pop("entry_point", None)
# The goal records what the scanner was first built for, so an edit never rewrites it.
validated_data.pop("goal", None)
# Compared as tagify()d names, since that is what set_tags_on_object stores.
diff --git a/products/replay_vision/backend/tests/test_api.py b/products/replay_vision/backend/tests/test_api.py
index 5b5edddb11bc..24bc5d4b7c36 100644
--- a/products/replay_vision/backend/tests/test_api.py
+++ b/products/replay_vision/backend/tests/test_api.py
@@ -1685,12 +1685,12 @@ class TestScannerLifecycleTelemetry(_VisionAPITestCase):
@parameterized.expand(
[
- ("app", {}, "ai", "ai"),
- ("wizard", {"HTTP_USER_AGENT": "posthog/wizard 1.2.3"}, "scratch", "wizard"),
+ ("app", {}, "ai", "ai", "replay_filters"),
+ ("wizard", {"HTTP_USER_AGENT": "posthog/wizard 1.2.3"}, "scratch", "wizard", "wizard"),
]
)
def test_create_reports_how_the_scanner_was_built(
- self, _name: str, headers: dict[str, Any], claimed: str, expected: str
+ self, _name: str, headers: dict[str, Any], claimed: str, expected: str, expected_entry_point: str
) -> None:
# A metric comparing AI-built against hand-built scanners needs to know how the form was filled.
#
@@ -1706,6 +1706,7 @@ class TestScannerLifecycleTelemetry(_VisionAPITestCase):
"scanner_config": {"prompt": "did checkout complete?"},
"model": ScannerModel.GEMINI_3_8_FLASH,
"creation_method": claimed,
+ "entry_point": "replay_filters",
},
format="json",
**headers,
@@ -1716,8 +1717,13 @@ class TestScannerLifecycleTelemetry(_VisionAPITestCase):
call for call in capture.call_args_list if call.kwargs.get("event") == "replay_vision_scanner_created"
]
self.assertEqual(created[0].kwargs["properties"]["creation_method"], expected)
+ # An entry point experiment compares creation by the surface the person started from, so only
+ # the app can claim one: any other caller reports itself, whatever it sends.
+ self.assertEqual(created[0].kwargs["properties"]["entry_point"], expected_entry_point)
# Telemetry only: it must not reach the model, whose constructor would reject it.
- self.assertFalse(hasattr(ReplayScanner.objects.get(id=resp.json()["id"]), "creation_method"))
+ scanner = ReplayScanner.objects.get(id=resp.json()["id"])
+ self.assertFalse(hasattr(scanner, "creation_method"))
+ self.assertFalse(hasattr(scanner, "entry_point"))
def test_create_without_a_request_reports_the_declared_surface(self) -> None:
# Max reaches the serializer directly, so there is no request to derive a surface from and
@@ -1755,13 +1761,14 @@ class TestScannerLifecycleTelemetry(_VisionAPITestCase):
with patch("posthoganalytics.capture") as capture:
resp = self.client.patch(
f"{self.scanners_url}{scanner.id}/",
- data={"name": "renamed", "creation_method": "template"},
+ data={"name": "renamed", "creation_method": "template", "entry_point": "replay_filters"},
format="json",
)
self.assertEqual(resp.status_code, 200, resp.json())
edited = [call for call in capture.call_args_list if call.kwargs.get("event") == "replay_vision_scanner_edited"]
self.assertNotIn("creation_method", edited[0].kwargs["properties"])
+ self.assertNotIn("entry_point", edited[0].kwargs["properties"])
@parameterized.expand(
[
diff --git a/products/replay_vision/frontend/generated/api.schemas.ts b/products/replay_vision/frontend/generated/api.schemas.ts
index afb9c1787aca..945685c2e1ab 100644
--- a/products/replay_vision/frontend/generated/api.schemas.ts
+++ b/products/replay_vision/frontend/generated/api.schemas.ts
@@ -1360,6 +1360,12 @@ export interface ReplayScannerApi {
* * `template` - Template
* * `scratch` - From scratch */
creation_method?: ScannerCreationMethodEnumApi | null
+ /**
+ * The app surface the creator started from, such as `replay_filters`. Reported to product analytics at creation so a scanner can be attributed to its entry point, and not stored on the scanner. Only the app's value is kept: any other caller is reported as its own source, such as `mcp`. Ignored on update.
+ * @nullable
+ * @pattern ^[a-z0-9_]{1,64}$
+ */
+ entry_point?: string | null
/** Set with `enabled: true` to turn on a scanner whose creator can no longer view recordings. The caller becomes the scanner's creator, and scheduled scans then run as the caller. Without it, that request fails with code `scanner_creator_lost_access`. */
take_over_as_creator?: boolean
/** Type-specific configuration. All scanner types require `prompt`; monitors add optional `allow_inconclusive`, classifiers add `tags`, scorers add `scale`, summarizers add optional `length`. */
@@ -1513,6 +1519,12 @@ export interface PatchedReplayScannerApi {
* * `template` - Template
* * `scratch` - From scratch */
creation_method?: ScannerCreationMethodEnumApi | null
+ /**
+ * The app surface the creator started from, such as `replay_filters`. Reported to product analytics at creation so a scanner can be attributed to its entry point, and not stored on the scanner. Only the app's value is kept: any other caller is reported as its own source, such as `mcp`. Ignored on update.
+ * @nullable
+ * @pattern ^[a-z0-9_]{1,64}$
+ */
+ entry_point?: string | null
/** Set with `enabled: true` to turn on a scanner whose creator can no longer view recordings. The caller becomes the scanner's creator, and scheduled scans then run as the caller. Without it, that request fails with code `scanner_creator_lost_access`. */
take_over_as_creator?: boolean
/** Type-specific configuration. All scanner types require `prompt`; monitors add optional `allow_inconclusive`, classifiers add `tags`, scorers add `scale`, summarizers add optional `length`. */
diff --git a/products/replay_vision/frontend/generated/api.zod.ts b/products/replay_vision/frontend/generated/api.zod.ts
index a513ba08923b..e1b7adf44c58 100644
--- a/products/replay_vision/frontend/generated/api.zod.ts
+++ b/products/replay_vision/frontend/generated/api.zod.ts
@@ -605,6 +605,7 @@ export const visionScannersCreateBodyTagsMax = 32
export const visionScannersCreateBodyGoalMax = 2000
+export const visionScannersCreateBodyEntryPointRegExp = new RegExp('^[a-z0-9_]{1,64}$')
export const visionScannersCreateBodySamplingRateMin = 0
export const visionScannersCreateBodySamplingRateMax = 1
@@ -656,6 +657,13 @@ export const VisionScannersCreateBody = /* @__PURE__ */ zod
.describe(
'How the creator built this scanner: from an AI draft, from a template, or from scratch. Reported to product analytics at creation and not stored on the scanner. Independent of any experiment the creator is in, since a person offered the AI flow can still fill the form by hand. Only the app can answer this, so a request from anywhere else reports the calling surface instead of whatever it sends here. Ignored on update.\n\n\* `ai` - AI draft\n\* `template` - Template\n\* `scratch` - From scratch'
),
+ entry_point: zod
+ .string()
+ .regex(visionScannersCreateBodyEntryPointRegExp)
+ .nullish()
+ .describe(
+ "The app surface the creator started from, such as `replay_filters`. Reported to product analytics at creation so a scanner can be attributed to its entry point, and not stored on the scanner. Only the app's value is kept: any other caller is reported as its own source, such as `mcp`. Ignored on update."
+ ),
take_over_as_creator: zod
.boolean()
.optional()
@@ -766,6 +774,7 @@ export const visionScannersPartialUpdateBodyTagsMax = 32
export const visionScannersPartialUpdateBodyGoalMax = 2000
+export const visionScannersPartialUpdateBodyEntryPointRegExp = new RegExp('^[a-z0-9_]{1,64}$')
export const visionScannersPartialUpdateBodySamplingRateMin = 0
export const visionScannersPartialUpdateBodySamplingRateMax = 1
@@ -819,6 +828,13 @@ export const VisionScannersPartialUpdateBody = /* @__PURE__ */ zod
.describe(
'How the creator built this scanner: from an AI draft, from a template, or from scratch. Reported to product analytics at creation and not stored on the scanner. Independent of any experiment the creator is in, since a person offered the AI flow can still fill the form by hand. Only the app can answer this, so a request from anywhere else reports the calling surface instead of whatever it sends here. Ignored on update.\n\n\* `ai` - AI draft\n\* `template` - Template\n\* `scratch` - From scratch'
),
+ entry_point: zod
+ .string()
+ .regex(visionScannersPartialUpdateBodyEntryPointRegExp)
+ .nullish()
+ .describe(
+ "The app surface the creator started from, such as `replay_filters`. Reported to product analytics at creation so a scanner can be attributed to its entry point, and not stored on the scanner. Only the app's value is kept: any other caller is reported as its own source, such as `mcp`. Ignored on update."
+ ),
take_over_as_creator: zod
.boolean()
.optional()
diff --git a/products/replay_vision/mcp/tools.yaml b/products/replay_vision/mcp/tools.yaml
index b5b74c1be40a..e84cca591c27 100644
--- a/products/replay_vision/mcp/tools.yaml
+++ b/products/replay_vision/mcp/tools.yaml
@@ -609,6 +609,7 @@ tools:
sentence for them, not a step for you.
exclude_params:
- query
+ - entry_point
- id
- scanner_version
- last_swept_at
@@ -1092,6 +1093,7 @@ tools:
`vision-quota-get` first — the scanner resumes sweeping every 5 minutes at the new, broader scope.
exclude_params:
- query
+ - entry_point
- id
- scanner_version
- last_swept_at
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index e5b208a9624d..a9cda96a567e 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -72147,6 +72147,12 @@ export namespace Schemas {
* * `template` - Template
* * `scratch` - From scratch */
creation_method?: ScannerCreationMethodEnum | null;
+ /**
+ * The app surface the creator started from, such as `replay_filters`. Reported to product analytics at creation so a scanner can be attributed to its entry point, and not stored on the scanner. Only the app's value is kept: any other caller is reported as its own source, such as `mcp`. Ignored on update.
+ * @nullable
+ * @pattern ^[a-z0-9_]{1,64}$
+ */
+ entry_point?: string | null;
/** Set with `enabled: true` to turn on a scanner whose creator can no longer view recordings. The caller becomes the scanner's creator, and scheduled scans then run as the caller. Without it, that request fails with code `scanner_creator_lost_access`. */
take_over_as_creator?: boolean;
/** Type-specific configuration. All scanner types require `prompt`; monitors add optional `allow_inconclusive`, classifiers add `tags`, scorers add `scale`, summarizers add optional `length`. */
@@ -85131,6 +85137,12 @@ export namespace Schemas {
* * `template` - Template
* * `scratch` - From scratch */
creation_method?: ScannerCreationMethodEnum | null;
+ /**
+ * The app surface the creator started from, such as `replay_filters`. Reported to product analytics at creation so a scanner can be attributed to its entry point, and not stored on the scanner. Only the app's value is kept: any other caller is reported as its own source, such as `mcp`. Ignored on update.
+ * @nullable
+ * @pattern ^[a-z0-9_]{1,64}$
+ */
+ entry_point?: string | null;
/** Set with `enabled: true` to turn on a scanner whose creator can no longer view recordings. The caller becomes the scanner's creator, and scheduled scans then run as the caller. Without it, that request fails with code `scanner_creator_lost_access`. */
take_over_as_creator?: boolean;
/** Type-specific configuration. All scanner types require `prompt`; monitors add optional `allow_inconclusive`, classifiers add `tags`, scorers add `scale`, summarizers add optional `length`. */