:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(activity-log): record destination tests that send real requests (#127963)
Tom Piccirello44a022574298
diff --git a/docs/internal/activity-logging.md b/docs/internal/activity-logging.mdindex 25f5defcf4a1..14d7a2cfc7da 100644--- a/docs/internal/activity-logging.md+++ b/docs/internal/activity-logging.md@@ -134,6 +134,12 @@ Never put a credential field in that branch.Destination activity logs record changes to `mappings` without their values.For `inputs` they keep each key and record only whether its value changed, so the history can name the changed input.+A destination test with `mock_async_functions` set to false writes a `tested` row before the test runs, so a failed test is recorded too.+If that row cannot be written, the test does not run.+A test of a native or Segment destination writes the row whatever `mock_async_functions` says, because those executors send real requests even when the test mocks.+Its context holds the `hog` source, the input and mapping input keys, and the ids of the integrations the inputs name, never an input value.+A test of an unsaved function has no `item_id`.+Any other mocked test writes no row, because it cannot send data out.Rows written before a field joined `field_with_masked_contents` still store its values.Guided Inbox onboarding first saves a draft when the sources step is completed; opening or editing that first step keeps the draft local.Later setup changes are recorded while masking the saved draft, including scout instructions and project evidence.diff --git a/frontend/src/scenes/hog-functions/misc/activityDescriptions.tsx b/frontend/src/scenes/hog-functions/misc/activityDescriptions.tsxindex cc2bbe5b4f92..6ea4e87def83 100644--- a/frontend/src/scenes/hog-functions/misc/activityDescriptions.tsx+++ b/frontend/src/scenes/hog-functions/misc/activityDescriptions.tsx@@ -275,6 +275,19 @@ export function hogFunctionActivityDescriber(logItem: ActivityLogItem, asNotific}}+ if (logItem.activity == 'tested') {+ const functionName = nameOrLinkToHogFunction(logItem?.item_id, logItem?.detail.name)++ return {+ summary: activityLogSummary(logItem, `Tested the ${objectNoun} with real requests`, functionName),+ description: (+ <>+ <ActivityLogUserName logItem={logItem} /> tested the {objectNoun} with real requests: {functionName}+ </>+ ),+ }+ }+const draftActivities: Record<string, string> = {draft_updated: 'staged changes for review on',published: 'published the staged changes to',diff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.pyindex 7c5c1492ffb2..bc8f4e2e81e2 100644--- a/posthog/models/activity_logging/activity_log.py+++ b/posthog/models/activity_logging/activity_log.py@@ -1570,7 +1570,7 @@ def log_activity("user": user,"was_impersonated": was_impersonated,"is_system": user is None,- "item_id": str(item_id),+ "item_id": str(item_id) if item_id is not None else None,"scope": scope,"activity": activity,"detail": detail,diff --git a/products/cdp/backend/api/hog_function.py b/products/cdp/backend/api/hog_function.pyindex cdc5168c91c6..eb36c60b6328 100644--- a/products/cdp/backend/api/hog_function.py+++ b/products/cdp/backend/api/hog_function.py@@ -70,6 +70,7 @@ from products.cdp.backend.presentation.hog_invocations.rerun import (HogInvocationRerunRequestSerializer,HogInvocationRerunResponseSerializer,)+from products.cdp.backend.services.destination_test_runs import destination_test_run_context, sends_real_requestsfrom products.cdp.backend.services.icons import CDPIconsServicefrom products.cdp.backend.site_functions import get_transpiled_functionfrom products.cdp.backend.validation import (@@ -1448,6 +1449,28 @@ class HogFunctionViewSet(# Remove the team from the configconfiguration.pop("team")+ mock_async_functions = serializer.validated_data["mock_async_functions"]+ if sends_real_requests(configuration, mock_async_functions=mock_async_functions):+ # The row is written before the worker call, so a test that then fails is still recorded.+ # Without `strict`, production drops a failed write and the test still sends real requests.+ log_activity(+ strict=True,+ organization_id=self.organization.id,+ team_id=self.team_id,+ user=cast(User, request.user),+ was_impersonated=is_impersonated(request),+ item_id=str(hog_function.id) if hog_function else None,+ scope="HogFunction",+ activity="tested",+ detail=Detail(+ name=configuration.get("name") or (hog_function.name if hog_function else None),+ type=humanize_hog_function_type(configuration.get("type")),+ context=destination_test_run_context(+ configuration, stored=hog_function, mock_async_functions=mock_async_functions+ ),+ ),+ )+res = create_hog_invocation_test(team_id=self.team_id,hog_function_id=str(hog_function.id) if hog_function else "new",diff --git a/products/cdp/backend/api/test/test_hog_function.py b/products/cdp/backend/api/test/test_hog_function.pyindex a1d88c9923cf..5e3b146d3486 100644--- a/products/cdp/backend/api/test/test_hog_function.py+++ b/products/cdp/backend/api/test/test_hog_function.py@@ -6,12 +6,14 @@ import time_machinefrom posthog.test.base import APIBaseTest, ClickhouseTestMixin, QueryMatchingTestfrom unittest.mock import ANY, MagicMock, patch-from django.db import connection+from django.conf import UserSettingsHolder, settings+from django.db import DatabaseError, connectionfrom django.test.utils import CaptureQueriesContextfrom parameterized import parameterizedfrom rest_framework import status+from posthog.models.activity_logging.activity_log import ActivityLogfrom posthog.models.integration import Integrationfrom products.access_control.backend.models.access_control import AccessControl@@ -2780,6 +2782,126 @@ class TestHogFunctionAPI(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):"value": "http://localhost:2080/0e02d917-563f-4050-9725-aad881b69937",}+ @parameterized.expand(+ [+ ("saved_function_real_requests", True, False, False, None),+ ("saved_function_runs_its_stored_config", True, False, True, None),+ ("new_function_real_requests", False, False, False, None),+ ("new_function_mocked_requests", False, True, False, None),+ ("native_destination_ignores_mocking", False, True, False, "native-example"),+ ("segment_destination_ignores_mocking", True, True, False, "segment-example"),+ ]+ )+ def test_test_invocation_with_real_requests_is_logged_without_input_values(+ self,+ _name: str,+ saved: bool,+ mock_async_functions: bool,+ omit_stored_fields: bool,+ template_id: Optional[str],+ ):+ integration = Integration.objects.create(team=self.team, kind="slack", config={}, created_by=self.user)+ mapping_integration = Integration.objects.create(team=self.team, kind="slack", config={}, created_by=self.user)+ configuration = {+ **EXAMPLE_FULL,+ "inputs_schema": [+ *EXAMPLE_FULL["inputs_schema"],+ {"key": "workspace", "type": "integration", "integration": "slack"},+ ],+ "inputs": {+ **EXAMPLE_FULL["inputs"],+ "workspace": {"value": integration.id},+ "payload": {"value": {"event": "{event}", "team": "{inputs.workspace.team}"}},+ },+ "mappings": [+ {+ "name": "Pageviews",+ "inputs_schema": [+ {"key": "mapping_workspace", "type": "integration", "integration": "slack"},+ {"key": "mapping_text", "type": "string"},+ ],+ "inputs": {+ "mapping_workspace": {"value": mapping_integration.id},+ "mapping_text": {"value": "{{ inputs.mapping_workspace.team }}", "templating": "liquid"},+ },+ }+ ],+ }+ if template_id:+ HogFunctionTemplate.objects.create(+ template_id=template_id,+ sha="1.0.0",+ name="Example",+ description="Example destination",+ code="return event",+ code_language="hog",+ inputs_schema=[],+ type="destination",+ status="stable",+ category=["Other"],+ free=True,+ )+ configuration["template_id"] = template_id+ function_id = "new"+ if saved:+ created = self.client.post(f"/api/projects/{self.team.id}/hog_functions/", data=configuration)+ assert created.status_code == status.HTTP_201_CREATED, created.json()+ function_id = created.json()["id"]++ tested_configuration = (+ {+ key: value+ for key, value in configuration.items()+ if key not in ("hog", "inputs", "inputs_schema", "mappings")+ }+ if omit_stored_fields+ else configuration+ )+ with patch("products.cdp.backend.api.hog_function.create_hog_invocation_test") as mock_invocation:+ mock_invocation.return_value = MagicMock(status_code=500, json=lambda: {"errors": ["Worker failed"]})+ response = self.client.post(+ f"/api/projects/{self.team.id}/hog_functions/{function_id}/invocations/",+ data={"configuration": tested_configuration, "mock_async_functions": mock_async_functions},+ )+ assert mock_invocation.call_count == 1, response.json()++ rows = list(ActivityLog.objects.filter(team_id=self.team.id, scope="HogFunction", activity="tested"))+ if mock_async_functions and not template_id:+ assert rows == []+ return+ assert len(rows) == 1+ assert rows[0].user == self.user+ assert rows[0].item_id == (function_id if saved else None)+ detail = rows[0].detail+ assert detail is not None+ assert detail["context"] == {+ "mock_async_functions": mock_async_functions,+ "integration_ids": sorted([integration.id, mapping_integration.id]),+ "input_keys": ["headers", "method", "payload", "url", "workspace"],+ "mapping_input_keys": ["mapping_text", "mapping_workspace"],+ "input_references": {"payload": ["workspace"]},+ "mapping_input_references": {"mapping_text": ["mapping_workspace"]},+ "hog": EXAMPLE_FULL["hog"],+ }+ assert "0e02d917-563f-4050-9725-aad881b69937" not in json.dumps(detail)++ def test_test_invocation_with_real_requests_does_not_run_when_its_activity_log_write_fails(self):+ # Under `settings.TEST`, `log_activity` raises on every failed write, strict or not. The override+ # stays in its module, because route modules read `settings.TEST` when the URL conf loads.+ production_settings = UserSettingsHolder(settings)+ production_settings.TEST = False+ with (+ patch("products.cdp.backend.api.hog_function.create_hog_invocation_test") as mock_invocation,+ patch.object(ActivityLog.objects, "create", side_effect=DatabaseError("insert rejected")),+ patch("posthog.models.activity_logging.activity_log.settings", production_settings),+ ):+ response = self.client.post(+ f"/api/projects/{self.team.id}/hog_functions/new/invocations/",+ data={"configuration": EXAMPLE_FULL, "mock_async_functions": False},+ )+ assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR+ mock_invocation.assert_not_called()+def test_test_invocation_rejects_a_posthog_connection_input(self):connection = Integration.objects.create(team=self.team, kind="posthog", created_by=self.user)diff --git a/products/cdp/backend/services/destination_test_runs.py b/products/cdp/backend/services/destination_test_runs.pynew file mode 100644index 000000000000..b9527571d689--- /dev/null+++ b/products/cdp/backend/services/destination_test_runs.py@@ -0,0 +1,144 @@+"""The activity log record of a hog function test that delivers real requests.++A test with `mock_async_functions` off runs the submitted code with real `fetch` calls and the+credentials of the integrations its inputs name. The record keeps what an investigator needs to+answer who ran which code with which integrations, and never an input value: values can hold+secrets, and the test resolves secret inputs to plaintext before it runs.+"""++import re+from collections.abc import Iterable+from typing import TYPE_CHECKING, Any, Optional++from posthog.hogql.parser import parse_string_template++from posthog.dataclasses import frozen+from posthog.models.activity_logging.activity_log import ActivityContextBase++from products.cdp.backend.validation import collect_inputs++if TYPE_CHECKING:+ from products.cdp.backend.models.hog_functions.hog_function import HogFunction++_CONTEXT_FIELDS = ("hog", "inputs", "inputs_schema", "mappings")+# The worker runs these templates with their own executors, which send real requests even when the test mocks.+_UNMOCKED_TEMPLATE_PREFIXES = ("native-", "segment-")+# Hog templates are parsed, but the parser does not read Liquid, so a Liquid input is matched by pattern.+_LIQUID_INPUT_REFERENCE = re.compile(r"""inputs(?:\.([A-Za-z_]\w*)|\[\s*['"]([^'"]+)['"]\s*\])""")+++# `ActivityDetailEncoder` serializes a context through `__dict__`, which a slotted class leaves empty.+@frozen(slots=False)+class HogFunctionTestRunContext(ActivityContextBase):+ mock_async_functions: bool+ integration_ids: list[int]+ input_keys: list[str]+ mapping_input_keys: list[str]+ # The other inputs that each input's template reads, by input key. An input can read an integration+ # input, so this shows where an integration value can flow.+ input_references: dict[str, list[str]]+ mapping_input_references: dict[str, list[str]]+ hog: Optional[str]+++def _integration_ids_in_value(value: Any) -> Iterable[int]:+ # An `integration_multi` input holds a list. The `{"integrationId": ...}` form matches+ # integration_usage.py, so the record can name an id the worker skips but never miss one.+ if isinstance(value, dict):+ value = value.get("integrationId")+ for candidate in value if isinstance(value, list) else [value]:+ if isinstance(candidate, int) and not isinstance(candidate, bool):+ yield candidate+++def _integration_ids(effective: dict[str, Any], mappings: list[dict[str, Any]]) -> set[int]:+ # Mirrors `loadIntegrationInputs` in nodejs/src/cdp/services/hog-inputs.service.ts. The worker merges+ # the inputs of each mapping, in order, over the top-level inputs, and reads every schema against that+ # merged set. A mapping input can therefore replace the integration that a top-level input names.+ schemas: list[Any] = []+ merged: dict[str, Any] = {}+ for scope in (effective, *mappings):+ if isinstance(scope.get("inputs_schema"), list):+ schemas.extend(scope["inputs_schema"])+ if isinstance(scope.get("inputs"), dict):+ merged.update(scope["inputs"])+ ids: set[int] = set()+ for schema in schemas:+ if not isinstance(schema, dict) or schema.get("type") not in ("integration", "integration_multi"):+ continue+ key = schema.get("key")+ entry = merged.get(key) if isinstance(key, str) else None+ if isinstance(entry, dict):+ ids.update(_integration_ids_in_value(entry.get("value")))+ return ids+++def _referenced_input_keys(value: Any, templating: Any) -> set[str]:+ if isinstance(value, dict):+ return set().union(*(_referenced_input_keys(item, templating) for item in value.values()))+ if isinstance(value, list):+ return set().union(*(_referenced_input_keys(item, templating) for item in value))+ if not isinstance(value, str):+ return set()+ if templating == "liquid":+ return {dotted or bracketed for dotted, bracketed in _LIQUID_INPUT_REFERENCE.findall(value)}+ try:+ return collect_inputs(parse_string_template(value))+ except Exception:+ # Validation already rejected a template that does not parse, so this is a stored row+ # written before that check. The record then lists no references for it.+ return set()+++def _input_references(inputs: Any) -> dict[str, set[str]]:+ if not isinstance(inputs, dict):+ return {}+ references: dict[str, set[str]] = {}+ for key, entry in inputs.items():+ if isinstance(entry, dict):+ if referenced := _referenced_input_keys(entry.get("value"), entry.get("templating")):+ references[key] = referenced+ return references+++def _effective_configuration(configuration: dict[str, Any], stored: Optional["HogFunction"]) -> dict[str, Any]:+ # The worker runs `{...storedFunction, ...configuration}`, so a field the request omits runs as stored.+ if stored is None:+ return configuration+ return {+ field: configuration[field] if field in configuration else getattr(stored, field, None)+ for field in _CONTEXT_FIELDS+ }+++def sends_real_requests(configuration: dict[str, Any], *, mock_async_functions: bool) -> bool:+ template_id = configuration.get("template_id")+ return not mock_async_functions or (+ isinstance(template_id, str) and template_id.startswith(_UNMOCKED_TEMPLATE_PREFIXES)+ )+++def destination_test_run_context(+ configuration: dict[str, Any], *, stored: Optional["HogFunction"], mock_async_functions: bool+) -> HogFunctionTestRunContext:+ effective = _effective_configuration(configuration, stored)+ inputs = effective.get("inputs")+ mappings = [mapping for mapping in effective.get("mappings") or [] if isinstance(mapping, dict)]++ mapping_input_keys: set[str] = set()+ mapping_input_references: dict[str, set[str]] = {}+ for mapping in mappings:+ if isinstance(mapping.get("inputs"), dict):+ mapping_input_keys.update(mapping["inputs"].keys())+ for key, referenced in _input_references(mapping.get("inputs")).items():+ mapping_input_references.setdefault(key, set()).update(referenced)++ return HogFunctionTestRunContext(+ mock_async_functions=mock_async_functions,+ integration_ids=sorted(_integration_ids(effective, mappings)),+ input_keys=sorted(inputs.keys()) if isinstance(inputs, dict) else [],+ mapping_input_keys=sorted(mapping_input_keys),+ input_references={key: sorted(keys) for key, keys in sorted(_input_references(inputs).items())},+ mapping_input_references={key: sorted(keys) for key, keys in sorted(mapping_input_references.items())},+ hog=effective.get("hog"),+ )diff --git a/products/cdp/backend/test/test_destination_test_runs.py b/products/cdp/backend/test/test_destination_test_runs.pynew file mode 100644index 000000000000..431e3159aa5f--- /dev/null+++ b/products/cdp/backend/test/test_destination_test_runs.py@@ -0,0 +1,22 @@+from django.test import SimpleTestCase++from products.cdp.backend.services.destination_test_runs import destination_test_run_context+++class TestDestinationTestRunContext(SimpleTestCase):+ def test_mapping_input_that_replaces_an_integration_input_records_the_replacement(self) -> None:+ configuration = {+ "hog": "return event",+ "inputs_schema": [{"key": "account", "type": "integration", "integration": "slack"}],+ "inputs": {"account": {"value": 5}},+ "mappings": [+ {+ "inputs_schema": [{"key": "account", "type": "number"}],+ "inputs": {"account": {"value": 9}},+ }+ ],+ }++ context = destination_test_run_context(configuration, stored=None, mock_async_functions=False)++ assert context.integration_ids == [9]