Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(activity-log): record destination tests that send real requests (#127963)

Tom Piccirello44a022574298
diff --git a/docs/internal/activity-logging.md b/docs/internal/activity-logging.md
index 25f5defcf4a1..14d7a2cfc7da 100644
--- a/docs/internal/activity-logging.md
+++ b/docs/internal/activity-logging.md
@@ -134,6 +134,12 @@ Never put a credential field in that branch.
Destination activity logs record changes to `mappings` without their values.
For `inputs` they keep each key and record only whether its value changed, so the history can name the changed input.
+A destination test with `mock_async_functions` set to false writes a `tested` row before the test runs, so a failed test is recorded too.
+If that row cannot be written, the test does not run.
+A test of a native or Segment destination writes the row whatever `mock_async_functions` says, because those executors send real requests even when the test mocks.
+Its context holds the `hog` source, the input and mapping input keys, and the ids of the integrations the inputs name, never an input value.
+A test of an unsaved function has no `item_id`.
+Any other mocked test writes no row, because it cannot send data out.
Rows written before a field joined `field_with_masked_contents` still store its values.
Guided Inbox onboarding first saves a draft when the sources step is completed; opening or editing that first step keeps the draft local.
Later setup changes are recorded while masking the saved draft, including scout instructions and project evidence.
diff --git a/frontend/src/scenes/hog-functions/misc/activityDescriptions.tsx b/frontend/src/scenes/hog-functions/misc/activityDescriptions.tsx
index cc2bbe5b4f92..6ea4e87def83 100644
--- a/frontend/src/scenes/hog-functions/misc/activityDescriptions.tsx
+++ b/frontend/src/scenes/hog-functions/misc/activityDescriptions.tsx
@@ -275,6 +275,19 @@ export function hogFunctionActivityDescriber(logItem: ActivityLogItem, asNotific
}
}
+ if (logItem.activity == 'tested') {
+ const functionName = nameOrLinkToHogFunction(logItem?.item_id, logItem?.detail.name)
+
+ return {
+ summary: activityLogSummary(logItem, `Tested the ${objectNoun} with real requests`, functionName),
+ description: (
+ <>
+ <ActivityLogUserName logItem={logItem} /> tested the {objectNoun} with real requests: {functionName}
+ </>
+ ),
+ }
+ }
+
const draftActivities: Record<string, string> = {
draft_updated: 'staged changes for review on',
published: 'published the staged changes to',
diff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.py
index 7c5c1492ffb2..bc8f4e2e81e2 100644
--- a/posthog/models/activity_logging/activity_log.py
+++ b/posthog/models/activity_logging/activity_log.py
@@ -1570,7 +1570,7 @@ def log_activity(
"user": user,
"was_impersonated": was_impersonated,
"is_system": user is None,
- "item_id": str(item_id),
+ "item_id": str(item_id) if item_id is not None else None,
"scope": scope,
"activity": activity,
"detail": detail,
diff --git a/products/cdp/backend/api/hog_function.py b/products/cdp/backend/api/hog_function.py
index cdc5168c91c6..eb36c60b6328 100644
--- a/products/cdp/backend/api/hog_function.py
+++ b/products/cdp/backend/api/hog_function.py
@@ -70,6 +70,7 @@ from products.cdp.backend.presentation.hog_invocations.rerun import (
HogInvocationRerunRequestSerializer,
HogInvocationRerunResponseSerializer,
)
+from products.cdp.backend.services.destination_test_runs import destination_test_run_context, sends_real_requests
from products.cdp.backend.services.icons import CDPIconsService
from products.cdp.backend.site_functions import get_transpiled_function
from products.cdp.backend.validation import (
@@ -1448,6 +1449,28 @@ class HogFunctionViewSet(
# Remove the team from the config
configuration.pop("team")
+ mock_async_functions = serializer.validated_data["mock_async_functions"]
+ if sends_real_requests(configuration, mock_async_functions=mock_async_functions):
+ # The row is written before the worker call, so a test that then fails is still recorded.
+ # Without `strict`, production drops a failed write and the test still sends real requests.
+ log_activity(
+ strict=True,
+ organization_id=self.organization.id,
+ team_id=self.team_id,
+ user=cast(User, request.user),
+ was_impersonated=is_impersonated(request),
+ item_id=str(hog_function.id) if hog_function else None,
+ scope="HogFunction",
+ activity="tested",
+ detail=Detail(
+ name=configuration.get("name") or (hog_function.name if hog_function else None),
+ type=humanize_hog_function_type(configuration.get("type")),
+ context=destination_test_run_context(
+ configuration, stored=hog_function, mock_async_functions=mock_async_functions
+ ),
+ ),
+ )
+
res = create_hog_invocation_test(
team_id=self.team_id,
hog_function_id=str(hog_function.id) if hog_function else "new",
diff --git a/products/cdp/backend/api/test/test_hog_function.py b/products/cdp/backend/api/test/test_hog_function.py
index a1d88c9923cf..5e3b146d3486 100644
--- a/products/cdp/backend/api/test/test_hog_function.py
+++ b/products/cdp/backend/api/test/test_hog_function.py
@@ -6,12 +6,14 @@ import time_machine
from posthog.test.base import APIBaseTest, ClickhouseTestMixin, QueryMatchingTest
from unittest.mock import ANY, MagicMock, patch
-from django.db import connection
+from django.conf import UserSettingsHolder, settings
+from django.db import DatabaseError, connection
from django.test.utils import CaptureQueriesContext
from parameterized import parameterized
from rest_framework import status
+from posthog.models.activity_logging.activity_log import ActivityLog
from posthog.models.integration import Integration
from products.access_control.backend.models.access_control import AccessControl
@@ -2780,6 +2782,126 @@ class TestHogFunctionAPI(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):
"value": "http://localhost:2080/0e02d917-563f-4050-9725-aad881b69937",
}
+ @parameterized.expand(
+ [
+ ("saved_function_real_requests", True, False, False, None),
+ ("saved_function_runs_its_stored_config", True, False, True, None),
+ ("new_function_real_requests", False, False, False, None),
+ ("new_function_mocked_requests", False, True, False, None),
+ ("native_destination_ignores_mocking", False, True, False, "native-example"),
+ ("segment_destination_ignores_mocking", True, True, False, "segment-example"),
+ ]
+ )
+ def test_test_invocation_with_real_requests_is_logged_without_input_values(
+ self,
+ _name: str,
+ saved: bool,
+ mock_async_functions: bool,
+ omit_stored_fields: bool,
+ template_id: Optional[str],
+ ):
+ integration = Integration.objects.create(team=self.team, kind="slack", config={}, created_by=self.user)
+ mapping_integration = Integration.objects.create(team=self.team, kind="slack", config={}, created_by=self.user)
+ configuration = {
+ **EXAMPLE_FULL,
+ "inputs_schema": [
+ *EXAMPLE_FULL["inputs_schema"],
+ {"key": "workspace", "type": "integration", "integration": "slack"},
+ ],
+ "inputs": {
+ **EXAMPLE_FULL["inputs"],
+ "workspace": {"value": integration.id},
+ "payload": {"value": {"event": "{event}", "team": "{inputs.workspace.team}"}},
+ },
+ "mappings": [
+ {
+ "name": "Pageviews",
+ "inputs_schema": [
+ {"key": "mapping_workspace", "type": "integration", "integration": "slack"},
+ {"key": "mapping_text", "type": "string"},
+ ],
+ "inputs": {
+ "mapping_workspace": {"value": mapping_integration.id},
+ "mapping_text": {"value": "{{ inputs.mapping_workspace.team }}", "templating": "liquid"},
+ },
+ }
+ ],
+ }
+ if template_id:
+ HogFunctionTemplate.objects.create(
+ template_id=template_id,
+ sha="1.0.0",
+ name="Example",
+ description="Example destination",
+ code="return event",
+ code_language="hog",
+ inputs_schema=[],
+ type="destination",
+ status="stable",
+ category=["Other"],
+ free=True,
+ )
+ configuration["template_id"] = template_id
+ function_id = "new"
+ if saved:
+ created = self.client.post(f"/api/projects/{self.team.id}/hog_functions/", data=configuration)
+ assert created.status_code == status.HTTP_201_CREATED, created.json()
+ function_id = created.json()["id"]
+
+ tested_configuration = (
+ {
+ key: value
+ for key, value in configuration.items()
+ if key not in ("hog", "inputs", "inputs_schema", "mappings")
+ }
+ if omit_stored_fields
+ else configuration
+ )
+ with patch("products.cdp.backend.api.hog_function.create_hog_invocation_test") as mock_invocation:
+ mock_invocation.return_value = MagicMock(status_code=500, json=lambda: {"errors": ["Worker failed"]})
+ response = self.client.post(
+ f"/api/projects/{self.team.id}/hog_functions/{function_id}/invocations/",
+ data={"configuration": tested_configuration, "mock_async_functions": mock_async_functions},
+ )
+ assert mock_invocation.call_count == 1, response.json()
+
+ rows = list(ActivityLog.objects.filter(team_id=self.team.id, scope="HogFunction", activity="tested"))
+ if mock_async_functions and not template_id:
+ assert rows == []
+ return
+ assert len(rows) == 1
+ assert rows[0].user == self.user
+ assert rows[0].item_id == (function_id if saved else None)
+ detail = rows[0].detail
+ assert detail is not None
+ assert detail["context"] == {
+ "mock_async_functions": mock_async_functions,
+ "integration_ids": sorted([integration.id, mapping_integration.id]),
+ "input_keys": ["headers", "method", "payload", "url", "workspace"],
+ "mapping_input_keys": ["mapping_text", "mapping_workspace"],
+ "input_references": {"payload": ["workspace"]},
+ "mapping_input_references": {"mapping_text": ["mapping_workspace"]},
+ "hog": EXAMPLE_FULL["hog"],
+ }
+ assert "0e02d917-563f-4050-9725-aad881b69937" not in json.dumps(detail)
+
+ def test_test_invocation_with_real_requests_does_not_run_when_its_activity_log_write_fails(self):
+ # Under `settings.TEST`, `log_activity` raises on every failed write, strict or not. The override
+ # stays in its module, because route modules read `settings.TEST` when the URL conf loads.
+ production_settings = UserSettingsHolder(settings)
+ production_settings.TEST = False
+ with (
+ patch("products.cdp.backend.api.hog_function.create_hog_invocation_test") as mock_invocation,
+ patch.object(ActivityLog.objects, "create", side_effect=DatabaseError("insert rejected")),
+ patch("posthog.models.activity_logging.activity_log.settings", production_settings),
+ ):
+ response = self.client.post(
+ f"/api/projects/{self.team.id}/hog_functions/new/invocations/",
+ data={"configuration": EXAMPLE_FULL, "mock_async_functions": False},
+ )
+ assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
+ mock_invocation.assert_not_called()
+
def test_test_invocation_rejects_a_posthog_connection_input(self):
connection = Integration.objects.create(team=self.team, kind="posthog", created_by=self.user)
diff --git a/products/cdp/backend/services/destination_test_runs.py b/products/cdp/backend/services/destination_test_runs.py
new file mode 100644
index 000000000000..b9527571d689
--- /dev/null
+++ b/products/cdp/backend/services/destination_test_runs.py
@@ -0,0 +1,144 @@
+"""The activity log record of a hog function test that delivers real requests.
+
+A test with `mock_async_functions` off runs the submitted code with real `fetch` calls and the
+credentials of the integrations its inputs name. The record keeps what an investigator needs to
+answer who ran which code with which integrations, and never an input value: values can hold
+secrets, and the test resolves secret inputs to plaintext before it runs.
+"""
+
+import re
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any, Optional
+
+from posthog.hogql.parser import parse_string_template
+
+from posthog.dataclasses import frozen
+from posthog.models.activity_logging.activity_log import ActivityContextBase
+
+from products.cdp.backend.validation import collect_inputs
+
+if TYPE_CHECKING:
+ from products.cdp.backend.models.hog_functions.hog_function import HogFunction
+
+_CONTEXT_FIELDS = ("hog", "inputs", "inputs_schema", "mappings")
+# The worker runs these templates with their own executors, which send real requests even when the test mocks.
+_UNMOCKED_TEMPLATE_PREFIXES = ("native-", "segment-")
+# Hog templates are parsed, but the parser does not read Liquid, so a Liquid input is matched by pattern.
+_LIQUID_INPUT_REFERENCE = re.compile(r"""inputs(?:\.([A-Za-z_]\w*)|\[\s*['"]([^'"]+)['"]\s*\])""")
+
+
+# `ActivityDetailEncoder` serializes a context through `__dict__`, which a slotted class leaves empty.
+@frozen(slots=False)
+class HogFunctionTestRunContext(ActivityContextBase):
+ mock_async_functions: bool
+ integration_ids: list[int]
+ input_keys: list[str]
+ mapping_input_keys: list[str]
+ # The other inputs that each input's template reads, by input key. An input can read an integration
+ # input, so this shows where an integration value can flow.
+ input_references: dict[str, list[str]]
+ mapping_input_references: dict[str, list[str]]
+ hog: Optional[str]
+
+
+def _integration_ids_in_value(value: Any) -> Iterable[int]:
+ # An `integration_multi` input holds a list. The `{"integrationId": ...}` form matches
+ # integration_usage.py, so the record can name an id the worker skips but never miss one.
+ if isinstance(value, dict):
+ value = value.get("integrationId")
+ for candidate in value if isinstance(value, list) else [value]:
+ if isinstance(candidate, int) and not isinstance(candidate, bool):
+ yield candidate
+
+
+def _integration_ids(effective: dict[str, Any], mappings: list[dict[str, Any]]) -> set[int]:
+ # Mirrors `loadIntegrationInputs` in nodejs/src/cdp/services/hog-inputs.service.ts. The worker merges
+ # the inputs of each mapping, in order, over the top-level inputs, and reads every schema against that
+ # merged set. A mapping input can therefore replace the integration that a top-level input names.
+ schemas: list[Any] = []
+ merged: dict[str, Any] = {}
+ for scope in (effective, *mappings):
+ if isinstance(scope.get("inputs_schema"), list):
+ schemas.extend(scope["inputs_schema"])
+ if isinstance(scope.get("inputs"), dict):
+ merged.update(scope["inputs"])
+ ids: set[int] = set()
+ for schema in schemas:
+ if not isinstance(schema, dict) or schema.get("type") not in ("integration", "integration_multi"):
+ continue
+ key = schema.get("key")
+ entry = merged.get(key) if isinstance(key, str) else None
+ if isinstance(entry, dict):
+ ids.update(_integration_ids_in_value(entry.get("value")))
+ return ids
+
+
+def _referenced_input_keys(value: Any, templating: Any) -> set[str]:
+ if isinstance(value, dict):
+ return set().union(*(_referenced_input_keys(item, templating) for item in value.values()))
+ if isinstance(value, list):
+ return set().union(*(_referenced_input_keys(item, templating) for item in value))
+ if not isinstance(value, str):
+ return set()
+ if templating == "liquid":
+ return {dotted or bracketed for dotted, bracketed in _LIQUID_INPUT_REFERENCE.findall(value)}
+ try:
+ return collect_inputs(parse_string_template(value))
+ except Exception:
+ # Validation already rejected a template that does not parse, so this is a stored row
+ # written before that check. The record then lists no references for it.
+ return set()
+
+
+def _input_references(inputs: Any) -> dict[str, set[str]]:
+ if not isinstance(inputs, dict):
+ return {}
+ references: dict[str, set[str]] = {}
+ for key, entry in inputs.items():
+ if isinstance(entry, dict):
+ if referenced := _referenced_input_keys(entry.get("value"), entry.get("templating")):
+ references[key] = referenced
+ return references
+
+
+def _effective_configuration(configuration: dict[str, Any], stored: Optional["HogFunction"]) -> dict[str, Any]:
+ # The worker runs `{...storedFunction, ...configuration}`, so a field the request omits runs as stored.
+ if stored is None:
+ return configuration
+ return {
+ field: configuration[field] if field in configuration else getattr(stored, field, None)
+ for field in _CONTEXT_FIELDS
+ }
+
+
+def sends_real_requests(configuration: dict[str, Any], *, mock_async_functions: bool) -> bool:
+ template_id = configuration.get("template_id")
+ return not mock_async_functions or (
+ isinstance(template_id, str) and template_id.startswith(_UNMOCKED_TEMPLATE_PREFIXES)
+ )
+
+
+def destination_test_run_context(
+ configuration: dict[str, Any], *, stored: Optional["HogFunction"], mock_async_functions: bool
+) -> HogFunctionTestRunContext:
+ effective = _effective_configuration(configuration, stored)
+ inputs = effective.get("inputs")
+ mappings = [mapping for mapping in effective.get("mappings") or [] if isinstance(mapping, dict)]
+
+ mapping_input_keys: set[str] = set()
+ mapping_input_references: dict[str, set[str]] = {}
+ for mapping in mappings:
+ if isinstance(mapping.get("inputs"), dict):
+ mapping_input_keys.update(mapping["inputs"].keys())
+ for key, referenced in _input_references(mapping.get("inputs")).items():
+ mapping_input_references.setdefault(key, set()).update(referenced)
+
+ return HogFunctionTestRunContext(
+ mock_async_functions=mock_async_functions,
+ integration_ids=sorted(_integration_ids(effective, mappings)),
+ input_keys=sorted(inputs.keys()) if isinstance(inputs, dict) else [],
+ mapping_input_keys=sorted(mapping_input_keys),
+ input_references={key: sorted(keys) for key, keys in sorted(_input_references(inputs).items())},
+ mapping_input_references={key: sorted(keys) for key, keys in sorted(mapping_input_references.items())},
+ hog=effective.get("hog"),
+ )
diff --git a/products/cdp/backend/test/test_destination_test_runs.py b/products/cdp/backend/test/test_destination_test_runs.py
new file mode 100644
index 000000000000..431e3159aa5f
--- /dev/null
+++ b/products/cdp/backend/test/test_destination_test_runs.py
@@ -0,0 +1,22 @@
+from django.test import SimpleTestCase
+
+from products.cdp.backend.services.destination_test_runs import destination_test_run_context
+
+
+class TestDestinationTestRunContext(SimpleTestCase):
+ def test_mapping_input_that_replaces_an_integration_input_records_the_replacement(self) -> None:
+ configuration = {
+ "hog": "return event",
+ "inputs_schema": [{"key": "account", "type": "integration", "integration": "slack"}],
+ "inputs": {"account": {"value": 5}},
+ "mappings": [
+ {
+ "inputs_schema": [{"key": "account", "type": "number"}],
+ "inputs": {"account": {"value": 9}},
+ }
+ ],
+ }
+
+ context = destination_test_run_context(configuration, stored=None, mock_async_functions=False)
+
+ assert context.integration_ids == [9]