Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(workflows): record workflow tests that send real requests (#130743)

Meikel Ratz83d62395d1e6
diff --git a/products/workflows/backend/facade/activity.py b/products/workflows/backend/facade/activity.py
index 1d396990f36e..bdf188921c42 100644
--- a/products/workflows/backend/facade/activity.py
+++ b/products/workflows/backend/facade/activity.py
@@ -6,10 +6,12 @@ from products.workflows.backend.services.hog_flow_activity import (
log_workflow_schedule_activity,
resume_workflow_email_sending,
)
+from products.workflows.backend.services.workflow_test_runs import log_workflow_test_run
__all__ = [
"bulk_delete_archived_workflows",
"log_workflow_activity",
"log_workflow_schedule_activity",
+ "log_workflow_test_run",
"resume_workflow_email_sending",
]
diff --git a/products/workflows/backend/presentation/views/hog_flow.py b/products/workflows/backend/presentation/views/hog_flow.py
index e5899d5659bc..fbefa3607bc0 100644
--- a/products/workflows/backend/presentation/views/hog_flow.py
+++ b/products/workflows/backend/presentation/views/hog_flow.py
@@ -152,6 +152,7 @@ from products.workflows.backend.facade.activity import (
bulk_delete_archived_workflows,
log_workflow_activity,
log_workflow_schedule_activity,
+ log_workflow_test_run,
resume_workflow_email_sending,
)
from products.workflows.backend.facade.api import (
@@ -6644,8 +6645,16 @@ class HogFlowViewSet(
if not payload.get("mock_async_functions", True):
configuration = payload.get("configuration") or {}
- self._require_replay_vision_caller_access(
- request, configuration.get("actions") if configuration else (hog_flow.actions if hog_flow else [])
+ # The worker runs `{...stored, ...configuration}`, so a field the request omits runs as stored.
+ actions = configuration["actions"] if "actions" in configuration else (hog_flow.actions if hog_flow else [])
+ self._require_replay_vision_caller_access(request, actions)
+ log_workflow_test_run(
+ actor=self._activity_actor(),
+ workflow_id=hog_flow.id if hog_flow else None,
+ name=configuration.get("name") or (hog_flow.name if hog_flow else None),
+ actions=actions,
+ current_action_id=payload.get("current_action_id"),
+ encrypted_inputs=hog_flow.encrypted_inputs if hog_flow and "actions" not in configuration else None,
)
res = create_hog_flow_invocation_test(
diff --git a/products/workflows/backend/services/workflow_test_runs.py b/products/workflows/backend/services/workflow_test_runs.py
new file mode 100644
index 000000000000..84323fd59214
--- /dev/null
+++ b/products/workflows/backend/services/workflow_test_runs.py
@@ -0,0 +1,121 @@
+"""The activity log record of a workflow test step that sends real requests.
+
+A test with `mock_async_functions` off runs one step with real side effects and the credentials of the
+integrations its inputs name. The record keeps who ran which step with which integrations, and never an
+input value: values can hold secrets.
+"""
+
+from collections.abc import Iterable
+from typing import Any, Optional
+from uuid import UUID
+
+from posthog.dataclasses import frozen
+from posthog.models.activity_logging.activity_log import ActivityContextBase, Detail, log_activity
+
+from products.cdp.backend.models.hog_function_template import HogFunctionTemplate
+from products.workflows.backend.facade.contracts import WorkflowActor
+
+_INTEGRATION_INPUT_TYPES = ("integration", "integration_multi")
+_EMAIL_INPUT_TYPES = ("native_email", "email")
+
+
+# `ActivityDetailEncoder` serializes a context through `__dict__`, which a slotted class leaves empty.
+@frozen(slots=False)
+class WorkflowTestRunContext(ActivityContextBase):
+ action_id: Optional[str]
+ action_type: Optional[str]
+ template_id: Optional[str]
+ input_keys: list[str]
+ integration_ids: list[int]
+
+
+def _integration_ids_in_value(value: Any) -> Iterable[int]:
+ if isinstance(value, dict):
+ value = value.get("integrationId")
+ for candidate in value if isinstance(value, list) else [value]:
+ if isinstance(candidate, int) and not isinstance(candidate, bool):
+ yield candidate
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+ return value if isinstance(value, dict) else {}
+
+
+def _email_sender_ids(value: Any) -> Iterable[int]:
+ # The email service picks one sender from `integrationIds` when the list is set, else uses `integrationId`.
+ sender = _as_dict(_as_dict(value).get("from"))
+ many = sender.get("integrationIds")
+ return _integration_ids_in_value(many if isinstance(many, list) and many else sender.get("integrationId"))
+
+
+def _integration_ids(template_id: Optional[str], config: dict[str, Any]) -> list[int]:
+ # A function step takes its inputs schema from its template, as the worker does when it loads the step.
+ template = HogFunctionTemplate.get_template(template_id) if template_id else None
+ schemas: list[Any] = list((template.inputs_schema if template else None) or [])
+ inputs = dict(_as_dict(config.get("inputs")))
+ # The worker merges each mapping's inputs over the step inputs before it loads integrations.
+ mappings = config.get("mappings")
+ for mapping in mappings if isinstance(mappings, list) else []:
+ mapping_schema = _as_dict(mapping).get("inputs_schema")
+ schemas.extend(mapping_schema if isinstance(mapping_schema, list) else [])
+ inputs.update(_as_dict(_as_dict(mapping).get("inputs")))
+ ids: set[int] = set()
+ for schema in schemas:
+ if not isinstance(schema, dict):
+ continue
+ value = _as_dict(inputs.get(str(schema.get("key")))).get("value")
+ if schema.get("type") in _INTEGRATION_INPUT_TYPES:
+ ids.update(_integration_ids_in_value(value))
+ elif schema.get("type") in _EMAIL_INPUT_TYPES:
+ ids.update(_email_sender_ids(value))
+ return sorted(ids)
+
+
+def _tested_action(actions: Any, current_action_id: Optional[str]) -> dict[str, Any]:
+ for action in actions if isinstance(actions, list) else []:
+ if not isinstance(action, dict):
+ continue
+ if action.get("id") == current_action_id if current_action_id else action.get("type") == "trigger":
+ return action
+ return {"id": current_action_id}
+
+
+def workflow_test_run_context(
+ actions: Any, current_action_id: Optional[str], encrypted_inputs: Any = None
+) -> WorkflowTestRunContext:
+ action = _tested_action(actions, current_action_id)
+ config = _as_dict(action.get("config"))
+ inputs = _as_dict(config.get("inputs"))
+ # A saved flow keeps its secret inputs apart from `actions`, and the worker folds them back in.
+ secret_keys = _as_dict(_as_dict(encrypted_inputs).get(str(action.get("id")))).keys()
+ template_id = config.get("template_id") if isinstance(config.get("template_id"), str) else None
+ return WorkflowTestRunContext(
+ action_id=action.get("id"),
+ action_type=action.get("type"),
+ template_id=template_id,
+ input_keys=sorted({*inputs.keys(), *secret_keys}),
+ integration_ids=_integration_ids(template_id, config),
+ )
+
+
+def log_workflow_test_run(
+ *,
+ actor: WorkflowActor,
+ workflow_id: Optional[UUID],
+ name: Optional[str],
+ actions: Any,
+ current_action_id: Optional[str],
+ encrypted_inputs: Any = None,
+) -> None:
+ # Without `strict`, production drops a failed write and the test still sends real requests.
+ log_activity(
+ strict=True,
+ organization_id=actor.organization_id,
+ team_id=actor.team_id,
+ user=actor.user,
+ was_impersonated=actor.was_impersonated,
+ item_id=str(workflow_id) if workflow_id else None,
+ scope="HogFlow",
+ activity="tested",
+ detail=Detail(name=name, context=workflow_test_run_context(actions, current_action_id, encrypted_inputs)),
+ )
diff --git a/products/workflows/backend/tests/api/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.py
index 26eaa9ee310a..602ef35de1de 100644
--- a/products/workflows/backend/tests/api/test_hog_flow.py
+++ b/products/workflows/backend/tests/api/test_hog_flow.py
@@ -11,8 +11,9 @@ from uuid import UUID, uuid4
from posthog.test.base import APIBaseTest, ClickhouseTestMixin
from unittest.mock import ANY, MagicMock, patch
+from django.conf import UserSettingsHolder, settings
from django.core.management import call_command
-from django.db import connection
+from django.db import DatabaseError, connection
from django.test import override_settings
from django.utils import timezone
@@ -73,6 +74,7 @@ from products.workflows.backend.presentation.views.hog_flow import (
_should_validate_strictly,
mint_audience_confirm_token,
)
+from products.workflows.backend.services.workflow_test_runs import workflow_test_run_context
from products.workflows.backend.tests.api.test_workflow_data_suggestions import _push_function_template
_AUDIENCE_CONDITION = {"key": "email", "type": "person", "value": "x", "operator": "icontains"}
@@ -3574,6 +3576,135 @@ class TestHogFlowAPI(APIBaseTest):
assert payload["globals"] == {"event": {"event": "$pageview", "distinct_id": "test-distinct-id"}}
assert payload["mock_async_functions"] is True
+ @parameterized.expand(
+ [
+ ("saved_workflow_real_requests", True, False),
+ ("new_workflow_real_requests", False, False),
+ ("mocked_requests", True, True),
+ ]
+ )
+ def test_test_invocation_with_real_requests_is_logged_without_input_values(
+ self, _name: str, saved: bool, mock_async_functions: bool
+ ):
+ workspace = Integration.objects.create(team=self.team, kind="slack", config={}, created_by=self.user)
+ hog_flow, _ = self._create_hog_flow_with_action(
+ {
+ "template_id": "template-slack",
+ "inputs": {
+ "slack_workspace": {"value": workspace.id},
+ "channel": {"value": "C0123456789"},
+ "text": {"value": "sent-by-the-test"},
+ },
+ }
+ )
+ flow_id = "new"
+ data: dict[str, Any] = {
+ "globals": {"event": {"event": "$pageview", "distinct_id": "d"}},
+ "mock_async_functions": mock_async_functions,
+ "current_action_id": "action_1",
+ }
+ if saved:
+ created = self.client.post(f"/api/projects/{self.team.id}/hog_flows", hog_flow)
+ assert created.status_code == 201, created.json()
+ flow_id = created.json()["id"]
+ HogFlow.objects.filter(id=flow_id).update(
+ encrypted_inputs={"action_1": {"api_token": {"value": "stored-secret"}}}
+ )
+ else:
+ data["configuration"] = hog_flow
+
+ with patch(
+ "products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test"
+ ) as mock_invoke:
+ mock_invoke.return_value = MagicMock(status_code=500, json=lambda: {"error": "Worker failed"})
+ self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/invocations/", data, format="json")
+ assert mock_invoke.call_count == 1
+
+ rows = list(ActivityLog.objects.filter(team_id=self.team.id, scope="HogFlow", activity="tested"))
+ if mock_async_functions:
+ assert rows == []
+ return
+ assert len(rows) == 1
+ assert rows[0].user == self.user
+ assert rows[0].item_id == (flow_id if saved else None)
+ detail = rows[0].detail
+ assert detail is not None
+ assert detail["name"] == "Test Flow"
+ assert detail["context"] == {
+ "action_id": "action_1",
+ "action_type": "function",
+ "template_id": "template-slack",
+ "input_keys": ["api_token", "channel", "slack_workspace", "text"]
+ if saved
+ else ["channel", "slack_workspace", "text"],
+ "integration_ids": [workspace.id],
+ }
+ assert "sent-by-the-test" not in json.dumps(detail)
+ assert "stored-secret" not in json.dumps(detail)
+
+ @parameterized.expand(
+ [
+ (
+ "email_single_sender",
+ "template-email",
+ {"inputs": {"email": {"value": {"from": {"integrationId": 7, "email": "[email protected]"}}}}},
+ [7],
+ ),
+ (
+ "email_sender_list_wins",
+ "template-email",
+ {"inputs": {"email": {"value": {"from": {"integrationId": 7, "integrationIds": [8, 9]}}}}},
+ [8, 9],
+ ),
+ (
+ "mapping_overrides_step_input",
+ "template-slack",
+ {
+ "inputs": {"slack_workspace": {"value": 1}},
+ "mappings": [
+ {
+ "inputs_schema": [{"key": "extra", "type": "integration"}],
+ "inputs": {"slack_workspace": {"value": 2}, "extra": {"value": 3}},
+ }
+ ],
+ },
+ [2, 3],
+ ),
+ ]
+ )
+ def test_test_run_context_records_the_integrations_the_worker_loads(
+ self, _name: str, template_id: str, config: dict[str, Any], expected_ids: list[int]
+ ):
+ sync_template_to_db(_email_function_template())
+ actions = [{"id": "action_1", "type": "function", "config": {"template_id": template_id, **config}}]
+ assert workflow_test_run_context(actions, "action_1").integration_ids == expected_ids
+
+ def test_test_invocation_with_real_requests_does_not_run_when_its_activity_log_write_fails(self):
+ hog_flow, _ = self._create_hog_flow_with_action(
+ {"template_id": "template-webhook", "inputs": {"url": {"value": "https://example.com"}}}
+ )
+ production_settings = UserSettingsHolder(settings)
+ production_settings.TEST = False
+ with (
+ patch(
+ "products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test"
+ ) as mock_invoke,
+ patch.object(ActivityLog.objects, "create", side_effect=DatabaseError("insert rejected")),
+ patch("posthog.models.activity_logging.activity_log.settings", production_settings),
+ ):
+ response = self.client.post(
+ f"/api/projects/{self.team.id}/hog_flows/new/invocations/",
+ {
+ "configuration": hog_flow,
+ "globals": {"event": {"event": "$pageview", "distinct_id": "d"}},
+ "mock_async_functions": False,
+ "current_action_id": "action_1",
+ },
+ format="json",
+ )
+ assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
+ mock_invoke.assert_not_called()
+
@parameterized.expand(
[
("cohort_condition_without_person_read", ["hog_flow:write", "group:read"], True, True, 403),
diff --git a/products/workflows/frontend/Workflows/misc/workflowActivityDescriber.tsx b/products/workflows/frontend/Workflows/misc/workflowActivityDescriber.tsx
index b52de2a6448a..450b4cfdc687 100644
--- a/products/workflows/frontend/Workflows/misc/workflowActivityDescriber.tsx
+++ b/products/workflows/frontend/Workflows/misc/workflowActivityDescriber.tsx
@@ -261,6 +261,20 @@ export function workflowActivityDescriber(logItem: ActivityLogItem, asNotificati
}
}
+ if (logItem.activity == 'tested') {
+ const workflowName = nameOrLinkToWorkflow(logItem?.item_id, logItem?.detail.name)
+
+ return {
+ summary: activityLogSummary(logItem, `Tested a ${objectNoun} step with real requests`, workflowName),
+ description: (
+ <>
+ <ActivityLogUserName logItem={logItem} /> tested a step of the {objectNoun} with real requests:{' '}
+ {workflowName}
+ </>
+ ),
+ }
+ }
+
if (logItem.activity == 'revision_restored') {
return {
summary: activityLogSummary(