:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(workflows): record workflow tests that send real requests (#130743)
Meikel Ratz83d62395d1e6
diff --git a/products/workflows/backend/facade/activity.py b/products/workflows/backend/facade/activity.pyindex 1d396990f36e..bdf188921c42 100644--- a/products/workflows/backend/facade/activity.py+++ b/products/workflows/backend/facade/activity.py@@ -6,10 +6,12 @@ from products.workflows.backend.services.hog_flow_activity import (log_workflow_schedule_activity,resume_workflow_email_sending,)+from products.workflows.backend.services.workflow_test_runs import log_workflow_test_run__all__ = ["bulk_delete_archived_workflows","log_workflow_activity","log_workflow_schedule_activity",+ "log_workflow_test_run","resume_workflow_email_sending",]diff --git a/products/workflows/backend/presentation/views/hog_flow.py b/products/workflows/backend/presentation/views/hog_flow.pyindex e5899d5659bc..fbefa3607bc0 100644--- a/products/workflows/backend/presentation/views/hog_flow.py+++ b/products/workflows/backend/presentation/views/hog_flow.py@@ -152,6 +152,7 @@ from products.workflows.backend.facade.activity import (bulk_delete_archived_workflows,log_workflow_activity,log_workflow_schedule_activity,+ log_workflow_test_run,resume_workflow_email_sending,)from products.workflows.backend.facade.api import (@@ -6644,8 +6645,16 @@ class HogFlowViewSet(if not payload.get("mock_async_functions", True):configuration = payload.get("configuration") or {}- self._require_replay_vision_caller_access(- request, configuration.get("actions") if configuration else (hog_flow.actions if hog_flow else [])+ # The worker runs `{...stored, ...configuration}`, so a field the request omits runs as stored.+ actions = configuration["actions"] if "actions" in configuration else (hog_flow.actions if hog_flow else [])+ self._require_replay_vision_caller_access(request, actions)+ log_workflow_test_run(+ actor=self._activity_actor(),+ workflow_id=hog_flow.id if hog_flow else None,+ name=configuration.get("name") or (hog_flow.name if hog_flow else None),+ actions=actions,+ current_action_id=payload.get("current_action_id"),+ encrypted_inputs=hog_flow.encrypted_inputs if hog_flow and "actions" not in configuration else None,)res = create_hog_flow_invocation_test(diff --git a/products/workflows/backend/services/workflow_test_runs.py b/products/workflows/backend/services/workflow_test_runs.pynew file mode 100644index 000000000000..84323fd59214--- /dev/null+++ b/products/workflows/backend/services/workflow_test_runs.py@@ -0,0 +1,121 @@+"""The activity log record of a workflow test step that sends real requests.++A test with `mock_async_functions` off runs one step with real side effects and the credentials of the+integrations its inputs name. The record keeps who ran which step with which integrations, and never an+input value: values can hold secrets.+"""++from collections.abc import Iterable+from typing import Any, Optional+from uuid import UUID++from posthog.dataclasses import frozen+from posthog.models.activity_logging.activity_log import ActivityContextBase, Detail, log_activity++from products.cdp.backend.models.hog_function_template import HogFunctionTemplate+from products.workflows.backend.facade.contracts import WorkflowActor++_INTEGRATION_INPUT_TYPES = ("integration", "integration_multi")+_EMAIL_INPUT_TYPES = ("native_email", "email")+++# `ActivityDetailEncoder` serializes a context through `__dict__`, which a slotted class leaves empty.+@frozen(slots=False)+class WorkflowTestRunContext(ActivityContextBase):+ action_id: Optional[str]+ action_type: Optional[str]+ template_id: Optional[str]+ input_keys: list[str]+ integration_ids: list[int]+++def _integration_ids_in_value(value: Any) -> Iterable[int]:+ if isinstance(value, dict):+ value = value.get("integrationId")+ for candidate in value if isinstance(value, list) else [value]:+ if isinstance(candidate, int) and not isinstance(candidate, bool):+ yield candidate+++def _as_dict(value: Any) -> dict[str, Any]:+ return value if isinstance(value, dict) else {}+++def _email_sender_ids(value: Any) -> Iterable[int]:+ # The email service picks one sender from `integrationIds` when the list is set, else uses `integrationId`.+ sender = _as_dict(_as_dict(value).get("from"))+ many = sender.get("integrationIds")+ return _integration_ids_in_value(many if isinstance(many, list) and many else sender.get("integrationId"))+++def _integration_ids(template_id: Optional[str], config: dict[str, Any]) -> list[int]:+ # A function step takes its inputs schema from its template, as the worker does when it loads the step.+ template = HogFunctionTemplate.get_template(template_id) if template_id else None+ schemas: list[Any] = list((template.inputs_schema if template else None) or [])+ inputs = dict(_as_dict(config.get("inputs")))+ # The worker merges each mapping's inputs over the step inputs before it loads integrations.+ mappings = config.get("mappings")+ for mapping in mappings if isinstance(mappings, list) else []:+ mapping_schema = _as_dict(mapping).get("inputs_schema")+ schemas.extend(mapping_schema if isinstance(mapping_schema, list) else [])+ inputs.update(_as_dict(_as_dict(mapping).get("inputs")))+ ids: set[int] = set()+ for schema in schemas:+ if not isinstance(schema, dict):+ continue+ value = _as_dict(inputs.get(str(schema.get("key")))).get("value")+ if schema.get("type") in _INTEGRATION_INPUT_TYPES:+ ids.update(_integration_ids_in_value(value))+ elif schema.get("type") in _EMAIL_INPUT_TYPES:+ ids.update(_email_sender_ids(value))+ return sorted(ids)+++def _tested_action(actions: Any, current_action_id: Optional[str]) -> dict[str, Any]:+ for action in actions if isinstance(actions, list) else []:+ if not isinstance(action, dict):+ continue+ if action.get("id") == current_action_id if current_action_id else action.get("type") == "trigger":+ return action+ return {"id": current_action_id}+++def workflow_test_run_context(+ actions: Any, current_action_id: Optional[str], encrypted_inputs: Any = None+) -> WorkflowTestRunContext:+ action = _tested_action(actions, current_action_id)+ config = _as_dict(action.get("config"))+ inputs = _as_dict(config.get("inputs"))+ # A saved flow keeps its secret inputs apart from `actions`, and the worker folds them back in.+ secret_keys = _as_dict(_as_dict(encrypted_inputs).get(str(action.get("id")))).keys()+ template_id = config.get("template_id") if isinstance(config.get("template_id"), str) else None+ return WorkflowTestRunContext(+ action_id=action.get("id"),+ action_type=action.get("type"),+ template_id=template_id,+ input_keys=sorted({*inputs.keys(), *secret_keys}),+ integration_ids=_integration_ids(template_id, config),+ )+++def log_workflow_test_run(+ *,+ actor: WorkflowActor,+ workflow_id: Optional[UUID],+ name: Optional[str],+ actions: Any,+ current_action_id: Optional[str],+ encrypted_inputs: Any = None,+) -> None:+ # Without `strict`, production drops a failed write and the test still sends real requests.+ log_activity(+ strict=True,+ organization_id=actor.organization_id,+ team_id=actor.team_id,+ user=actor.user,+ was_impersonated=actor.was_impersonated,+ item_id=str(workflow_id) if workflow_id else None,+ scope="HogFlow",+ activity="tested",+ detail=Detail(name=name, context=workflow_test_run_context(actions, current_action_id, encrypted_inputs)),+ )diff --git a/products/workflows/backend/tests/api/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.pyindex 26eaa9ee310a..602ef35de1de 100644--- a/products/workflows/backend/tests/api/test_hog_flow.py+++ b/products/workflows/backend/tests/api/test_hog_flow.py@@ -11,8 +11,9 @@ from uuid import UUID, uuid4from posthog.test.base import APIBaseTest, ClickhouseTestMixinfrom unittest.mock import ANY, MagicMock, patch+from django.conf import UserSettingsHolder, settingsfrom django.core.management import call_command-from django.db import connection+from django.db import DatabaseError, connectionfrom django.test import override_settingsfrom django.utils import timezone@@ -73,6 +74,7 @@ from products.workflows.backend.presentation.views.hog_flow import (_should_validate_strictly,mint_audience_confirm_token,)+from products.workflows.backend.services.workflow_test_runs import workflow_test_run_contextfrom products.workflows.backend.tests.api.test_workflow_data_suggestions import _push_function_template_AUDIENCE_CONDITION = {"key": "email", "type": "person", "value": "x", "operator": "icontains"}@@ -3574,6 +3576,135 @@ class TestHogFlowAPI(APIBaseTest):assert payload["globals"] == {"event": {"event": "$pageview", "distinct_id": "test-distinct-id"}}assert payload["mock_async_functions"] is True+ @parameterized.expand(+ [+ ("saved_workflow_real_requests", True, False),+ ("new_workflow_real_requests", False, False),+ ("mocked_requests", True, True),+ ]+ )+ def test_test_invocation_with_real_requests_is_logged_without_input_values(+ self, _name: str, saved: bool, mock_async_functions: bool+ ):+ workspace = Integration.objects.create(team=self.team, kind="slack", config={}, created_by=self.user)+ hog_flow, _ = self._create_hog_flow_with_action(+ {+ "template_id": "template-slack",+ "inputs": {+ "slack_workspace": {"value": workspace.id},+ "channel": {"value": "C0123456789"},+ "text": {"value": "sent-by-the-test"},+ },+ }+ )+ flow_id = "new"+ data: dict[str, Any] = {+ "globals": {"event": {"event": "$pageview", "distinct_id": "d"}},+ "mock_async_functions": mock_async_functions,+ "current_action_id": "action_1",+ }+ if saved:+ created = self.client.post(f"/api/projects/{self.team.id}/hog_flows", hog_flow)+ assert created.status_code == 201, created.json()+ flow_id = created.json()["id"]+ HogFlow.objects.filter(id=flow_id).update(+ encrypted_inputs={"action_1": {"api_token": {"value": "stored-secret"}}}+ )+ else:+ data["configuration"] = hog_flow++ with patch(+ "products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test"+ ) as mock_invoke:+ mock_invoke.return_value = MagicMock(status_code=500, json=lambda: {"error": "Worker failed"})+ self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/invocations/", data, format="json")+ assert mock_invoke.call_count == 1++ rows = list(ActivityLog.objects.filter(team_id=self.team.id, scope="HogFlow", activity="tested"))+ if mock_async_functions:+ assert rows == []+ return+ assert len(rows) == 1+ assert rows[0].user == self.user+ assert rows[0].item_id == (flow_id if saved else None)+ detail = rows[0].detail+ assert detail is not None+ assert detail["name"] == "Test Flow"+ assert detail["context"] == {+ "action_id": "action_1",+ "action_type": "function",+ "template_id": "template-slack",+ "input_keys": ["api_token", "channel", "slack_workspace", "text"]+ if saved+ else ["channel", "slack_workspace", "text"],+ "integration_ids": [workspace.id],+ }+ assert "sent-by-the-test" not in json.dumps(detail)+ assert "stored-secret" not in json.dumps(detail)++ @parameterized.expand(+ [+ (+ "email_single_sender",+ "template-email",+ {"inputs": {"email": {"value": {"from": {"integrationId": 7, "email": "[email protected]"}}}}},+ [7],+ ),+ (+ "email_sender_list_wins",+ "template-email",+ {"inputs": {"email": {"value": {"from": {"integrationId": 7, "integrationIds": [8, 9]}}}}},+ [8, 9],+ ),+ (+ "mapping_overrides_step_input",+ "template-slack",+ {+ "inputs": {"slack_workspace": {"value": 1}},+ "mappings": [+ {+ "inputs_schema": [{"key": "extra", "type": "integration"}],+ "inputs": {"slack_workspace": {"value": 2}, "extra": {"value": 3}},+ }+ ],+ },+ [2, 3],+ ),+ ]+ )+ def test_test_run_context_records_the_integrations_the_worker_loads(+ self, _name: str, template_id: str, config: dict[str, Any], expected_ids: list[int]+ ):+ sync_template_to_db(_email_function_template())+ actions = [{"id": "action_1", "type": "function", "config": {"template_id": template_id, **config}}]+ assert workflow_test_run_context(actions, "action_1").integration_ids == expected_ids++ def test_test_invocation_with_real_requests_does_not_run_when_its_activity_log_write_fails(self):+ hog_flow, _ = self._create_hog_flow_with_action(+ {"template_id": "template-webhook", "inputs": {"url": {"value": "https://example.com"}}}+ )+ production_settings = UserSettingsHolder(settings)+ production_settings.TEST = False+ with (+ patch(+ "products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test"+ ) as mock_invoke,+ patch.object(ActivityLog.objects, "create", side_effect=DatabaseError("insert rejected")),+ patch("posthog.models.activity_logging.activity_log.settings", production_settings),+ ):+ response = self.client.post(+ f"/api/projects/{self.team.id}/hog_flows/new/invocations/",+ {+ "configuration": hog_flow,+ "globals": {"event": {"event": "$pageview", "distinct_id": "d"}},+ "mock_async_functions": False,+ "current_action_id": "action_1",+ },+ format="json",+ )+ assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR+ mock_invoke.assert_not_called()+@parameterized.expand([("cohort_condition_without_person_read", ["hog_flow:write", "group:read"], True, True, 403),diff --git a/products/workflows/frontend/Workflows/misc/workflowActivityDescriber.tsx b/products/workflows/frontend/Workflows/misc/workflowActivityDescriber.tsxindex b52de2a6448a..450b4cfdc687 100644--- a/products/workflows/frontend/Workflows/misc/workflowActivityDescriber.tsx+++ b/products/workflows/frontend/Workflows/misc/workflowActivityDescriber.tsx@@ -261,6 +261,20 @@ export function workflowActivityDescriber(logItem: ActivityLogItem, asNotificati}}+ if (logItem.activity == 'tested') {+ const workflowName = nameOrLinkToWorkflow(logItem?.item_id, logItem?.detail.name)++ return {+ summary: activityLogSummary(logItem, `Tested a ${objectNoun} step with real requests`, workflowName),+ description: (+ <>+ <ActivityLogUserName logItem={logItem} /> tested a step of the {objectNoun} with real requests:{' '}+ {workflowName}+ </>+ ),+ }+ }+if (logItem.activity == 'revision_restored') {return {summary: activityLogSummary(