:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(weekly-digest): encode the workflows webhook body before posting (#127219)
Harley Alexandercc83ffc50550
diff --git a/posthog/temporal/tests/weekly_digest/test_activities.py b/posthog/temporal/tests/weekly_digest/test_activities.pyindex 2cbe3e255c6b..962462ef5960 100644--- a/posthog/temporal/tests/weekly_digest/test_activities.py+++ b/posthog/temporal/tests/weekly_digest/test_activities.py@@ -766,11 +766,14 @@ def test_send_weekly_digest_batch_emails_subscribed_members_once(assert post.args == (WEBHOOK_SETTINGS["WEEKLY_DIGEST_WORKFLOW_WEBHOOK_URL"],)assert post.kwargs["headers"] == {"Authorization": "Bearer test-webhook-secret",- "Idempotency-Key": f"{digest.key}:{subscribed.id}",+ "Content-Type": "application/json",+ "Idempotency-Key": f"{digest.key}:{subscribed.uuid}",}- assert post.kwargs["json"]["event"] == "weekly digest ready"- assert post.kwargs["json"]["distinct_id"] == (subscribed.distinct_id or subscribed.email)- properties = post.kwargs["json"]["properties"]+ # The digest holds UUIDs and datetimes, so the body must be encoded before it reaches requests.+ body = json.loads(post.kwargs["data"])+ assert body["event"] == "weekly digest ready"+ assert body["distinct_id"] == (subscribed.distinct_id or subscribed.email)+ properties = body["properties"]assert properties["to"] == subscribed.email(team_payload,) = properties["teams"]assert team_payload["team_id"] == team.id@@ -843,6 +846,7 @@ def test_send_weekly_digest_batch_stops_sending_once_the_activity_is_cancelled(("queue_full", 1),("oversized", 0),("failed_webhook", 0),+ ("unexpected_webhook_error", 0),],)def test_send_weekly_digest_batch_leaves_organization_unsent_when_an_event_is_dropped(@@ -868,6 +872,8 @@ def test_send_weekly_digest_batch_leaves_organization_unsent_when_an_event_is_drsession = MagicMock()session.post.return_value.status_code = 503+ if drop == "unexpected_webhook_error":+ session.post.side_effect = TypeError("Object of type UUID is not JSON serializable")with (override_settings(**WEBHOOK_SETTINGS),@@ -876,7 +882,10 @@ def test_send_weekly_digest_batch_leaves_organization_unsent_when_an_event_is_dr"posthog.temporal.weekly_digest.activities.DIGEST_PAYLOAD_SIZE_LIMIT",100 if drop == "oversized" else DIGEST_PAYLOAD_SIZE_LIMIT,),- patch("posthog.temporal.weekly_digest.activities._routes_to_workflows", return_value=drop == "failed_webhook"),+ patch(+ "posthog.temporal.weekly_digest.activities._routes_to_workflows",+ return_value=drop in ("failed_webhook", "unexpected_webhook_error"),+ ),patch("posthog.temporal.weekly_digest.activities.requests.Session", return_value=session),patch("posthog.temporal.weekly_digest.activities.time.sleep"),):diff --git a/posthog/temporal/weekly_digest/activities.py b/posthog/temporal/weekly_digest/activities.pyindex 054a1255243c..f2b064f27542 100644--- a/posthog/temporal/weekly_digest/activities.py+++ b/posthog/temporal/weekly_digest/activities.py@@ -9,6 +9,7 @@ from typing import Any, ParamSpec, Protocol, TypeVarfrom uuid import UUIDfrom django.conf import settings+from django.core.serializers.json import DjangoJSONEncoderfrom django.db.models import F, QuerySet, Windowfrom django.db.models.functions import RowNumberfrom django.utils import timezone@@ -1114,11 +1115,12 @@ def _record_workflows_send_failure(reason: str) -> None:).add(1)-def _post_to_workflows(session: requests.Session, body: dict[str, Any], idempotency_key: str) -> None:+def _post_to_workflows(session: requests.Session, body: bytes, idempotency_key: str) -> None:# The secret authenticates the post, so only the digest send can start the workflow. The idempotency key# makes a retried post, in this attempt or the next one, start no second run.headers = {"Authorization": f"Bearer {settings.WEEKLY_DIGEST_WORKFLOW_WEBHOOK_SECRET}",+ "Content-Type": "application/json","Idempotency-Key": idempotency_key,}reason = "connection_error"@@ -1129,7 +1131,7 @@ def _post_to_workflows(session: requests.Session, body: dict[str, Any], idempotetry:response = session.post(settings.WEEKLY_DIGEST_WORKFLOW_WEBHOOK_URL,- json=body,+ data=body,headers=headers,timeout=WORKFLOWS_WEBHOOK_TIMEOUT_SECONDS,)@@ -1277,22 +1279,30 @@ def _send_weekly_digest_batch(input: SendWeeklyDigestBatchInput) -> None:"distinct_id": _workflows_distinct_id(user),"properties": {**workflows_payload, "to": user.email},}- if len(json.dumps(body, default=str).encode()) > WORKFLOWS_WEBHOOK_BODY_LIMIT:- logger.error(- "Digest exceeds the Workflows webhook size limit, leaving the organization unsent",- organization_id=organization.id,- user_id=user.id,- )- _record_workflows_send_failure("oversized")- failed_organization_ids.add(str(organization.id))- continue- _wait_for_workflows_post(r, input.digest.key)- partial = Truetry:- _post_to_workflows(workflows_session, body, f"{input.digest.key}:{user.id}")+ # The payload holds UUIDs and datetimes, which plain json cannot encode.+ encoded_body = json.dumps(body, cls=DjangoJSONEncoder).encode()+ if len(encoded_body) > WORKFLOWS_WEBHOOK_BODY_LIMIT:+ logger.error(+ "Digest exceeds the Workflows webhook size limit, leaving the organization unsent",+ organization_id=organization.id,+ user_id=user.id,+ )+ _record_workflows_send_failure("oversized")+ failed_organization_ids.add(str(organization.id))+ continue+ _wait_for_workflows_post(r, input.digest.key)+ partial = True+ # Both regions post to one workflow and user ids repeat across regions, so the key uses the UUID.+ _post_to_workflows(workflows_session, encoded_body, f"{input.digest.key}:{user.uuid}")except WorkflowsWebhookError:failed_organization_ids.add(str(organization.id))raise+ except Exception:+ # Any other error still leaves the organization unsent, so the next attempt resends it.+ _record_workflows_send_failure("unexpected_error")+ failed_organization_ids.add(str(organization.id))+ raiseelif len(json.dumps(payload, default=str).encode()) > DIGEST_PAYLOAD_SIZE_LIMIT:logger.error("Digest exceeds the event size limit, leaving the organization unsent",diff --git a/posthog/temporal/weekly_digest/queries.py b/posthog/temporal/weekly_digest/queries.pyindex fb046424503b..603fe7ca678b 100644--- a/posthog/temporal/weekly_digest/queries.py+++ b/posthog/temporal/weekly_digest/queries.py@@ -68,7 +68,7 @@ def query_org_members(organization: Organization) -> QuerySet:return (OrganizationMembership.objects.filter(organization_id=organization.id).select_related("user")- .only("id", "user__distinct_id", "user__first_name", "user__email")+ .only("id", "user__distinct_id", "user__first_name", "user__email", "user__uuid")).order_by("id")