Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(weekly-digest): encode the workflows webhook body before posting (#127219)

Harley Alexandercc83ffc50550
diff --git a/posthog/temporal/tests/weekly_digest/test_activities.py b/posthog/temporal/tests/weekly_digest/test_activities.py
index 2cbe3e255c6b..962462ef5960 100644
--- a/posthog/temporal/tests/weekly_digest/test_activities.py
+++ b/posthog/temporal/tests/weekly_digest/test_activities.py
@@ -766,11 +766,14 @@ def test_send_weekly_digest_batch_emails_subscribed_members_once(
assert post.args == (WEBHOOK_SETTINGS["WEEKLY_DIGEST_WORKFLOW_WEBHOOK_URL"],)
assert post.kwargs["headers"] == {
"Authorization": "Bearer test-webhook-secret",
- "Idempotency-Key": f"{digest.key}:{subscribed.id}",
+ "Content-Type": "application/json",
+ "Idempotency-Key": f"{digest.key}:{subscribed.uuid}",
}
- assert post.kwargs["json"]["event"] == "weekly digest ready"
- assert post.kwargs["json"]["distinct_id"] == (subscribed.distinct_id or subscribed.email)
- properties = post.kwargs["json"]["properties"]
+ # The digest holds UUIDs and datetimes, so the body must be encoded before it reaches requests.
+ body = json.loads(post.kwargs["data"])
+ assert body["event"] == "weekly digest ready"
+ assert body["distinct_id"] == (subscribed.distinct_id or subscribed.email)
+ properties = body["properties"]
assert properties["to"] == subscribed.email
(team_payload,) = properties["teams"]
assert team_payload["team_id"] == team.id
@@ -843,6 +846,7 @@ def test_send_weekly_digest_batch_stops_sending_once_the_activity_is_cancelled(
("queue_full", 1),
("oversized", 0),
("failed_webhook", 0),
+ ("unexpected_webhook_error", 0),
],
)
def test_send_weekly_digest_batch_leaves_organization_unsent_when_an_event_is_dropped(
@@ -868,6 +872,8 @@ def test_send_weekly_digest_batch_leaves_organization_unsent_when_an_event_is_dr
session = MagicMock()
session.post.return_value.status_code = 503
+ if drop == "unexpected_webhook_error":
+ session.post.side_effect = TypeError("Object of type UUID is not JSON serializable")
with (
override_settings(**WEBHOOK_SETTINGS),
@@ -876,7 +882,10 @@ def test_send_weekly_digest_batch_leaves_organization_unsent_when_an_event_is_dr
"posthog.temporal.weekly_digest.activities.DIGEST_PAYLOAD_SIZE_LIMIT",
100 if drop == "oversized" else DIGEST_PAYLOAD_SIZE_LIMIT,
),
- patch("posthog.temporal.weekly_digest.activities._routes_to_workflows", return_value=drop == "failed_webhook"),
+ patch(
+ "posthog.temporal.weekly_digest.activities._routes_to_workflows",
+ return_value=drop in ("failed_webhook", "unexpected_webhook_error"),
+ ),
patch("posthog.temporal.weekly_digest.activities.requests.Session", return_value=session),
patch("posthog.temporal.weekly_digest.activities.time.sleep"),
):
diff --git a/posthog/temporal/weekly_digest/activities.py b/posthog/temporal/weekly_digest/activities.py
index 054a1255243c..f2b064f27542 100644
--- a/posthog/temporal/weekly_digest/activities.py
+++ b/posthog/temporal/weekly_digest/activities.py
@@ -9,6 +9,7 @@ from typing import Any, ParamSpec, Protocol, TypeVar
from uuid import UUID
from django.conf import settings
+from django.core.serializers.json import DjangoJSONEncoder
from django.db.models import F, QuerySet, Window
from django.db.models.functions import RowNumber
from django.utils import timezone
@@ -1114,11 +1115,12 @@ def _record_workflows_send_failure(reason: str) -> None:
).add(1)
-def _post_to_workflows(session: requests.Session, body: dict[str, Any], idempotency_key: str) -> None:
+def _post_to_workflows(session: requests.Session, body: bytes, idempotency_key: str) -> None:
# The secret authenticates the post, so only the digest send can start the workflow. The idempotency key
# makes a retried post, in this attempt or the next one, start no second run.
headers = {
"Authorization": f"Bearer {settings.WEEKLY_DIGEST_WORKFLOW_WEBHOOK_SECRET}",
+ "Content-Type": "application/json",
"Idempotency-Key": idempotency_key,
}
reason = "connection_error"
@@ -1129,7 +1131,7 @@ def _post_to_workflows(session: requests.Session, body: dict[str, Any], idempote
try:
response = session.post(
settings.WEEKLY_DIGEST_WORKFLOW_WEBHOOK_URL,
- json=body,
+ data=body,
headers=headers,
timeout=WORKFLOWS_WEBHOOK_TIMEOUT_SECONDS,
)
@@ -1277,22 +1279,30 @@ def _send_weekly_digest_batch(input: SendWeeklyDigestBatchInput) -> None:
"distinct_id": _workflows_distinct_id(user),
"properties": {**workflows_payload, "to": user.email},
}
- if len(json.dumps(body, default=str).encode()) > WORKFLOWS_WEBHOOK_BODY_LIMIT:
- logger.error(
- "Digest exceeds the Workflows webhook size limit, leaving the organization unsent",
- organization_id=organization.id,
- user_id=user.id,
- )
- _record_workflows_send_failure("oversized")
- failed_organization_ids.add(str(organization.id))
- continue
- _wait_for_workflows_post(r, input.digest.key)
- partial = True
try:
- _post_to_workflows(workflows_session, body, f"{input.digest.key}:{user.id}")
+ # The payload holds UUIDs and datetimes, which plain json cannot encode.
+ encoded_body = json.dumps(body, cls=DjangoJSONEncoder).encode()
+ if len(encoded_body) > WORKFLOWS_WEBHOOK_BODY_LIMIT:
+ logger.error(
+ "Digest exceeds the Workflows webhook size limit, leaving the organization unsent",
+ organization_id=organization.id,
+ user_id=user.id,
+ )
+ _record_workflows_send_failure("oversized")
+ failed_organization_ids.add(str(organization.id))
+ continue
+ _wait_for_workflows_post(r, input.digest.key)
+ partial = True
+ # Both regions post to one workflow and user ids repeat across regions, so the key uses the UUID.
+ _post_to_workflows(workflows_session, encoded_body, f"{input.digest.key}:{user.uuid}")
except WorkflowsWebhookError:
failed_organization_ids.add(str(organization.id))
raise
+ except Exception:
+ # Any other error still leaves the organization unsent, so the next attempt resends it.
+ _record_workflows_send_failure("unexpected_error")
+ failed_organization_ids.add(str(organization.id))
+ raise
elif len(json.dumps(payload, default=str).encode()) > DIGEST_PAYLOAD_SIZE_LIMIT:
logger.error(
"Digest exceeds the event size limit, leaving the organization unsent",
diff --git a/posthog/temporal/weekly_digest/queries.py b/posthog/temporal/weekly_digest/queries.py
index fb046424503b..603fe7ca678b 100644
--- a/posthog/temporal/weekly_digest/queries.py
+++ b/posthog/temporal/weekly_digest/queries.py
@@ -68,7 +68,7 @@ def query_org_members(organization: Organization) -> QuerySet:
return (
OrganizationMembership.objects.filter(organization_id=organization.id)
.select_related("user")
- .only("id", "user__distinct_id", "user__first_name", "user__email")
+ .only("id", "user__distinct_id", "user__first_name", "user__email", "user__uuid")
).order_by("id")