Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(workflows): mark broadcast runs failed when their sends error (#129075)

Harley Alexander5d39c9b47e5a
diff --git a/posthog/clickhouse/query_tagging.py b/posthog/clickhouse/query_tagging.py
index eb93bc6dad6b..e448cc2e3cc3 100644
--- a/posthog/clickhouse/query_tagging.py
+++ b/posthog/clickhouse/query_tagging.py
@@ -118,6 +118,7 @@ class Feature(StrEnum):
PREAGGREGATION = "preaggregation"
DATA_DELETION = "data_deletion"
DATA_QUALITY_CHECK = "data_quality_check" # one data quality check assertion against its subject
+ BATCH_RUN_OUTCOME = "batch_run_outcome" # a workflow batch run checks whether its sends errored
ENRICHMENT = "enrichment" # background tasks that derive/sync data (not customer-facing)
EVENT_FILTERS = "event_filters"
SCHEMA_INTROSPECTION = "schema_introspection"
diff --git a/products/workflows/backend/facade/enums.py b/products/workflows/backend/facade/enums.py
index 7a20efb9a8fc..4710d68269b2 100644
--- a/products/workflows/backend/facade/enums.py
+++ b/products/workflows/backend/facade/enums.py
@@ -39,7 +39,10 @@ class HogFlowBatchJobState(LabeledStrEnum):
QUEUED = "queued"
ACTIVE = "active"
COMPLETED = "completed"
+ # Set after a completed run settles with some of its sends errored, and the rest sent.
+ PARTIALLY_FAILED = "partially_failed", "Partially failed"
CANCELLED = "cancelled"
+ # The run could not fan out to its audience, or it settled with every send errored.
FAILED = "failed"
diff --git a/products/workflows/backend/facade/tasks.py b/products/workflows/backend/facade/tasks.py
index 69f264323ab7..9c64d0482c91 100644
--- a/products/workflows/backend/facade/tasks.py
+++ b/products/workflows/backend/facade/tasks.py
@@ -1,4 +1,8 @@
-from products.workflows.backend.tasks.batch_jobs import resume_batch_jobs_waiting_for_realtime_cohort
+from products.workflows.backend.tasks.batch_jobs import (
+ resume_batch_jobs_waiting_for_realtime_cohort,
+ schedule_batch_run_settle,
+ settle_batch_run_outcome,
+)
from products.workflows.backend.tasks.email_sending_tiers import recompute_workflows_email_sending_tiers
from products.workflows.backend.tasks.hog_flows import refresh_affected_hog_flows
from products.workflows.backend.tasks.ses_account_reputation import (
@@ -15,5 +19,7 @@ __all__ = [
"recompute_workflows_email_sending_tiers",
"reconcile_ses_tenant_states",
"refresh_affected_hog_flows",
+ "schedule_batch_run_settle",
+ "settle_batch_run_outcome",
"sweep_workflow_email_deliverability",
]
diff --git a/products/workflows/backend/migrations/0041_hogflowbatchjob_partially_failed_status.py b/products/workflows/backend/migrations/0041_hogflowbatchjob_partially_failed_status.py
new file mode 100644
index 000000000000..4cd557a828f6
--- /dev/null
+++ b/products/workflows/backend/migrations/0041_hogflowbatchjob_partially_failed_status.py
@@ -0,0 +1,29 @@
+# Generated by Django 6.1.3 on 2026-11-04 09:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("workflows", "0040_team_workflows_config_frequency_cap_hold"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="hogflowbatchjob",
+ name="status",
+ field=models.CharField(
+ choices=[
+ ("waiting", "Waiting"),
+ ("queued", "Queued"),
+ ("active", "Active"),
+ ("completed", "Completed"),
+ ("partially_failed", "Partially failed"),
+ ("cancelled", "Cancelled"),
+ ("failed", "Failed"),
+ ],
+ default="queued",
+ max_length=20,
+ ),
+ ),
+ ]
diff --git a/products/workflows/backend/migrations/max_migration.txt b/products/workflows/backend/migrations/max_migration.txt
index 2fcdef123a71..f19ce8868624 100644
--- a/products/workflows/backend/migrations/max_migration.txt
+++ b/products/workflows/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0040_team_workflows_config_frequency_cap_hold
+0041_hogflowbatchjob_partially_failed_status
diff --git a/products/workflows/backend/presentation/views/hog_flow.py b/products/workflows/backend/presentation/views/hog_flow.py
index 9cb4976ff9f2..af4e8d931a26 100644
--- a/products/workflows/backend/presentation/views/hog_flow.py
+++ b/products/workflows/backend/presentation/views/hog_flow.py
@@ -289,6 +289,7 @@ from products.workflows.backend.facade.suggestions_scout import (
suggestions_scout_status,
sync_suggestions_scout,
)
+from products.workflows.backend.facade.tasks import schedule_batch_run_settle
from products.workflows.backend.facade.templates import get_function_template_schema
from products.workflows.backend.facade.validation import (
DURATION_PATTERN,
@@ -347,6 +348,7 @@ from products.workflows.backend.facade.writes import (
from products.workflows.backend.presentation.views.graph_operations import apply_graph_operations
from products.workflows.backend.presentation.views.graph_validation import validate_graph
from products.workflows.backend.presentation.views.hog_flow_batch_job import (
+ BATCH_RUN_STATUS_HELP_TEXT,
HogFlowBatchJobCancelResponseSerializer,
HogFlowBatchJobSerializer,
)
@@ -2688,7 +2690,7 @@ class SendCalendarQuerySerializer(serializers.Serializer):
class SendCalendarBatchJobSerializer(serializers.Serializer):
id = serializers.UUIDField(read_only=True, help_text="ID of the batch run.")
status = serializers.ChoiceField(
- choices=HogFlowBatchJobState.choices, read_only=True, help_text="Status of the batch run."
+ choices=HogFlowBatchJobState.choices, read_only=True, help_text=BATCH_RUN_STATUS_HELP_TEXT
)
created_at = serializers.DateTimeField(read_only=True, help_text="When the batch run started.")
@@ -5020,14 +5022,16 @@ class HogFlowVersionMetricsRequestSerializer(AppMetricsRequestSerializer):
OpenApiTypes.STR,
description=(
"Comma-separated broadcast statuses as the broadcasts UI shows them: draft, scheduled, sending, "
- "sent, failed, archived. Scheduled, sending, sent and failed come from the latest run and "
- "whether a schedule still has sends to come."
+ "sent, partially_sent, failed, archived. Scheduled, sending, sent, partially_sent and failed "
+ "come from the latest run and whether a schedule still has sends to come. A latest run that is "
+ "completed reads as sent, partially_failed as partially_sent, and failed or cancelled as failed. "
+ "With a schedule still to come, a completed or partially_failed latest run reads as scheduled."
),
),
OpenApiParameter(
"latest_send_first",
OpenApiTypes.BOOL,
- description="Pass `true` to order by when each workflow's newest completed run finished, newest first. Workflows that never finished a run come last.",
+ description="Pass `true` to order by when each workflow's newest completed or partially_failed run finished, newest first. Workflows that never finished such a run come last.",
),
]
),
@@ -7885,7 +7889,8 @@ class InternalHogFlowViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, AppMetricsMi
terminal status, returns 200 without re-writing — the resolver retries
this call via cyclotron retry semantics, so safe repeats are required.
- Accepts: { status: "completed" | "failed" }
+ Accepts: { status: "completed" | "failed" }. A completed run can later move to
+ failed or partially_failed, once its child runs show errored sends.
"""
if request.method != "PUT":
return Response({"error": "Method not allowed"}, status=405)
@@ -7910,6 +7915,7 @@ class InternalHogFlowViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, AppMetricsMi
terminal_states = {
HogFlowBatchJobState.COMPLETED,
+ HogFlowBatchJobState.PARTIALLY_FAILED,
HogFlowBatchJobState.FAILED,
HogFlowBatchJobState.CANCELLED,
}
@@ -7926,6 +7932,10 @@ class InternalHogFlowViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, AppMetricsMi
)
try:
+ if new_status == HogFlowBatchJobState.COMPLETED:
+ # The child runs send after completed, so a later check sets failed or partially_failed. It is
+ # queued before the write, so the resolver's retry queues it again if queueing fails.
+ schedule_batch_run_settle(team_id=team.id, batch_job_id=str(batch_job.id))
set_batch_job_status(team_id=team.id, batch_job_id=batch_job.id, status=HogFlowBatchJobState(new_status))
return Response(
InternalBatchJobStatusSerializer(
diff --git a/products/workflows/backend/presentation/views/hog_flow_batch_job.py b/products/workflows/backend/presentation/views/hog_flow_batch_job.py
index f773bc784ebc..5fb03fbaa42c 100644
--- a/products/workflows/backend/presentation/views/hog_flow_batch_job.py
+++ b/products/workflows/backend/presentation/views/hog_flow_batch_job.py
@@ -7,16 +7,24 @@ from products.workflows.backend.facade.enums import HogFlowBatchJobState
logger = structlog.get_logger(__name__)
+BATCH_RUN_STATUS_HELP_TEXT = (
+ "Status of the batch run. waiting: the audience's cohort is still calculating. queued and active: the run "
+ "is enrolling its audience. completed: the whole audience was enrolled. When every enrolled person's run "
+ "has ended, or a day after the run completed, PostHog checks for errored sends. A completed run stays "
+ "completed when no send errored, becomes "
+ "partially_failed when some sends errored, and becomes failed when every send errored. failed also means "
+ "the run could not enroll its audience. cancelled: someone stopped the run. A send errors when its step "
+ "throws, for example on a template that does not render. Provider failures such as bounces are not errored "
+ "sends; read them from the run's metrics."
+)
+
class HogFlowBatchJobSerializer(serializers.Serializer):
id = serializers.UUIDField(read_only=True, help_text="ID of the batch run.")
status = serializers.ChoiceField(
choices=HogFlowBatchJobState.choices,
required=False,
- help_text=(
- "Not currently tracked — stays at its initial value. Use the workflow logs/metrics "
- "endpoints for run outcome."
- ),
+ help_text=BATCH_RUN_STATUS_HELP_TEXT,
)
hog_flow = serializers.UUIDField(source="hog_flow_id", help_text="ID of the workflow this batch run belongs to.")
filters = serializers.JSONField(
diff --git a/products/workflows/backend/services/batch_jobs.py b/products/workflows/backend/services/batch_jobs.py
index 414ea76b4220..b3142831b8c8 100644
--- a/products/workflows/backend/services/batch_jobs.py
+++ b/products/workflows/backend/services/batch_jobs.py
@@ -115,11 +115,17 @@ def filter_by_broadcast_status(queryset: QuerySet[HogFlow], statuses: Collection
"archived": Q(status=HogFlow.State.ARCHIVED),
"sending": live & Q(_latest_run_status__in=running),
"sent": live & Q(_latest_run_status=HogFlowBatchJobState.COMPLETED, _has_pending_schedule=False),
+ # Kept apart from failed, because a sender who sees Failed can resend and reach the same people twice.
+ "partially_sent": live
+ & Q(_latest_run_status=HogFlowBatchJobState.PARTIALLY_FAILED, _has_pending_schedule=False),
"scheduled": live
& (
(
Q(_has_pending_schedule=True)
- & (Q(_latest_run_status__isnull=True) | Q(_latest_run_status=HogFlowBatchJobState.COMPLETED))
+ & (
+ Q(_latest_run_status__isnull=True)
+ | Q(_latest_run_status__in=[HogFlowBatchJobState.COMPLETED, HogFlowBatchJobState.PARTIALLY_FAILED])
+ )
)
| (nothing_to_come & ~Q(origin_product="broadcasts"))
),
@@ -139,10 +145,12 @@ def hog_flow_ids_with_broadcast_status(*, team_id: int, statuses: Collection[str
def annotate_latest_send(queryset: QuerySet[HogFlow]) -> QuerySet[HogFlow]:
- """Adds `_latest_send_at`: when the workflow's newest completed run finished, or null."""
+ """Adds `_latest_send_at`: when the workflow's newest run that sent to anyone finished, or null."""
latest_send = (
HogFlowBatchJob.objects.filter(
- team_id=OuterRef("team_id"), hog_flow_id=OuterRef("pk"), status=HogFlowBatchJobState.COMPLETED
+ team_id=OuterRef("team_id"),
+ hog_flow_id=OuterRef("pk"),
+ status__in=[HogFlowBatchJobState.COMPLETED, HogFlowBatchJobState.PARTIALLY_FAILED],
)
.order_by("-updated_at")
.values("updated_at")[:1]
diff --git a/products/workflows/backend/services/batch_run_outcome.py b/products/workflows/backend/services/batch_run_outcome.py
new file mode 100644
index 000000000000..d6986c487d0c
--- /dev/null
+++ b/products/workflows/backend/services/batch_run_outcome.py
@@ -0,0 +1,157 @@
+"""The final status of a completed batch run, read from the app metrics of its child runs.
+
+The resolver marks a run completed when it has enqueued the whole audience, before the child runs
+execute. This module checks the child runs after that point and moves the run to failed or
+partially_failed when its sends errored.
+"""
+
+from datetime import UTC, datetime
+from typing import Any
+from uuid import UUID
+
+import structlog
+from prometheus_client import Counter
+
+from posthog.schema import ProductKey
+
+from posthog.clickhouse.client.execute import sync_execute
+from posthog.clickhouse.query_tagging import Feature, tags_context
+from posthog.clickhouse.workload import Workload
+from posthog.dataclasses import frozen
+
+from products.workflows.backend.facade.contracts import WorkflowBatchJobNotFound
+from products.workflows.backend.facade.enums import HogFlowBatchJobState
+from products.workflows.backend.models.hog_flow.hog_flow import HogFlow
+from products.workflows.backend.services.batch_jobs import get_batch_job, set_batch_job_status
+
+logger = structlog.get_logger(__name__)
+
+# The executor records run-level metrics without a step, so ClickHouse stores an empty instance_id.
+RUN_LEVEL_INSTANCE_ID = ""
+# The executor records `failed` for every action type, so only these steps count as a send that errored.
+SEND_ACTION_TYPES = frozenset({"function_email", "function_sms", "function_push"})
+
+BATCH_RUN_OUTCOMES = Counter(
+ "workflows_batch_run_outcome_total",
+ "Completed batch runs checked for errored sends, by the status found and whether every child run had ended.",
+ labelnames=["outcome", "settled"],
+)
+
+
+@frozen
+class BatchRunSendCounts:
+ # Child runs the resolver started: the run-level `triggered` metric.
+ started: int
+ # Child runs that ended: the run-level `succeeded` and `failed` metrics, plus `canceled`.
+ finished: int
+ # The highest `failed` count of any send step. With on_error set to continue, an errored run still counts
+ # as succeeded at run level. The maximum, unlike a sum, counts a run that errors at two send steps once.
+ errored: int
+
+ @property
+ def settled(self) -> bool:
+ # No `triggered` metric yet can also mean the metrics have not reached ClickHouse, so keep checking.
+ return self.started > 0 and self.finished >= self.started
+
+ @property
+ def outcome(self) -> HogFlowBatchJobState:
+ if self.errored == 0 or self.started == 0:
+ return HogFlowBatchJobState.COMPLETED
+ if self.errored >= self.started:
+ return HogFlowBatchJobState.FAILED
+ return HogFlowBatchJobState.PARTIALLY_FAILED
+
+
+def batch_run_send_counts_from_rows(
+ rows: list[tuple[str, str, int]], send_step_ids: frozenset[str]
+) -> BatchRunSendCounts:
+ """Fold `(instance_id, metric_name, count)` rows of one batch run into its send counts."""
+ started = finished = 0
+ errored_by_step: dict[str, int] = {}
+ for instance_id, metric_name, count in rows:
+ if metric_name == "canceled":
+ # A canceled run records the step it stopped at, so its instance_id is not empty.
+ finished += count
+ elif instance_id == RUN_LEVEL_INSTANCE_ID:
+ if metric_name == "triggered":
+ started += count
+ elif metric_name in ("succeeded", "failed"):
+ finished += count
+ elif metric_name == "failed" and instance_id in send_step_ids:
+ errored_by_step[instance_id] = errored_by_step.get(instance_id, 0) + count
+ return BatchRunSendCounts(started=started, finished=finished, errored=max(errored_by_step.values(), default=0))
+
+
+def send_step_ids(*, team_id: int, hog_flow_id: UUID) -> frozenset[str]:
+ actions = HogFlow.objects.filter(team_id=team_id, id=hog_flow_id).values_list("actions", flat=True).first()
+ return frozenset(
+ action["id"] for action in actions or [] if isinstance(action, dict) and action.get("type") in SEND_ACTION_TYPES
+ )
+
+
+def fetch_batch_run_send_counts(
+ *, team_id: int, batch_job_id: UUID, run_created_at: datetime, send_step_ids: frozenset[str]
+) -> BatchRunSendCounts:
+ # The table keeps one timestamp per collapsed clock hour, so the lower bound is the start of the
+ # hour the run was created in. A bound in the middle of that hour could drop the run's first rows.
+ after = run_created_at.astimezone(UTC).replace(minute=0, second=0, microsecond=0)
+ with tags_context(product=ProductKey.WORKFLOWS, feature=Feature.BATCH_RUN_OUTCOME):
+ rows: Any = sync_execute(
+ """
+ SELECT instance_id, metric_name, sum(count) AS count
+ FROM app_metrics2
+ WHERE team_id = %(team_id)s
+ AND app_source = 'hog_flow'
+ AND app_source_id = %(app_source_id)s
+ AND timestamp >= toDateTime64(%(after)s, 6)
+ AND metric_name IN ('triggered', 'succeeded', 'failed', 'canceled')
+ GROUP BY instance_id, metric_name
+ """,
+ {"team_id": team_id, "app_source_id": str(batch_job_id), "after": after.strftime("%Y-%m-%dT%H:%M:%S")},
+ workload=Workload.OFFLINE,
+ team_id=team_id,
+ )
+ return batch_run_send_counts_from_rows([(str(row[0]), str(row[1]), int(row[2])) for row in rows], send_step_ids)
+
+
+def settle_batch_job_outcome(*, team_id: int, batch_job_id: str, final_check: bool) -> bool:
+ """Write the status a completed run settles at. Returns False while its child runs still execute.
+
+ On the final check the status comes from the counts so far, so that a child run that never
+ records an end cannot hold the check open forever.
+ """
+ try:
+ job = get_batch_job(team_id=team_id, batch_job_id=batch_job_id)
+ except WorkflowBatchJobNotFound:
+ # The run or its team was deleted, so there is nothing to check and a retry cannot help.
+ return True
+ if job.status != HogFlowBatchJobState.COMPLETED:
+ return True
+ counts = fetch_batch_run_send_counts(
+ team_id=team_id,
+ batch_job_id=job.id,
+ run_created_at=job.created_at,
+ send_step_ids=send_step_ids(team_id=team_id, hog_flow_id=job.hog_flow_id),
+ )
+ if not counts.settled and not final_check:
+ return False
+ outcome = counts.outcome
+ BATCH_RUN_OUTCOMES.labels(outcome=outcome.value, settled=str(counts.settled).lower()).inc()
+ if not counts.settled:
+ logger.warning(
+ "batch_run_outcome_unsettled",
+ team_id=team_id,
+ batch_job_id=batch_job_id,
+ started=counts.started,
+ finished=counts.finished,
+ errored=counts.errored,
+ )
+ if outcome != HogFlowBatchJobState.COMPLETED:
+ # Conditional, so a status that something else wrote since the read above wins.
+ set_batch_job_status(
+ team_id=team_id,
+ batch_job_id=job.id,
+ status=outcome,
+ from_statuses=[HogFlowBatchJobState.COMPLETED],
+ )
+ return True
diff --git a/products/workflows/backend/services/hog_flow_reads.py b/products/workflows/backend/services/hog_flow_reads.py
index e42ea390d215..9864f9024efd 100644
--- a/products/workflows/backend/services/hog_flow_reads.py
+++ b/products/workflows/backend/services/hog_flow_reads.py
@@ -478,7 +478,7 @@ def annotate_broadcast_shape(queryset: QuerySet) -> QuerySet:
)
-BROADCAST_STATUSES = ("draft", "scheduled", "sending", "sent", "failed", "archived")
+BROADCAST_STATUSES = ("draft", "scheduled", "sending", "sent", "partially_sent", "failed", "archived")
class _WorkflowFilterSet(FilterSet):
diff --git a/products/workflows/backend/tasks/batch_jobs.py b/products/workflows/backend/tasks/batch_jobs.py
index 164a85657c0a..384545f6eec0 100644
--- a/products/workflows/backend/tasks/batch_jobs.py
+++ b/products/workflows/backend/tasks/batch_jobs.py
@@ -1,9 +1,20 @@
-from celery import shared_task
+from celery import Task, shared_task
from posthog.scoping_audit import skip_team_scope_audit
from posthog.tasks.utils import CeleryQueue
from products.workflows.backend.services.batch_jobs import resume_batch_jobs_waiting_for_cohort
+from products.workflows.backend.services.batch_run_outcome import settle_batch_job_outcome
+
+# The first check waits for the child runs of a small audience to execute and for their metrics to
+# reach ClickHouse. Later checks follow at a fixed interval, and the check gives up waiting after a day.
+BATCH_RUN_SETTLE_FIRST_CHECK_SECONDS = 5 * 60
+BATCH_RUN_SETTLE_CHECK_INTERVAL_SECONDS = 10 * 60
+BATCH_RUN_SETTLE_MAX_RETRIES = 24 * 6
+
+
+class BatchRunNotSettled(Exception):
+ pass
@shared_task(
@@ -24,3 +35,26 @@ from products.workflows.backend.services.batch_jobs import resume_batch_jobs_wai
@skip_team_scope_audit
def resume_batch_jobs_waiting_for_realtime_cohort(team_id: int, cohort_id: int) -> None:
resume_batch_jobs_waiting_for_cohort(team_id=team_id, cohort_id=cohort_id)
+
+
+@shared_task(
+ bind=True,
+ ignore_result=True,
+ queue=CeleryQueue.DEFAULT.value,
+ # A check only reads metrics and makes a conditional write, so a task a dying worker held runs again.
+ acks_late=True,
+ reject_on_worker_lost=True,
+ autoretry_for=(Exception,),
+ retry_backoff=False,
+ default_retry_delay=BATCH_RUN_SETTLE_CHECK_INTERVAL_SECONDS,
+ max_retries=BATCH_RUN_SETTLE_MAX_RETRIES,
+)
+@skip_team_scope_audit
+def settle_batch_run_outcome(self: Task, team_id: int, batch_job_id: str) -> None:
+ final_check = self.request.retries >= BATCH_RUN_SETTLE_MAX_RETRIES
+ if not settle_batch_job_outcome(team_id=team_id, batch_job_id=batch_job_id, final_check=final_check):
+ raise BatchRunNotSettled(batch_job_id)
+
+
+def schedule_batch_run_settle(*, team_id: int, batch_job_id: str) -> None:
+ settle_batch_run_outcome.apply_async(args=(team_id, batch_job_id), countdown=BATCH_RUN_SETTLE_FIRST_CHECK_SECONDS)
diff --git a/products/workflows/backend/test/test_hog_flow_batch_job.py b/products/workflows/backend/test/test_hog_flow_batch_job.py
index db9ddbd3bb8f..bb22f4045162 100644
--- a/products/workflows/backend/test/test_hog_flow_batch_job.py
+++ b/products/workflows/backend/test/test_hog_flow_batch_job.py
@@ -2,6 +2,7 @@ from contextlib import nullcontext
from datetime import UTC, datetime
import pytest
+from posthog.test.base import BaseTest, ClickhouseTestMixin
from unittest.mock import MagicMock, patch
from django.db import connection
@@ -15,6 +16,7 @@ from parameterized import parameterized
from posthog.models.team import Team
from posthog.models.team.util import delete_team_records
from posthog.models.user import User
+from posthog.test.fixtures import create_app_metric2
from products.cohorts.backend.facade.ready_signals import realtime_cohort_ready
from products.cohorts.backend.models.cohort import Cohort, CohortType
@@ -24,6 +26,7 @@ from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job imp
REALTIME_COHORT_WAIT_LIMIT,
handle_hog_flow_batch_job_created,
)
+from products.workflows.backend.services.batch_run_outcome import settle_batch_job_outcome
@override_settings(REALTIME_COHORT_TEAM_ALLOWLIST="all")
@@ -383,3 +386,73 @@ class TestHogFlowBatchJob(TestCase):
batch_job = HogFlowBatchJob.objects.create(team=self.team, hog_flow=self.hog_flow, variables=variables)
assert batch_job.variables == variables
+
+
+class TestBatchRunOutcome(ClickhouseTestMixin, BaseTest):
+ def _completed_run(
+ self, step_failures: dict[str, int], *, started: int, finished: int, aborted: int
+ ) -> HogFlowBatchJob:
+ hog_flow = HogFlow.objects.create(
+ team=self.team,
+ name="Broadcast",
+ actions=[{"id": "email", "type": "function_email"}, {"id": "webhook", "type": "function"}],
+ )
+ with patch(
+ "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"
+ ):
+ job = HogFlowBatchJob.objects.create(team=self.team, hog_flow=hog_flow, status="completed")
+
+ def record(metric_name: str, count: int, instance_id: str = "") -> None:
+ if count:
+ create_app_metric2(
+ team_id=self.team.id,
+ app_source="hog_flow",
+ timestamp=job.created_at,
+ app_source_id=str(job.id),
+ instance_id=instance_id,
+ metric_name=metric_name,
+ count=count,
+ )
+
+ record("triggered", started)
+ record("succeeded", finished - aborted)
+ record("failed", aborted)
+ for step, count in step_failures.items():
+ record("failed", count, instance_id=step)
+ return job
+
+ @parameterized.expand(
+ [
+ ("no_send_errored", {}, 0, "completed"),
+ ("some_sends_errored", {"email": 1}, 0, "partially_failed"),
+ ("every_send_errored", {"email": 3}, 0, "failed"),
+ ("every_send_errored_and_aborted_the_run", {"email": 3}, 3, "failed"),
+ ("the_same_people_errored_at_two_steps", {"email": 2, "webhook": 2}, 0, "partially_failed"),
+ ("every_send_delivered_and_a_later_webhook_errored", {"webhook": 3}, 0, "completed"),
+ ]
+ )
+ def test_settled_run_status_follows_its_errored_sends(self, _name, step_failures, aborted, expected_status):
+ job = self._completed_run(step_failures, started=3, finished=3, aborted=aborted)
+
+ assert settle_batch_job_outcome(team_id=self.team.id, batch_job_id=str(job.id), final_check=False)
+ job.refresh_from_db()
+ assert job.status == expected_status
+
+ @parameterized.expand(
+ [
+ ("still_sending", 3, False, False, "completed"),
+ ("still_sending_final_check", 3, True, True, "partially_failed"),
+ ("metrics_not_arrived", 0, False, False, "completed"),
+ ]
+ )
+ def test_unsettled_run_keeps_its_status_until_the_final_check(
+ self, _name, started, final_check, settled, expected_status
+ ):
+ job = self._completed_run({"email": 1}, started=started, finished=min(started, 1), aborted=0)
+
+ outcome_written = settle_batch_job_outcome(
+ team_id=self.team.id, batch_job_id=str(job.id), final_check=final_check
+ )
+ assert outcome_written is settled
+ job.refresh_from_db()
+ assert job.status == expected_status
diff --git a/products/workflows/backend/tests/api/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.py
index 0bac06521953..01b0c3b3993d 100644
--- a/products/workflows/backend/tests/api/test_hog_flow.py
+++ b/products/workflows/backend/tests/api/test_hog_flow.py
@@ -766,12 +766,24 @@ class TestHogFlowAPI(APIBaseTest):
@parameterized.expand(
[
("draft", {"Draft"}),
- ("scheduled", {"Scheduled", "Recurring between runs", "Workflow waiting for an API send"}),
+ (
+ "scheduled",
+ {
+ "Scheduled",
+ "Recurring between runs",
+ "Recurring after some sends errored",
+ "Workflow waiting for an API send",
+ },
+ ),
("sending", {"Sending"}),
("sent", {"Sent", "Sent after a failed run"}),
+ ("partially_sent", {"Some sends errored"}),
("failed", {"Run failed", "Launch never finished"}),
("archived", {"Archived"}),
- ("sent,failed", {"Sent", "Sent after a failed run", "Run failed", "Launch never finished"}),
+ (
+ "sent,failed",
+ {"Sent", "Sent after a failed run", "Run failed", "Launch never finished"},
+ ),
]
)
@patch(
@@ -811,6 +823,8 @@ class TestHogFlowAPI(APIBaseTest):
create("Sending", runs=("active",))
create("Sent", runs=("completed",))
create("Sent after a failed run", runs=("failed", "completed"))
+ create("Some sends errored", runs=("completed", "partially_failed"))
+ create("Recurring after some sends errored", runs=("partially_failed",), schedule="active")
create("Run failed", runs=("failed",), schedule="active")
create("Launch never finished")
create("Workflow waiting for an API send", origin_product=None)
@@ -5121,38 +5135,48 @@ class TestHogFlowAPI(APIBaseTest):
assert response.status_code == 400, response.json()
assert "Unsupported dedupe_key" in response.json().get("error", "")
+ @parameterized.expand([("completed", True), ("failed", False)])
@override_settings(INTERNAL_API_SECRET="test-secret-123")
+ @patch("products.workflows.backend.tasks.batch_jobs.settle_batch_run_outcome.apply_async")
@patch(
"products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"
)
- def test_internal_update_batch_job_status_marks_completed(self, _mock_dispatch):
+ def test_internal_update_batch_job_status_writes_terminal_status(
+ self, new_status, checks_sends, _mock_dispatch, mock_settle
+ ):
hog_flow = HogFlow.objects.create(team=self.team, name="Test", trigger={}, actions=[], edges=[])
batch_job = HogFlowBatchJob.objects.create(team=self.team, hog_flow=hog_flow, status="active")
response = self.client.put(
f"/api/projects/{self.team.id}/internal/hog_flows/batch_jobs/{batch_job.id}/status",
- {"status": "completed"},
+ {"status": new_status},
content_type="application/json",
headers={"x-internal-api-secret": "test-secret-123"},
)
assert response.status_code == 200, response.json()
- assert response.json()["status"] == "completed"
+ assert response.json()["status"] == new_status
assert response.json()["no_op"] is False
batch_job.refresh_from_db()
- assert batch_job.status == "completed"
+ assert batch_job.status == new_status
+ assert mock_settle.called is checks_sends
+ if checks_sends:
+ assert mock_settle.call_args.kwargs["args"] == (self.team.id, str(batch_job.id))
+ @parameterized.expand([("completed", "failed"), ("partially_failed", "completed")])
@override_settings(INTERNAL_API_SECRET="test-secret-123")
@patch(
"products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation"
)
- def test_internal_update_batch_job_status_is_idempotent_when_already_terminal(self, _mock_dispatch):
+ def test_internal_update_batch_job_status_is_idempotent_when_already_terminal(
+ self, current_status, requested_status, _mock_dispatch
+ ):
hog_flow = HogFlow.objects.create(team=self.team, name="Test", trigger={}, actions=[], edges=[])
- batch_job = HogFlowBatchJob.objects.create(team=self.team, hog_flow=hog_flow, status="completed")
+ batch_job = HogFlowBatchJob.objects.create(team=self.team, hog_flow=hog_flow, status=current_status)
response = self.client.put(
f"/api/projects/{self.team.id}/internal/hog_flows/batch_jobs/{batch_job.id}/status",
- {"status": "failed"},
+ {"status": requested_status},
content_type="application/json",
headers={"x-internal-api-secret": "test-secret-123"},
)
@@ -5160,9 +5184,9 @@ class TestHogFlowAPI(APIBaseTest):
# Already terminal → no-op, original status preserved
assert response.status_code == 200, response.json()
assert response.json()["no_op"] is True
- assert response.json()["status"] == "completed"
+ assert response.json()["status"] == current_status
batch_job.refresh_from_db()
- assert batch_job.status == "completed"
+ assert batch_job.status == current_status
@override_settings(INTERNAL_API_SECRET="test-secret-123")
@patch(
diff --git a/products/workflows/frontend/Broadcasts/broadcastPreviousSendLogic.test.ts b/products/workflows/frontend/Broadcasts/broadcastPreviousSendLogic.test.ts
index 8419f9a5e2a5..af082537f761 100644
--- a/products/workflows/frontend/Broadcasts/broadcastPreviousSendLogic.test.ts
+++ b/products/workflows/frontend/Broadcasts/broadcastPreviousSendLogic.test.ts
@@ -62,6 +62,13 @@ describe('broadcastPreviousSendLogic', () => {
expected: { broadcastId: 'flow-1', state: 'sent', job: completed },
sentCount: 12,
},
+ {
+ case: 'a partly failed run still reached people',
+ flows: [{ id: 'flow-1', status: 'active' }],
+ jobs: { 'flow-1': [{ id: 'job-5', status: 'partially_failed' }] },
+ expected: { broadcastId: 'flow-1', state: 'sent', job: { id: 'job-5', status: 'partially_failed' } },
+ sentCount: 12,
+ },
{
case: 'a failed run is the only history',
flows: [{ id: 'flow-1', status: 'active' }],
diff --git a/products/workflows/frontend/Broadcasts/broadcastPreviousSendLogic.ts b/products/workflows/frontend/Broadcasts/broadcastPreviousSendLogic.ts
index 351996d90aa1..817941a7a40b 100644
--- a/products/workflows/frontend/Broadcasts/broadcastPreviousSendLogic.ts
+++ b/products/workflows/frontend/Broadcasts/broadcastPreviousSendLogic.ts
@@ -108,7 +108,8 @@ export const broadcastPreviousSendLogic = kea<broadcastPreviousSendLogicType>([
let pending: PreviousSend | null = null
for (const flow of (page.results ?? []).filter((flow) => flow.status !== 'draft')) {
const jobs = await hogFlowsBatchJobsList(projectId, flow.id)
- const sent = jobs.find((job) => job.status === 'completed')
+ // A partly failed run still reached some people, so it counts as a send.
+ const sent = jobs.find((job) => job.status === 'completed' || job.status === 'partially_failed')
if (sent) {
return { broadcastId: flow.id, state: 'sent', job: sent }
}
diff --git a/products/workflows/frontend/SendCalendar/sendCalendarUtils.test.ts b/products/workflows/frontend/SendCalendar/sendCalendarUtils.test.ts
index 626569352e69..f0d29640f8f0 100644
--- a/products/workflows/frontend/SendCalendar/sendCalendarUtils.test.ts
+++ b/products/workflows/frontend/SendCalendar/sendCalendarUtils.test.ts
@@ -114,6 +114,7 @@ describe('sendCalendarUtils', () => {
batch_jobs: [
job('sent', 'completed', '2026-10-20T08:00:05Z'),
job('failed', 'failed', '2026-10-19T09:00:00Z'),
+ job('partial', 'partially_failed', '2026-10-21T07:00:00Z'),
job('outside', 'completed', '2026-10-05T08:00:00Z'),
],
} as unknown as SendCalendarFlowApi,
@@ -127,6 +128,7 @@ describe('sendCalendarUtils', () => {
['paused', '2026-10-30T16:00:00.000Z', 'Every week on Friday'],
['sent', '2026-10-20T08:00:05.000Z', null],
['failed', '2026-10-19T09:00:00.000Z', null],
+ ['sent', '2026-10-21T07:00:00.000Z', null],
])
})
diff --git a/products/workflows/frontend/SendCalendar/sendCalendarUtils.ts b/products/workflows/frontend/SendCalendar/sendCalendarUtils.ts
index 422a43fcfb8f..e2f7864e9ccf 100644
--- a/products/workflows/frontend/SendCalendar/sendCalendarUtils.ts
+++ b/products/workflows/frontend/SendCalendar/sendCalendarUtils.ts
@@ -218,7 +218,8 @@ export function calendarSendsForFlow(
}
function batchJobState(status: SendCalendarBatchJobApi['status']): CalendarSendState {
- if (status === 'completed') {
+ // A partially failed run reached part of its audience. Showing it as failed could prompt a resend to those people.
+ if (status === 'completed' || status === 'partially_failed') {
return 'sent'
}
if (status === 'failed' || status === 'cancelled') {
diff --git a/products/workflows/frontend/Workflows/WorkflowBatchInvocations.tsx b/products/workflows/frontend/Workflows/WorkflowBatchInvocations.tsx
index fece25783f08..855a7072a5c7 100644
--- a/products/workflows/frontend/Workflows/WorkflowBatchInvocations.tsx
+++ b/products/workflows/frontend/Workflows/WorkflowBatchInvocations.tsx
@@ -38,6 +38,7 @@ const HedgehogGreek = pngHoggie(greekPng)
const STATUS_TAG_TYPE: Record<HogFlowBatchJob['status'], 'success' | 'danger' | 'warning' | 'default'> = {
completed: 'success',
+ partially_failed: 'warning',
failed: 'danger',
cancelled: 'default',
waiting: 'warning',
@@ -61,7 +62,7 @@ function BatchRunHeader({ job, hogFlowId }: { job: HogFlowBatchJob; hogFlowId: s
<strong className="truncate">{job.id}</strong>
<LemonTag type={STATUS_TAG_TYPE[job.status] ?? 'default'}>
{/* The stored enum value is 'cancelled'; display American spelling. */}
- {job.status === 'cancelled' ? 'canceled' : job.status}
+ {job.status === 'cancelled' ? 'canceled' : job.status.replace('_', ' ')}
</LemonTag>
</div>
<div className="flex items-center gap-2">
diff --git a/products/workflows/frontend/Workflows/hogflows/types.ts b/products/workflows/frontend/Workflows/hogflows/types.ts
index 2e64d5fe89e4..131640a88e56 100644
--- a/products/workflows/frontend/Workflows/hogflows/types.ts
+++ b/products/workflows/frontend/Workflows/hogflows/types.ts
@@ -81,7 +81,7 @@ export const HogFlowBatchJobSchema = z.object({
id: z.string(),
hog_flow: z.string(),
variables: z.record(z.string(), z.any()),
- status: z.enum(['waiting', 'queued', 'active', 'completed', 'cancelled', 'failed']),
+ status: z.enum(['waiting', 'queued', 'active', 'completed', 'partially_failed', 'cancelled', 'failed']),
filters: z.any(),
created_at: z.string(),
updated_at: z.string(),
diff --git a/products/workflows/frontend/generated/api.schemas.ts b/products/workflows/frontend/generated/api.schemas.ts
index 0ce1715293ec..d6138097e788 100644
--- a/products/workflows/frontend/generated/api.schemas.ts
+++ b/products/workflows/frontend/generated/api.schemas.ts
@@ -1156,6 +1156,7 @@ export interface MessageAssetApi {
* * `queued` - Queued
* * `active` - Active
* * `completed` - Completed
+ * * `partially_failed` - Partially failed
* * `cancelled` - Cancelled
* * `failed` - Failed
*/
@@ -1166,6 +1167,7 @@ export const HogFlowBatchJobStateEnumApi = {
Queued: 'queued',
Active: 'active',
Completed: 'completed',
+ PartiallyFailed: 'partially_failed',
Cancelled: 'cancelled',
Failed: 'failed',
} as const
@@ -1173,12 +1175,13 @@ export const HogFlowBatchJobStateEnumApi = {
export interface HogFlowBatchJobApi {
/** ID of the batch run. */
readonly id: string
- /** Not currently tracked — stays at its initial value. Use the workflow logs/metrics endpoints for run outcome.
+ /** Status of the batch run. waiting: the audience's cohort is still calculating. queued and active: the run is enrolling its audience. completed: the whole audience was enrolled. When every enrolled person's run has ended, or a day after the run completed, PostHog checks for errored sends. A completed run stays completed when no send errored, becomes partially_failed when some sends errored, and becomes failed when every send errored. failed also means the run could not enroll its audience. cancelled: someone stopped the run. A send errors when its step throws, for example on a template that does not render. Provider failures such as bounces are not errored sends; read them from the run's metrics.
*
* * `waiting` - Waiting
* * `queued` - Queued
* * `active` - Active
* * `completed` - Completed
+ * * `partially_failed` - Partially failed
* * `cancelled` - Cancelled
* * `failed` - Failed */
status?: HogFlowBatchJobStateEnumApi
@@ -1208,6 +1211,7 @@ export interface HogFlowBatchJobCancelResponseApi {
* * `queued` - Queued
* * `active` - Active
* * `completed` - Completed
+ * * `partially_failed` - Partially failed
* * `cancelled` - Cancelled
* * `failed` - Failed */
status: HogFlowBatchJobStateEnumApi
@@ -2302,12 +2306,13 @@ export type SendCalendarFlowApiAudiencePropertiesItem = { [key: string]: unknown
export interface SendCalendarBatchJobApi {
/** ID of the batch run. */
readonly id: string
- /** Status of the batch run.
+ /** Status of the batch run. waiting: the audience's cohort is still calculating. queued and active: the run is enrolling its audience. completed: the whole audience was enrolled. When every enrolled person's run has ended, or a day after the run completed, PostHog checks for errored sends. A completed run stays completed when no send errored, becomes partially_failed when some sends errored, and becomes failed when every send errored. failed also means the run could not enroll its audience. cancelled: someone stopped the run. A send errors when its step throws, for example on a template that does not render. Provider failures such as bounces are not errored sends; read them from the run's metrics.
*
* * `waiting` - Waiting
* * `queued` - Queued
* * `active` - Active
* * `completed` - Completed
+ * * `partially_failed` - Partially failed
* * `cancelled` - Cancelled
* * `failed` - Failed */
readonly status: HogFlowBatchJobStateEnumApi
@@ -3122,7 +3127,7 @@ export type HogFlowsListParams = {
*/
broadcast_eligible?: boolean
/**
- * Comma-separated broadcast statuses as the broadcasts UI shows them: draft, scheduled, sending, sent, failed, archived. Scheduled, sending, sent and failed come from the latest run and whether a schedule still has sends to come.
+ * Comma-separated broadcast statuses as the broadcasts UI shows them: draft, scheduled, sending, sent, partially_sent, failed, archived. Scheduled, sending, sent, partially_sent and failed come from the latest run and whether a schedule still has sends to come. A latest run that is completed reads as sent, partially_failed as partially_sent, and failed or cancelled as failed. With a schedule still to come, a completed or partially_failed latest run reads as scheduled.
*/
broadcast_status?: string
created_at?: string
@@ -3132,7 +3137,7 @@ export type HogFlowsListParams = {
created_by?: string
id?: string
/**
- * Pass `true` to order by when each workflow's newest completed run finished, newest first. Workflows that never finished a run come last.
+ * Pass `true` to order by when each workflow's newest completed or partially_failed run finished, newest first. Workflows that never finished such a run come last.
*/
latest_send_first?: boolean
/**
diff --git a/products/workflows/frontend/generated/api.ts b/products/workflows/frontend/generated/api.ts
index c7e040850600..600619e42763 100644
--- a/products/workflows/frontend/generated/api.ts
+++ b/products/workflows/frontend/generated/api.ts
@@ -1768,7 +1768,8 @@ export const getInternalHogFlowsBatchJobsStatusUpdateUrl = (teamId: string, batc
* terminal status, returns 200 without re-writing — the resolver retries
* this call via cyclotron retry semantics, so safe repeats are required.
*
- * Accepts: { status: "completed" | "failed" }
+ * Accepts: { status: "completed" | "failed" }. A completed run can later move to
+ * failed or partially_failed, once its child runs show errored sends.
*/
export const internalHogFlowsBatchJobsStatusUpdate = async (
teamId: string,
diff --git a/products/workflows/frontend/generated/api.zod.ts b/products/workflows/frontend/generated/api.zod.ts
index 5d635dbd4acd..9c32bb91600c 100644
--- a/products/workflows/frontend/generated/api.zod.ts
+++ b/products/workflows/frontend/generated/api.zod.ts
@@ -1846,13 +1846,13 @@ export const HogFlowsActionsEmailPartialUpdateBody = /* @__PURE__ */ zod.object(
export const HogFlowsBatchJobsCreateBody = /* @__PURE__ */ zod.object({
status: zod
- .enum(['waiting', 'queued', 'active', 'completed', 'cancelled', 'failed'])
+ .enum(['waiting', 'queued', 'active', 'completed', 'partially_failed', 'cancelled', 'failed'])
.describe(
- '\* `waiting` - Waiting\n\* `queued` - Queued\n\* `active` - Active\n\* `completed` - Completed\n\* `cancelled` - Cancelled\n\* `failed` - Failed'
+ '\* `waiting` - Waiting\n\* `queued` - Queued\n\* `active` - Active\n\* `completed` - Completed\n\* `partially_failed` - Partially failed\n\* `cancelled` - Cancelled\n\* `failed` - Failed'
)
.optional()
.describe(
- 'Not currently tracked — stays at its initial value. Use the workflow logs\/metrics endpoints for run outcome.\n\n\* `waiting` - Waiting\n\* `queued` - Queued\n\* `active` - Active\n\* `completed` - Completed\n\* `cancelled` - Cancelled\n\* `failed` - Failed'
+ "Status of the batch run. waiting: the audience's cohort is still calculating. queued and active: the run is enrolling its audience. completed: the whole audience was enrolled. When every enrolled person's run has ended, or a day after the run completed, PostHog checks for errored sends. A completed run stays completed when no send errored, becomes partially_failed when some sends errored, and becomes failed when every send errored. failed also means the run could not enroll its audience. cancelled: someone stopped the run. A send errors when its step throws, for example on a template that does not render. Provider failures such as bounces are not errored sends; read them from the run's metrics.\n\n\* `waiting` - Waiting\n\* `queued` - Queued\n\* `active` - Active\n\* `completed` - Completed\n\* `partially_failed` - Partially failed\n\* `cancelled` - Cancelled\n\* `failed` - Failed"
),
hog_flow: zod.uuid().describe('ID of the workflow this batch run belongs to.'),
variables: zod.unknown().optional().describe('Variable value overrides applied to this run.'),
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index cc8a1f0a28c5..8ec4e05f695d 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -52258,6 +52258,7 @@ export namespace Schemas {
* * `queued` - Queued
* * `active` - Active
* * `completed` - Completed
+ * * `partially_failed` - Partially failed
* * `cancelled` - Cancelled
* * `failed` - Failed
*/
@@ -52269,6 +52270,7 @@ export namespace Schemas {
Queued: 'queued',
Active: 'active',
Completed: 'completed',
+ PartiallyFailed: 'partially_failed',
Cancelled: 'cancelled',
Failed: 'failed',
} as const;
@@ -52276,12 +52278,13 @@ export namespace Schemas {
export interface HogFlowBatchJob {
/** ID of the batch run. */
readonly id: string;
- /** Not currently tracked — stays at its initial value. Use the workflow logs/metrics endpoints for run outcome.
+ /** Status of the batch run. waiting: the audience's cohort is still calculating. queued and active: the run is enrolling its audience. completed: the whole audience was enrolled. When every enrolled person's run has ended, or a day after the run completed, PostHog checks for errored sends. A completed run stays completed when no send errored, becomes partially_failed when some sends errored, and becomes failed when every send errored. failed also means the run could not enroll its audience. cancelled: someone stopped the run. A send errors when its step throws, for example on a template that does not render. Provider failures such as bounces are not errored sends; read them from the run's metrics.
*
* * `waiting` - Waiting
* * `queued` - Queued
* * `active` - Active
* * `completed` - Completed
+ * * `partially_failed` - Partially failed
* * `cancelled` - Cancelled
* * `failed` - Failed */
status?: HogFlowBatchJobStateEnum;
@@ -52311,6 +52314,7 @@ export namespace Schemas {
* * `queued` - Queued
* * `active` - Active
* * `completed` - Completed
+ * * `partially_failed` - Partially failed
* * `cancelled` - Cancelled
* * `failed` - Failed */
status: HogFlowBatchJobStateEnum;
@@ -103931,12 +103935,13 @@ export namespace Schemas {
export interface SendCalendarBatchJob {
/** ID of the batch run. */
readonly id: string;
- /** Status of the batch run.
+ /** Status of the batch run. waiting: the audience's cohort is still calculating. queued and active: the run is enrolling its audience. completed: the whole audience was enrolled. When every enrolled person's run has ended, or a day after the run completed, PostHog checks for errored sends. A completed run stays completed when no send errored, becomes partially_failed when some sends errored, and becomes failed when every send errored. failed also means the run could not enroll its audience. cancelled: someone stopped the run. A send errors when its step throws, for example on a template that does not render. Provider failures such as bounces are not errored sends; read them from the run's metrics.
*
* * `waiting` - Waiting
* * `queued` - Queued
* * `active` - Active
* * `completed` - Completed
+ * * `partially_failed` - Partially failed
* * `cancelled` - Cancelled
* * `failed` - Failed */
readonly status: HogFlowBatchJobStateEnum;
@@ -126981,7 +126986,7 @@ export namespace Schemas {
*/
broadcast_eligible?: boolean;
/**
- * Comma-separated broadcast statuses as the broadcasts UI shows them: draft, scheduled, sending, sent, failed, archived. Scheduled, sending, sent and failed come from the latest run and whether a schedule still has sends to come.
+ * Comma-separated broadcast statuses as the broadcasts UI shows them: draft, scheduled, sending, sent, partially_sent, failed, archived. Scheduled, sending, sent, partially_sent and failed come from the latest run and whether a schedule still has sends to come. A latest run that is completed reads as sent, partially_failed as partially_sent, and failed or cancelled as failed. With a schedule still to come, a completed or partially_failed latest run reads as scheduled.
*/
broadcast_status?: string;
created_at?: string;
@@ -126991,7 +126996,7 @@ export namespace Schemas {
created_by?: string;
id?: string;
/**
- * Pass `true` to order by when each workflow's newest completed run finished, newest first. Workflows that never finished a run come last.
+ * Pass `true` to order by when each workflow's newest completed or partially_failed run finished, newest first. Workflows that never finished such a run come last.
*/
latest_send_first?: boolean;
/**
diff --git a/services/mcp/src/generated/workflows/api.ts b/services/mcp/src/generated/workflows/api.ts
index b2439e01fdb4..16f69ede9aff 100644
--- a/services/mcp/src/generated/workflows/api.ts
+++ b/services/mcp/src/generated/workflows/api.ts
@@ -29,7 +29,7 @@ export const HogFlowsListQueryParams = () => zod.object({
.string()
.optional()
.describe(
- 'Comma-separated broadcast statuses as the broadcasts UI shows them: draft, scheduled, sending, sent, failed, archived. Scheduled, sending, sent and failed come from the latest run and whether a schedule still has sends to come.'
+ 'Comma-separated broadcast statuses as the broadcasts UI shows them: draft, scheduled, sending, sent, partially_sent, failed, archived. Scheduled, sending, sent, partially_sent and failed come from the latest run and whether a schedule still has sends to come. A latest run that is completed reads as sent, partially_failed as partially_sent, and failed or cancelled as failed. With a schedule still to come, a completed or partially_failed latest run reads as scheduled.'
),
created_at: zod.iso.datetime({ offset: true }).optional(),
created_by: zod.string().optional().describe('Filter to workflows created by the user with this uuid.'),
@@ -38,7 +38,7 @@ export const HogFlowsListQueryParams = () => zod.object({
.boolean()
.optional()
.describe(
- "Pass `true` to order by when each workflow's newest completed run finished, newest first. Workflows that never finished a run come last."
+ "Pass `true` to order by when each workflow's newest completed or partially_failed run finished, newest first. Workflows that never finished such a run come last."
),
limit: zod.number().max(hogFlowsListQueryLimitMax).optional().describe('Number of results to return per page.'),
offset: zod.number().optional().describe('The initial index from which to return the results.'),
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/workflows-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/workflows-list.json
index 48119e241016..0bd1fd77e164 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/workflows-list.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/workflows-list.json
@@ -6,7 +6,7 @@
"type": "boolean"
},
"broadcast_status": {
- "description": "Comma-separated broadcast statuses as the broadcasts UI shows them: draft, scheduled, sending, sent, failed, archived. Scheduled, sending, sent and failed come from the latest run and whether a schedule still has sends to come.",
+ "description": "Comma-separated broadcast statuses as the broadcasts UI shows them: draft, scheduled, sending, sent, partially_sent, failed, archived. Scheduled, sending, sent, partially_sent and failed come from the latest run and whether a schedule still has sends to come. A latest run that is completed reads as sent, partially_failed as partially_sent, and failed or cancelled as failed. With a schedule still to come, a completed or partially_failed latest run reads as scheduled.",
"type": "string"
},
"created_at": {
@@ -22,7 +22,7 @@
"type": "string"
},
"latest_send_first": {
- "description": "Pass `true` to order by when each workflow's newest completed run finished, newest first. Workflows that never finished a run come last.",
+ "description": "Pass `true` to order by when each workflow's newest completed or partially_failed run finished, newest first. Workflows that never finished such a run come last.",
"type": "boolean"
},
"limit": {