:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from datetime import UTC, datetime, timedelta
from typing import TypedDict
from uuid import UUID
import structlog
from celery import shared_task
from prometheus_client import Counter
from posthog.redis import get_client, redis
from posthog.scoping_audit import skip_team_scope_audit
from products.data_warehouse.backend.logic.external_data_source.notifications import (
get_team_ids_with_recent_sync_failures,
notify_external_data_sync_failures,
)
from products.managed_warehouse.backend.facade.connection import reconcile_managed_warehouse_tables
from products.managed_warehouse.backend.facade.cp_teams import (
get_org_team_membership,
list_enabled_backfill_team_memberships,
)
logger = structlog.get_logger(__name__)
# Digest tasks scheduled, by trigger: "inline" (a sync just failed, from jobs.py),
# "cdc" (a CDC failure path), or "catchup" (the daily sweep below). increase(...{trigger="catchup"}[1d]) is the
# catch-up fan-out — how many teams the daily sweep re-notifies. The inline count is
# the burst denominator: comparing it to delivered emails shows how hard the 15-min
# countdown + campaign-key dedup are collapsing bursts.
EXTERNAL_DATA_FAILURE_DIGEST_SCHEDULED_COUNTER = Counter(
"external_data_failure_digest_scheduled_total",
"External data failure digest tasks scheduled, by trigger source.",
labelnames=["trigger"],
)
# Task executions, by outcome: "processed" took the per-team lock and ran the send
# funnel; "lock_contended" lost the race to a concurrent send and skipped. A high
# lock_contended share is the per-team serialization working, not a problem.
EXTERNAL_DATA_FAILURE_DIGEST_TASK_COUNTER = Counter(
"external_data_failure_digest_task_total",
"External data failure digest task executions, by outcome.",
labelnames=["outcome"],
)
# Schemas of one source tend to fail within seconds of each other (e.g. a dead
# credential failing every schema on the next run). The digest waits this long so
# the whole burst lands in one email instead of racing the first failure's send.
EXTERNAL_DATA_FAILURE_DIGEST_DELAY_SECONDS = 15 * 60
# Generous bound on one digest build + synchronous send; the lock auto-expires
# after this if a worker dies mid-flight.
EXTERNAL_DATA_FAILURE_DIGEST_LOCK_TIMEOUT_SECONDS = 120
# A team can temporarily have dynamic, project-reader, and static compatibility
# catalogs, and each metadata query has a 10-minute database timeout. Cover the
# serial queries so another task cannot overlap catalog writes near the lock boundary.
MANAGED_WAREHOUSE_RECONCILE_LOCK_TIMEOUT_SECONDS = 30 * 60
MANAGED_WAREHOUSE_RECONCILE_INTERVAL_SECONDS = 60
MANAGED_WAREHOUSE_RECONCILE_SWEEP_SECONDS = 30 * 60
MANAGED_WAREHOUSE_RECONCILE_WAVE_SECONDS = 60
MANAGED_WAREHOUSE_RECONCILE_EXPIRY_GRACE_SECONDS = 5 * 60
class _ManagedWarehouseReconcileWaveItem(TypedDict):
team_id: int
organization_id: str
countdown: int
@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.reconcile_managed_warehouse_tables")
@skip_team_scope_audit
def reconcile_managed_warehouse_tables_task(team_id: int, organization_id: str) -> None:
try:
with get_client().lock(
f"managed_warehouse_reconcile:{team_id}",
timeout=MANAGED_WAREHOUSE_RECONCILE_LOCK_TIMEOUT_SECONDS,
blocking=False,
):
reconcile_managed_warehouse_tables(team_id=team_id, organization_id=organization_id)
except redis.exceptions.LockError:
logger.info("Managed warehouse table reconciliation already in flight", team_id=team_id)
def schedule_managed_warehouse_tables_reconcile(*, team_id: int, organization_id: str | UUID) -> None:
client = get_client()
schedule_key = f"managed_warehouse_reconcile_scheduled:{team_id}"
if not client.set(schedule_key, "1", ex=MANAGED_WAREHOUSE_RECONCILE_INTERVAL_SECONDS, nx=True):
return
try:
reconcile_managed_warehouse_tables_task.delay(team_id=team_id, organization_id=str(organization_id))
except Exception:
client.delete(schedule_key)
raise
@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.reconcile_managed_warehouse_tables_wave")
@skip_team_scope_audit
def reconcile_managed_warehouse_tables_wave_task(
items: list[_ManagedWarehouseReconcileWaveItem], wave_due_at: str
) -> None:
wave_due_datetime = datetime.fromisoformat(wave_due_at)
for item in items:
eta = wave_due_datetime + timedelta(seconds=item["countdown"])
expires = eta + timedelta(seconds=MANAGED_WAREHOUSE_RECONCILE_EXPIRY_GRACE_SECONDS)
if eta <= datetime.now(UTC):
continue
reconcile_managed_warehouse_tables_task.apply_async(
kwargs={"team_id": item["team_id"], "organization_id": item["organization_id"]},
eta=eta,
expires=expires,
)
@shared_task(
ignore_result=True,
name="products.data_warehouse.backend.tasks.soft_delete_managed_warehouse_sources",
autoretry_for=(Exception,),
retry_backoff=60,
retry_backoff_max=3600,
max_retries=10,
)
@skip_team_scope_audit
def soft_delete_managed_warehouse_sources_task(organization_id: str) -> None:
from products.managed_warehouse.backend.facade.connection import ( # noqa: PLC0415
soft_delete_legacy_managed_warehouse_sources,
)
soft_delete_legacy_managed_warehouse_sources(organization_id=organization_id)
@shared_task(
ignore_result=True,
name="products.data_warehouse.backend.tasks.soft_delete_managed_warehouse_sources_v2",
autoretry_for=(Exception,),
retry_backoff=60,
retry_backoff_max=3600,
max_retries=10,
)
@skip_team_scope_audit
def soft_delete_managed_warehouse_sources_v2_task(organization_id: str, expected_generation: int) -> None:
from products.managed_warehouse.backend.facade.connection import (
soft_delete_managed_warehouse_sources, # noqa: PLC0415
)
soft_delete_managed_warehouse_sources(
organization_id=organization_id,
expected_generation=expected_generation,
)
def schedule_soft_delete_managed_warehouse_sources(*, organization_id: str | UUID, expected_generation: int) -> None:
soft_delete_managed_warehouse_sources_v2_task.delay(
organization_id=str(organization_id),
expected_generation=expected_generation,
)
@shared_task(
ignore_result=True,
name="products.data_warehouse.backend.tasks.ensure_managed_warehouse_direct_source_v2",
autoretry_for=(Exception,),
retry_backoff=60,
retry_backoff_max=3600,
max_retries=10,
)
@skip_team_scope_audit
def ensure_managed_warehouse_direct_source_v2_task(
team_id: int,
organization_id: str,
expected_generation: int,
) -> None:
from products.managed_warehouse.backend.facade.connection import ( # noqa: PLC0415
ensure_managed_warehouse_direct_source,
)
ensure_managed_warehouse_direct_source(
team_id=team_id,
organization_id=organization_id,
expected_generation=expected_generation,
)
def schedule_managed_warehouse_direct_source_ensure(
*, team_id: int, organization_id: str | UUID, expected_generation: int
) -> None:
ensure_managed_warehouse_direct_source_v2_task.delay(
team_id=team_id,
organization_id=str(organization_id),
expected_generation=expected_generation,
)
@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.reconcile_all_managed_warehouse_tables")
@skip_team_scope_audit
def reconcile_all_managed_warehouse_tables_task() -> None:
rows = list_enabled_backfill_team_memberships()
if rows is None:
# Periodic sweep: an unreachable control plane just skips this run — the next
# scheduled sweep retries.
logger.warning("Managed warehouse reconcile sweep skipped: control plane unreachable")
return
if not rows:
return
ordered_rows = sorted(rows, key=lambda row: (row.organization_id, row.team_id))
row_count = len(ordered_rows)
sweep_started_at = datetime.now(UTC)
waves: dict[int, list[_ManagedWarehouseReconcileWaveItem]] = {}
for index, row in enumerate(ordered_rows):
countdown = index * MANAGED_WAREHOUSE_RECONCILE_SWEEP_SECONDS // row_count
wave_countdown = countdown - countdown % MANAGED_WAREHOUSE_RECONCILE_WAVE_SECONDS
waves.setdefault(wave_countdown, []).append(
{
"team_id": row.team_id,
"organization_id": row.organization_id,
"countdown": countdown - wave_countdown,
}
)
for wave_countdown, items in waves.items():
wave_due_at = sweep_started_at + timedelta(seconds=wave_countdown)
reconcile_managed_warehouse_tables_wave_task.apply_async(
kwargs={"items": items, "wave_due_at": wave_due_at.isoformat()},
countdown=wave_countdown,
expires=wave_due_at + timedelta(seconds=MANAGED_WAREHOUSE_RECONCILE_EXPIRY_GRACE_SECONDS),
)
def schedule_external_data_failure_digest(team_id: int, *, trigger: str = "inline") -> None:
"""Schedule the per-team failure digest email after the burst-collapse delay.
Safe to call on every failure: the task dedupes via a per-team lock and a
per-digest-day campaign key.
"""
send_external_data_failure_digest_task.apply_async(
args=[team_id], countdown=EXTERNAL_DATA_FAILURE_DIGEST_DELAY_SECONDS
)
EXTERNAL_DATA_FAILURE_DIGEST_SCHEDULED_COUNTER.labels(trigger=trigger).inc()
@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.send_external_data_failure_digest_task")
@skip_team_scope_audit
def send_external_data_failure_digest_task(team_id: int) -> None:
# Serialize per team: a racing task could mistake an in-flight winner's delivery
# for its own and stamp schemas the delivered email never contained, permanently
# silencing a paused schema. The loser skips instead — anything the winner didn't
# cover stays un-stamped for a later task or the catch-up.
try:
with get_client().lock(
f"external_data_failure_digest:{team_id}",
timeout=EXTERNAL_DATA_FAILURE_DIGEST_LOCK_TIMEOUT_SECONDS,
blocking=False,
):
notify_external_data_sync_failures(team_id)
EXTERNAL_DATA_FAILURE_DIGEST_TASK_COUNTER.labels(outcome="processed").inc()
except redis.exceptions.LockError:
EXTERNAL_DATA_FAILURE_DIGEST_TASK_COUNTER.labels(outcome="lock_contended").inc()
logger.info("External data failure digest already in flight for team, skipping", team_id=team_id)
@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.sync_team_earliest_event_date")
@skip_team_scope_audit # reads the duckgres control plane, not team-scoped Django models
def sync_team_earliest_event_date(team_id: int) -> None:
"""Resolve and persist a team's earliest event date after provision/onboard.
The clamped earliest event date is stored on the team's duckgres control-plane row —
the full-backfill sensor's read source. Idempotent: an already-cached date is never
recomputed. Best-effort: a failed push (or an unreachable control plane) is logged
and dropped — the full-backfill sensor resolves and persists the date lazily on a
later tick regardless.
A team with no events yet stores today's date, so its backfill history starts on the
day it is provisioned (see resolve_team_earliest_event_date).
"""
# Deferred: ducklake pulls duckdb in via common, and posthog.models must not load
# while Celery imports task modules — keep both off this module's import path.
from products.managed_warehouse.backend.facade.api import ( # noqa: PLC0415
get_org_id_for_team,
resolve_team_earliest_event_date,
update_team_earliest_event_date,
)
organization_id = get_org_id_for_team(team_id)
row = get_org_team_membership(organization_id, team_id)
if row is None:
logger.info("No duckling team row for team; skipping earliest event date sync", team_id=team_id)
return
if row.earliest_event_date is not None:
return
resolved = resolve_team_earliest_event_date(team_id)
update_team_earliest_event_date(organization_id, team_id, resolved)
@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.send_external_data_failure_digest_catchup")
@skip_team_scope_audit
def send_external_data_failure_digest_catchup() -> None:
"""Flush sync failures the one-email-per-day block swallowed.
Runs daily just after the digest day rolls over (10:15 UTC, see
EXTERNAL_DATA_DIGEST_DAY_BOUNDARY_HOUR_UTC) and the date-keyed campaign
block resets. Any team whose schemas failed in the last 24 hours and are
still failing gets a fresh digest — this guarantees every error is
eventually communicated, including schemas paused by a non-retryable error
that will never fail again to re-trigger the inline notification path.
"""
team_ids = get_team_ids_with_recent_sync_failures()
for team_id in team_ids:
send_external_data_failure_digest_task.delay(team_id)
if team_ids:
EXTERNAL_DATA_FAILURE_DIGEST_SCHEDULED_COUNTER.labels(trigger="catchup").inc(len(team_ids))
logger.info("Dispatched external data failure digest catch-up for %d teams", len(team_ids))