Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

13.1 KB
Raw Blame
from datetime import UTC, datetime, timedelta
from typing import TypedDict
from uuid import UUID

import structlog
from celery import shared_task
from prometheus_client import Counter

from posthog.redis import get_client, redis
from posthog.scoping_audit import skip_team_scope_audit

from products.data_warehouse.backend.logic.external_data_source.notifications import (
    get_team_ids_with_recent_sync_failures,
    notify_external_data_sync_failures,
)
from products.managed_warehouse.backend.facade.connection import reconcile_managed_warehouse_tables
from products.managed_warehouse.backend.facade.cp_teams import (
    get_org_team_membership,
    list_enabled_backfill_team_memberships,
)

logger = structlog.get_logger(__name__)

# Digest tasks scheduled, by trigger: "inline" (a sync just failed, from jobs.py),
# "cdc" (a CDC failure path), or "catchup" (the daily sweep below). increase(...{trigger="catchup"}[1d]) is the
# catch-up fan-out — how many teams the daily sweep re-notifies. The inline count is
# the burst denominator: comparing it to delivered emails shows how hard the 15-min
# countdown + campaign-key dedup are collapsing bursts.
EXTERNAL_DATA_FAILURE_DIGEST_SCHEDULED_COUNTER = Counter(
    "external_data_failure_digest_scheduled_total",
    "External data failure digest tasks scheduled, by trigger source.",
    labelnames=["trigger"],
)

# Task executions, by outcome: "processed" took the per-team lock and ran the send
# funnel; "lock_contended" lost the race to a concurrent send and skipped. A high
# lock_contended share is the per-team serialization working, not a problem.
EXTERNAL_DATA_FAILURE_DIGEST_TASK_COUNTER = Counter(
    "external_data_failure_digest_task_total",
    "External data failure digest task executions, by outcome.",
    labelnames=["outcome"],
)

# Schemas of one source tend to fail within seconds of each other (e.g. a dead
# credential failing every schema on the next run). The digest waits this long so
# the whole burst lands in one email instead of racing the first failure's send.
EXTERNAL_DATA_FAILURE_DIGEST_DELAY_SECONDS = 15 * 60

# Generous bound on one digest build + synchronous send; the lock auto-expires
# after this if a worker dies mid-flight.
EXTERNAL_DATA_FAILURE_DIGEST_LOCK_TIMEOUT_SECONDS = 120
# A team can temporarily have dynamic, project-reader, and static compatibility
# catalogs, and each metadata query has a 10-minute database timeout. Cover the
# serial queries so another task cannot overlap catalog writes near the lock boundary.
MANAGED_WAREHOUSE_RECONCILE_LOCK_TIMEOUT_SECONDS = 30 * 60
MANAGED_WAREHOUSE_RECONCILE_INTERVAL_SECONDS = 60
MANAGED_WAREHOUSE_RECONCILE_SWEEP_SECONDS = 30 * 60
MANAGED_WAREHOUSE_RECONCILE_WAVE_SECONDS = 60
MANAGED_WAREHOUSE_RECONCILE_EXPIRY_GRACE_SECONDS = 5 * 60


class _ManagedWarehouseReconcileWaveItem(TypedDict):
    team_id: int
    organization_id: str
    countdown: int


@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.reconcile_managed_warehouse_tables")
@skip_team_scope_audit
def reconcile_managed_warehouse_tables_task(team_id: int, organization_id: str) -> None:
    try:
        with get_client().lock(
            f"managed_warehouse_reconcile:{team_id}",
            timeout=MANAGED_WAREHOUSE_RECONCILE_LOCK_TIMEOUT_SECONDS,
            blocking=False,
        ):
            reconcile_managed_warehouse_tables(team_id=team_id, organization_id=organization_id)
    except redis.exceptions.LockError:
        logger.info("Managed warehouse table reconciliation already in flight", team_id=team_id)


def schedule_managed_warehouse_tables_reconcile(*, team_id: int, organization_id: str | UUID) -> None:
    client = get_client()
    schedule_key = f"managed_warehouse_reconcile_scheduled:{team_id}"
    if not client.set(schedule_key, "1", ex=MANAGED_WAREHOUSE_RECONCILE_INTERVAL_SECONDS, nx=True):
        return
    try:
        reconcile_managed_warehouse_tables_task.delay(team_id=team_id, organization_id=str(organization_id))
    except Exception:
        client.delete(schedule_key)
        raise


@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.reconcile_managed_warehouse_tables_wave")
@skip_team_scope_audit
def reconcile_managed_warehouse_tables_wave_task(
    items: list[_ManagedWarehouseReconcileWaveItem], wave_due_at: str
) -> None:
    wave_due_datetime = datetime.fromisoformat(wave_due_at)
    for item in items:
        eta = wave_due_datetime + timedelta(seconds=item["countdown"])
        expires = eta + timedelta(seconds=MANAGED_WAREHOUSE_RECONCILE_EXPIRY_GRACE_SECONDS)
        if eta <= datetime.now(UTC):
            continue
        reconcile_managed_warehouse_tables_task.apply_async(
            kwargs={"team_id": item["team_id"], "organization_id": item["organization_id"]},
            eta=eta,
            expires=expires,
        )


@shared_task(
    ignore_result=True,
    name="products.data_warehouse.backend.tasks.soft_delete_managed_warehouse_sources",
    autoretry_for=(Exception,),
    retry_backoff=60,
    retry_backoff_max=3600,
    max_retries=10,
)
@skip_team_scope_audit
def soft_delete_managed_warehouse_sources_task(organization_id: str) -> None:
    from products.managed_warehouse.backend.facade.connection import (  # noqa: PLC0415
        soft_delete_legacy_managed_warehouse_sources,
    )

    soft_delete_legacy_managed_warehouse_sources(organization_id=organization_id)


@shared_task(
    ignore_result=True,
    name="products.data_warehouse.backend.tasks.soft_delete_managed_warehouse_sources_v2",
    autoretry_for=(Exception,),
    retry_backoff=60,
    retry_backoff_max=3600,
    max_retries=10,
)
@skip_team_scope_audit
def soft_delete_managed_warehouse_sources_v2_task(organization_id: str, expected_generation: int) -> None:
    from products.managed_warehouse.backend.facade.connection import (
        soft_delete_managed_warehouse_sources,  # noqa: PLC0415
    )

    soft_delete_managed_warehouse_sources(
        organization_id=organization_id,
        expected_generation=expected_generation,
    )


def schedule_soft_delete_managed_warehouse_sources(*, organization_id: str | UUID, expected_generation: int) -> None:
    soft_delete_managed_warehouse_sources_v2_task.delay(
        organization_id=str(organization_id),
        expected_generation=expected_generation,
    )


@shared_task(
    ignore_result=True,
    name="products.data_warehouse.backend.tasks.ensure_managed_warehouse_direct_source_v2",
    autoretry_for=(Exception,),
    retry_backoff=60,
    retry_backoff_max=3600,
    max_retries=10,
)
@skip_team_scope_audit
def ensure_managed_warehouse_direct_source_v2_task(
    team_id: int,
    organization_id: str,
    expected_generation: int,
) -> None:
    from products.managed_warehouse.backend.facade.connection import (  # noqa: PLC0415
        ensure_managed_warehouse_direct_source,
    )

    ensure_managed_warehouse_direct_source(
        team_id=team_id,
        organization_id=organization_id,
        expected_generation=expected_generation,
    )


def schedule_managed_warehouse_direct_source_ensure(
    *, team_id: int, organization_id: str | UUID, expected_generation: int
) -> None:
    ensure_managed_warehouse_direct_source_v2_task.delay(
        team_id=team_id,
        organization_id=str(organization_id),
        expected_generation=expected_generation,
    )


@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.reconcile_all_managed_warehouse_tables")
@skip_team_scope_audit
def reconcile_all_managed_warehouse_tables_task() -> None:
    rows = list_enabled_backfill_team_memberships()
    if rows is None:
        # Periodic sweep: an unreachable control plane just skips this run — the next
        # scheduled sweep retries.
        logger.warning("Managed warehouse reconcile sweep skipped: control plane unreachable")
        return
    if not rows:
        return
    ordered_rows = sorted(rows, key=lambda row: (row.organization_id, row.team_id))
    row_count = len(ordered_rows)
    sweep_started_at = datetime.now(UTC)
    waves: dict[int, list[_ManagedWarehouseReconcileWaveItem]] = {}
    for index, row in enumerate(ordered_rows):
        countdown = index * MANAGED_WAREHOUSE_RECONCILE_SWEEP_SECONDS // row_count
        wave_countdown = countdown - countdown % MANAGED_WAREHOUSE_RECONCILE_WAVE_SECONDS
        waves.setdefault(wave_countdown, []).append(
            {
                "team_id": row.team_id,
                "organization_id": row.organization_id,
                "countdown": countdown - wave_countdown,
            }
        )
    for wave_countdown, items in waves.items():
        wave_due_at = sweep_started_at + timedelta(seconds=wave_countdown)
        reconcile_managed_warehouse_tables_wave_task.apply_async(
            kwargs={"items": items, "wave_due_at": wave_due_at.isoformat()},
            countdown=wave_countdown,
            expires=wave_due_at + timedelta(seconds=MANAGED_WAREHOUSE_RECONCILE_EXPIRY_GRACE_SECONDS),
        )


def schedule_external_data_failure_digest(team_id: int, *, trigger: str = "inline") -> None:
    """Schedule the per-team failure digest email after the burst-collapse delay.

    Safe to call on every failure: the task dedupes via a per-team lock and a
    per-digest-day campaign key.
    """
    send_external_data_failure_digest_task.apply_async(
        args=[team_id], countdown=EXTERNAL_DATA_FAILURE_DIGEST_DELAY_SECONDS
    )
    EXTERNAL_DATA_FAILURE_DIGEST_SCHEDULED_COUNTER.labels(trigger=trigger).inc()


@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.send_external_data_failure_digest_task")
@skip_team_scope_audit
def send_external_data_failure_digest_task(team_id: int) -> None:
    # Serialize per team: a racing task could mistake an in-flight winner's delivery
    # for its own and stamp schemas the delivered email never contained, permanently
    # silencing a paused schema. The loser skips instead — anything the winner didn't
    # cover stays un-stamped for a later task or the catch-up.
    try:
        with get_client().lock(
            f"external_data_failure_digest:{team_id}",
            timeout=EXTERNAL_DATA_FAILURE_DIGEST_LOCK_TIMEOUT_SECONDS,
            blocking=False,
        ):
            notify_external_data_sync_failures(team_id)
            EXTERNAL_DATA_FAILURE_DIGEST_TASK_COUNTER.labels(outcome="processed").inc()
    except redis.exceptions.LockError:
        EXTERNAL_DATA_FAILURE_DIGEST_TASK_COUNTER.labels(outcome="lock_contended").inc()
        logger.info("External data failure digest already in flight for team, skipping", team_id=team_id)


@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.sync_team_earliest_event_date")
@skip_team_scope_audit  # reads the duckgres control plane, not team-scoped Django models
def sync_team_earliest_event_date(team_id: int) -> None:
    """Resolve and persist a team's earliest event date after provision/onboard.

    The clamped earliest event date is stored on the team's duckgres control-plane row —
    the full-backfill sensor's read source. Idempotent: an already-cached date is never
    recomputed. Best-effort: a failed push (or an unreachable control plane) is logged
    and dropped — the full-backfill sensor resolves and persists the date lazily on a
    later tick regardless.

    A team with no events yet stores today's date, so its backfill history starts on the
    day it is provisioned (see resolve_team_earliest_event_date).
    """
    # Deferred: ducklake pulls duckdb in via common, and posthog.models must not load
    # while Celery imports task modules — keep both off this module's import path.
    from products.managed_warehouse.backend.facade.api import (  # noqa: PLC0415
        get_org_id_for_team,
        resolve_team_earliest_event_date,
        update_team_earliest_event_date,
    )

    organization_id = get_org_id_for_team(team_id)
    row = get_org_team_membership(organization_id, team_id)
    if row is None:
        logger.info("No duckling team row for team; skipping earliest event date sync", team_id=team_id)
        return
    if row.earliest_event_date is not None:
        return
    resolved = resolve_team_earliest_event_date(team_id)
    update_team_earliest_event_date(organization_id, team_id, resolved)


@shared_task(ignore_result=True, name="products.data_warehouse.backend.tasks.send_external_data_failure_digest_catchup")
@skip_team_scope_audit
def send_external_data_failure_digest_catchup() -> None:
    """Flush sync failures the one-email-per-day block swallowed.

    Runs daily just after the digest day rolls over (10:15 UTC, see
    EXTERNAL_DATA_DIGEST_DAY_BOUNDARY_HOUR_UTC) and the date-keyed campaign
    block resets. Any team whose schemas failed in the last 24 hours and are
    still failing gets a fresh digest — this guarantees every error is
    eventually communicated, including schemas paused by a non-retryable error
    that will never fail again to re-trigger the inline notification path.
    """
    team_ids = get_team_ids_with_recent_sync_failures()
    for team_id in team_ids:
        send_external_data_failure_digest_task.delay(team_id)

    if team_ids:
        EXTERNAL_DATA_FAILURE_DIGEST_SCHEDULED_COUNTER.labels(trigger="catchup").inc(len(team_ids))
        logger.info("Dispatched external data failure digest catch-up for %d teams", len(team_ids))