Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

17.1 KB
Raw Blame
"""Execute one check and record the result.

Never called in-request: every path into here comes from a Temporal activity, because a check runs
an arbitrary warehouse query and there is no response worth waiting for.

A check that cannot be compiled or executed is recorded as ``errored``, not raised. One broken
check must not take down the rest of its suite.
"""

import time
from dataclasses import dataclass, replace
from datetime import UTC, datetime
from enum import Enum, auto
from typing import TYPE_CHECKING, Any

from django.db import transaction

from posthog.hogql.context import HogQLContext
from posthog.hogql.modifiers import create_default_modifiers_for_team
from posthog.hogql.query import execute_hogql_query

if TYPE_CHECKING:
    from posthog.hogql.database.database import Database

from posthog.clickhouse.query_tagging import Feature, Product, tags_context
from posthog.exceptions_capture import capture_exception
from posthog.models.scoping import team_scope
from posthog.models.team import Team
from posthog.models.user import User

from ..facade.enums import CheckRunStatus, SubjectStatus, SubjectType, SuiteRunTrigger
from ..models import DataQualityCheck, DataQualitySuiteRun
from .compiler import compile_check, related_subject_ref
from .contracts import CompiledCheck, Evaluation, SubjectRef
from .run_records import record_check_run
from .staged_audit import StagedSubjectOverride, build_staged_database, replayable_failing_rows_query
from .subject_access import check_type_reads_beyond_subject, pin_referenced_subjects
from .subjects import resolve_subject
from .types.common import within_bounds

QUERY_TYPE = "data_quality_check"

FAILING_STATUSES = (CheckRunStatus.FAILED, CheckRunStatus.ERRORED)

STAGED_FILES_UNREADABLE = "The staged files could not be read, so this data was not audited."


class _ReferenceState(Enum):
    NOT_SUPPLIED = auto()


@dataclass(frozen=True)
class CheckOutcome:
    """What one check execution produced, before it is written down."""

    status: CheckRunStatus
    failed_row_count: int | None = None
    observed_value: float | None = None
    compiled_query: str = ""
    error: str = ""
    referenced_subjects: list[dict[str, str]] | None | _ReferenceState = _ReferenceState.NOT_SUPPLIED
    audited_staged_refresh: bool = False


def run_check(
    check: DataQualityCheck,
    suite_run: DataQualitySuiteRun,
    team: Team,
    staged: StagedSubjectOverride | None = None,
    staged_database_cache: "dict[Any, Database | None] | None" = None,
) -> CheckOutcome:
    """Compile, execute, and persist one check. Updates the check's denormalized status.

    Runs under an explicit team scope because there is no request context out here, and saving a
    check reads its own previous state through the fail-closed manager to build the activity log.
    ``staged`` redirects the subject's table at unpublished files for write-audit-publish runs (see
    logic/staged_audit); the cache is shared across a batch so each ACL posture builds one database.
    """
    started_at = datetime.now(UTC)
    monotonic_start = time.monotonic()

    try:
        outcome = _execute(check, suite_run, team, staged, staged_database_cache)
    except Exception as err:
        outcome = CheckOutcome(status=CheckRunStatus.ERRORED, error=str(err))
    outcome = replace(outcome, audited_staged_refresh=staged is not None)

    duration_ms = int((time.monotonic() - monotonic_start) * 1000)
    finished_at = datetime.now(UTC)
    with team_scope(team.id):
        _record_run(check, suite_run, outcome, started_at, finished_at, duration_ms)
        _update_check(check, outcome, finished_at)
    return outcome


def record_unrunnable_check(
    check: DataQualityCheck,
    suite_run: DataQualitySuiteRun,
    team: Team,
    reason: str,
    audited_staged_refresh: bool = False,
) -> CheckOutcome:
    """A check with no run row reads, in the health state and the API, exactly like one that passed."""
    outcome = CheckOutcome(status=CheckRunStatus.ERRORED, error=reason, audited_staged_refresh=audited_staged_refresh)
    finished_at = datetime.now(UTC)
    with team_scope(team.id):
        _record_run(check, suite_run, outcome, finished_at, finished_at, duration_ms=0)
        _update_check(check, outcome, finished_at)
    return outcome


@dataclass(frozen=True)
class _Authorization:
    """How one run's query executes against warehouse access control."""

    run_as: User | None
    bypass: bool


def _authorize(check: DataQualityCheck, suite_run: DataQualitySuiteRun) -> _Authorization | None:
    """How a run's query must execute so HogQL enforces the right warehouse access control.

    A check that reads beyond its declared subject -- ``custom_sql`` runs arbitrary HogQL,
    ``relationships`` also reads its target subject -- can otherwise be used to read a failing-row
    count over a table the caller was denied. The defense is to run the query as a user whose
    warehouse ACL HogQL will apply, so a denied object errors the check instead of leaking.

    - Manual runs execute as their initiator.
    - Automated runs (materialization, source sync) have no initiator. A check constrained to only
      its declared subject exposes nothing the definition doesn't already name, so the service
      bypass is safe. A check that reads a further subject has no such guarantee, so it executes as
      whoever last authored the definition -- the creator only until someone edits it, since after
      an edit it is the editor who was authorized against what it now reads. With nobody to
      authorize against, returning ``None`` errors the run rather than bypassing the ACL.
    """
    if suite_run.trigger == SuiteRunTrigger.MANUAL:
        if suite_run.created_by is None and _needs_a_principal(check):
            return None
        return _Authorization(run_as=suite_run.created_by, bypass=suite_run.created_by is None)
    if not _needs_a_principal(check):
        return _Authorization(run_as=None, bypass=True)
    principal = check.definition_author or check.created_by
    if principal is None:
        return None
    return _Authorization(run_as=principal, bypass=False)


def _needs_a_principal(check: DataQualityCheck) -> bool:
    """Whether the run must execute as a user rather than under the service bypass."""
    return check_type_reads_beyond_subject(check.check_type) or check.subject_type == SubjectType.POSTHOG_TABLE


def _staged_database(
    team: Team,
    staged: StagedSubjectOverride,
    authorization: _Authorization,
    cache: "dict[Any, Database | None] | None",
) -> "Database | None":
    """The staged-override database for this run's ACL posture, built once per posture per batch."""
    key = authorization.run_as.id if authorization.run_as is not None else "__bypass__"
    if cache is not None and key in cache:
        return cache[key]
    database = build_staged_database(
        team,
        staged.saved_query_id,
        staged.queryable_folder,
        user=authorization.run_as,
        bypass_warehouse_access_control=authorization.bypass,
    )
    if cache is not None:
        cache[key] = database
    return database


def staged_subject_is_reachable(
    team: Team,
    staged: StagedSubjectOverride,
    cache: "dict[Any, Database | None] | None" = None,
) -> bool:
    """Whether the staged files can be repointed at all, before any check runs.

    Independent of which user a check executes as, so a batch settles it once. Shares the runner's
    cache, so the database built here is the one the service-bypass runs reuse.
    """
    return _staged_database(team, staged, _Authorization(run_as=None, bypass=True), cache) is not None


def _execute(
    check: DataQualityCheck,
    suite_run: DataQualitySuiteRun,
    team: Team,
    staged: StagedSubjectOverride | None = None,
    staged_database_cache: "dict[Any, Database | None] | None" = None,
) -> CheckOutcome:
    # A hard-deleted subject nulls the FK; there is no id left to resolve.
    if check.subject_uuid is None:
        check.subject_status = SubjectStatus.ORPHANED
        return CheckOutcome(status=CheckRunStatus.SKIPPED, error="The subject was deleted.")

    subject = resolve_subject(team.id, check.subject_type, check.subject_uuid)
    if not subject.exists:
        check.subject_status = SubjectStatus.ORPHANED
        return CheckOutcome(status=CheckRunStatus.SKIPPED, error="The subject no longer resolves.")

    check.subject_status = SubjectStatus.ACTIVE
    check.subject_name = subject.name

    authorization = _authorize(check, suite_run)
    if authorization is None:
        return CheckOutcome(
            status=CheckRunStatus.ERRORED,
            error="A check that reads another subject needs an initiator or author to authorize its warehouse access.",
        )

    related = related_subject_ref(check.check_type, check.config)
    compiled = compile_check(
        check_type=check.check_type,
        subject=subject,
        column_name=check.column_name,
        config=check.config,
        related_subject=resolve_subject(team.id, *related) if related else None,
    )
    if staged is not None and subject.subject_type == SubjectType.METRIC:
        return CheckOutcome(status=CheckRunStatus.ERRORED, error="Metric checks cannot audit staged data.")
    if staged is not None:
        compiled = _replayable_against_sources(compiled, team, staged)
    referenced_subjects = pin_referenced_subjects(team.id, check.check_type, check.config, subject=subject)
    try:
        outcome = _execute_compiled(check, subject, compiled, team, authorization, staged, staged_database_cache)
    except Exception as err:
        outcome = CheckOutcome(
            status=CheckRunStatus.ERRORED,
            error=str(err),
            compiled_query=compiled.printed_failing_rows_query,
        )
    return replace(outcome, referenced_subjects=referenced_subjects)


def _replayable_against_sources(compiled: CompiledCheck, team: Team, staged: StagedSubjectOverride) -> CompiledCheck:
    try:
        replayable = replayable_failing_rows_query(team.pk, staged.saved_query_id, compiled.failing_rows)
    except Exception as err:
        capture_exception(err)
        replayable = None
    return replace(compiled, printed_failing_rows_query=replayable or "")


def _execute_compiled(
    check: DataQualityCheck,
    subject: SubjectRef,
    compiled: CompiledCheck,
    team: Team,
    authorization: _Authorization,
    staged: StagedSubjectOverride | None,
    staged_database_cache: "dict[Any, Database | None] | None",
) -> CheckOutcome:
    database = _staged_database(team, staged, authorization, staged_database_cache) if staged is not None else None
    if staged is not None and database is None:
        # Falling back to the unmodified database would read the live view and rule on data the
        # publish would not write, so a staged run that cannot repoint errors instead.
        return CheckOutcome(status=CheckRunStatus.ERRORED, error=STAGED_FILES_UNREADABLE)

    with tags_context(
        product=Product.DATA_QUALITY,
        feature=Feature.DATA_QUALITY_CHECK,
        data_quality_check_id=str(check.id),
        data_quality_check_type=check.check_type,
        data_quality_subject_type=subject.subject_type,
        data_quality_subject_id=subject.subject_uuid,
    ):
        # The query runs as the user whose warehouse ACL HogQL should enforce (see _authorize). The
        # service-level bypass is only used where there is no actor and the query is constrained to
        # the check's declared subject, so it can't reach a warehouse object the definition doesn't
        # already name.
        modifiers = create_default_modifiers_for_team(team)
        if database is not None:
            response = execute_hogql_query(
                query=compiled.query,
                team=team,
                query_type=QUERY_TYPE,
                modifiers=modifiers,
                context=HogQLContext(
                    team_id=team.pk,
                    user=authorization.run_as,
                    database=database,
                    modifiers=modifiers,
                    bypass_warehouse_access_control=authorization.bypass,
                ),
            )
        else:
            response = execute_hogql_query(
                query=compiled.query,
                team=team,
                query_type=QUERY_TYPE,
                user=authorization.run_as,
                modifiers=modifiers,
                bypass_warehouse_access_control=authorization.bypass,
            )
    return _interpret(compiled, check.config, response.results, response.columns or [])


def _interpret(
    compiled: CompiledCheck,
    config: dict[str, Any],
    results: list[Any] | None,
    columns: list[str],
) -> CheckOutcome:
    if not results:
        return CheckOutcome(
            status=CheckRunStatus.ERRORED,
            compiled_query=compiled.printed_failing_rows_query,
            error="The check query returned no rows.",
        )

    row = dict(zip(columns, results[0]))
    observed = _as_float(row.get("observed_value"))

    if compiled.evaluation is Evaluation.BOUNDS:
        status = (
            CheckRunStatus.PASSED
            if within_bounds(observed, config.get("min"), config.get("max"))
            else CheckRunStatus.FAILED
        )
        failed_row_count = None
    else:
        failed_row_count = int(row.get("failure_count") or 0)
        status = CheckRunStatus.PASSED if failed_row_count == 0 else CheckRunStatus.FAILED

    return CheckOutcome(
        status=status,
        failed_row_count=failed_row_count,
        observed_value=observed,
        compiled_query=compiled.printed_failing_rows_query,
    )


def _as_float(value: Any) -> float | None:
    try:
        return float(value) if value is not None else None
    except (TypeError, ValueError):
        return None


def _record_run(
    check: DataQualityCheck,
    suite_run: DataQualitySuiteRun,
    outcome: CheckOutcome,
    started_at: datetime,
    finished_at: datetime,
    duration_ms: int,
) -> None:
    if check.subject_uuid is None:
        return
    record_check_run(
        check.team_id,
        quality_check=check,
        suite_run=suite_run,
        subject_type=check.subject_type,
        subject_uuid=check.subject_uuid,
        subject_name=check.subject_name,
        check_type=check.check_type,
        check_fingerprint=check.fingerprint,
        column_name=check.column_name,
        check_config=check.config,
        check_severity=check.severity,
        referenced_subjects=(
            outcome.referenced_subjects
            if outcome.referenced_subjects is not _ReferenceState.NOT_SUPPLIED
            else pin_referenced_subjects(
                check.team_id,
                check.check_type,
                check.config,
                subject=resolve_subject(check.team_id, check.subject_type, check.subject_uuid),
            )
        ),
        status=outcome.status,
        failed_row_count=outcome.failed_row_count,
        observed_value=outcome.observed_value,
        compiled_query=outcome.compiled_query,
        audited_staged_refresh=outcome.audited_staged_refresh,
        error=outcome.error,
        duration_ms=duration_ms,
        started_at=started_at,
        finished_at=finished_at,
    )


def _update_check(check: DataQualityCheck, outcome: CheckOutcome, finished_at: datetime) -> None:
    """Stamps the check with the instant its run row carries, so failing_since matches the opening run's finished_at."""
    check.last_status = outcome.status
    check.last_run_at = finished_at
    updated = ["last_status", "last_run_at", "subject_name", "subject_status", "updated_at"]
    if outcome.status is CheckRunStatus.PASSED:
        check.last_succeeded_at = finished_at
        # Written only by the run that earned it. A failing run holds whatever this row said when its
        # batch loaded it, so listing the column unconditionally would let it overwrite a success a
        # concurrent run committed in between.
        updated.append("last_succeeded_at")
    if outcome.status not in FAILING_STATUSES:
        # Any run that did not fail ends the streak: a pass clears it, and a skipped run (its subject
        # deleted or unresolved) is not a failure to measure from. Cleared unconditionally, unlike the
        # guarded claim below, because only a failing run holds whatever this row said when its batch
        # loaded it -- so only it must defer to the conditional update rather than overwrite a state a
        # concurrent run committed in between.
        check.failing_since = None
        updated.append("failing_since")
    # One transaction, so the row lock the status write takes is held across the streak claim. Apart,
    # the two commit separately and a concurrent pass can land in the gap: the claim then matches
    # failing_since IS NULL and stamps a streak onto a row that already reads passed.
    with transaction.atomic():
        check.save(update_fields=updated)
        if outcome.status in FAILING_STATUSES:
            _claim_failing_streak(check, finished_at)


def _claim_failing_streak(check: DataQualityCheck, failed_at: datetime) -> None:
    claimed = (
        DataQualityCheck.objects.for_team(check.team_id)
        .filter(id=check.id, failing_since__isnull=True)
        .update(failing_since=failed_at)
    )
    if claimed:
        check.failing_since = failed_at
        return
    check.refresh_from_db(fields=["failing_since"])