Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

74.9 KB
Raw Blame
import { Message } from 'node-rdkafka'
import { Pool } from 'pg'
import { Counter, Gauge, Histogram } from 'prom-client'

import { HogFlow, HogFlowAction } from '~/cdp/schema/hogflow'
import {
    StepResume,
    WorkflowStepResumeSchema,
    counterStepResume,
    processStepResumes,
} from '~/cdp/services/hogflows/step-resume.service'
import { parseWorkflowStepDispatchKey } from '~/cdp/utils/workflow-step-dispatch-key'
import {
    KAFKA_CDP_INTERNAL_EVENTS,
    KAFKA_COHORT_MEMBERSHIP_CHANGED,
    KAFKA_EVENTS_JSON,
    KAFKA_PERSON,
    KAFKA_PERSON_DISTINCT_ID,
} from '~/common/config/kafka-topics'
import { KafkaConsumerInterface, START_AT_LATEST, createKafkaConsumer } from '~/common/kafka/consumer'
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
import { isValidTeamId } from '~/common/utils/team-manager'
import { UUIDT } from '~/common/utils/utils'

import {
    ClickHousePerson,
    ClickHousePersonDistinctId2,
    HealthCheckResult,
    PluginsServerConfig,
    RawClickHouseEvent,
    Team,
} from '../../types'
import { CdpInternalEventSchema } from '../schema'
import { COHORT_MEMBERSHIP_CHANGED_EVENT } from '../services/cohorts/cohort-membership-repository'
import {
    hasEventOrActionTarget,
    matchesWaitUntilCondition,
    runFilterBytecode,
} from '../services/hogflows/hogflow-utils'
import { CyclotronPerson, HogFlowInvocationContext, HogFunctionInvocationGlobals, PinnedConversionGoal } from '../types'
import {
    convertInternalEventToHogFunctionInvocationGlobals,
    convertToHogFunctionInvocationGlobals,
    getPersonDisplayName,
} from '../utils'
import { convertToHogFunctionFilterGlobal } from '../utils/hog-function-filtering'
import { CdpConsumerBase, CdpConsumerBaseDeps } from './cdp-base.consumer'
import { convertCohortMembershipChangeToGlobals } from './cdp-cohort-membership-trigger.consumer'
import { CohortMembershipChangeSchema } from './cdp-cohort-membership.consumer'
import { ParseBatchOptions, parseBatch, skipIfMalformed } from './message-parsing'

// A waker only needs signals that arrive after a job parks, never history. auto.offset.reset=latest
// makes a fresh consumer group start at the tip instead of replaying the (unconsumable at prod
// volume) topic backlog. Once the group has committed offsets, both consumer impls resume from them
// regardless of this value — so steady-state recovery is committed-offset resume; latest only governs
// the first bootstrap and offset-loss edge cases (covered by the deferred lag alerting follow-up).
const startAtLatest = START_AT_LATEST

// How stale the has-live-watchers gate may be. A team that just enrolled its first run while its only
// goal flow was already paused waits at most this long to be admitted.
const WATCHER_TEAMS_REFRESH_INTERVAL_MS = 30_000

const counterHogflowMatcherCandidatesEvaluated = new Counter({
    name: 'cdp_hogflow_matcher_candidates_evaluated',
    help: 'Parked hogflow jobs the matcher loaded from cyclotron and evaluated against a batch.',
})

const counterHogflowMatcherJobsWoken = new Counter({
    name: 'cdp_hogflow_matcher_jobs_woken',
    help: 'Parked hogflow jobs the matcher woke because an incoming event matched.',
})

const counterHogflowMatcherConversionsCounted = new Counter({
    name: 'cdp_hogflow_matcher_conversions_counted',
    help: "Conversions counted by claiming a run's watcher row. Deleting the row is the claim, so a run counts at most once however many matching events arrive.",
})

// A person merge repoints the merged-away person's distinct_ids at the survivor. A wait parked while
// that distinct_id belonged to the old person still references the old person_id, so the survivor's
// person/event updates (keyed on the new id) can't wake it. We re-key such waits onto the survivor;
// this counts the re-keyed jobs.
const counterHogflowMatcherJobsRekeyedOnMove = new Counter({
    name: 'cdp_hogflow_matcher_jobs_rekeyed_on_distinct_id_move',
    help: 'Parked wait_until_condition jobs re-keyed to the surviving person after a distinct_id was repointed by a merge.',
})

// A wait parked before its distinct_id had a person keeps a null person_id, and person wakes are keyed on
// person_id alone, so nothing can ever wake it — only the polling re-check advances it. The distinct_id's
// first mapping (version 0) is where that anchor becomes available. 'seen' is every first mapping admitted
// for a team with a wait flow, which is the load this path adds; 'filled' is the jobs it actually anchored.
// A 'seen' rate far above 'filled' is expected — watch the gap for query cost, not correctness.
const counterHogflowMatcherFirstMapping = new Counter({
    name: 'cdp_hogflow_matcher_first_mapping_total',
    help: 'distinct_id first mappings processed for parked waits with no person anchor, by outcome.',
    labelNames: ['outcome'],
})

const counterHogflowMatcherWatchersRekeyed = new Counter({
    name: 'cdp_hogflow_matcher_watchers_rekeyed',
    help: 'Conversion watchers repointed onto a merge survivor. Without this they would be addressed to a person no update will ever mention again.',
})

const counterHogflowMatcherConversionEventSkipped = new Counter({
    name: 'cdp_hogflow_matcher_conversion_event_skipped',
    help: 'Conversions counted as a metric but not emitted as $workflows_conversion, because the run had no distinct_id to attribute a capture event to.',
})

const counterHogflowMatcherWatchersSkipped = new Counter({
    name: 'cdp_hogflow_matcher_watchers_skipped',
    help: 'Watcher rows skipped because their pinned goal was missing or unreadable. Each one is a run that can never record a conversion.',
})

const counterHogflowMatcherWatchersEvaluated = new Counter({
    name: 'cdp_hogflow_matcher_watchers_evaluated',
    help: 'Conversion watcher rows loaded and evaluated against a batch.',
})

const gaugeHogflowMatcherTeamsWithWatchers = new Gauge({
    name: 'cdp_hogflow_matcher_teams_with_live_watchers',
    help: 'Teams admitted through the parse gate purely because they still have unexpired conversion watchers.',
})

const histogramHogflowMatcherFindWatchers = new Histogram({
    name: 'cdp_hogflow_matcher_find_watchers_seconds',
    help: 'Duration of the conversion_watchers lookup.',
    buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
})

// Latency of the cyclotron lookup for parked jobs. Watch this for cyclotron-node
// read pressure as the wait-until-event feature ramps.
const histogramHogflowMatcherFindParkedJobs = new Histogram({
    name: 'cdp_hogflow_matcher_find_parked_jobs_seconds',
    help: 'Duration of the findParkedJobs cyclotron query.',
    labelNames: ['stream'],
    buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
})

// Which Kafka stream a batch came from. Labels the find_parked_jobs metric so the person and
// internal-event query load is distinguishable from the events firehose on the shared cyclotron DB.
type WakeSource = 'events' | 'person' | 'internal_events' | 'cohort_membership'

export const counterHogflowMatcherEventSkipped = new Counter({
    name: 'cdp_hogflow_matcher_event_skipped',
    help: 'An incoming event was dropped before matching: no identifiers (distinct_id or person_id), an invalid team id, unknown team, a backfilled cohort membership change, or a captured event that uses the reserved cohort membership event name.',
    labelNames: ['reason'],
})

type ParkedCandidate = {
    id: string
    teamId: number
    functionId: string
    parentRunId: string | null
    actionId: string | null
    distinctId: string | null
    personId: string | null
}

// A distinct_id repointed by a merge: the distinct_id and the survivor person it now resolves to.
// version orders repoints for the same distinct_id — the highest wins when a batch carries several.
type PersonDistinctIdMove = {
    teamId: number
    distinctId: string
    newPersonId: string
    version: number
}

// A parked job the matcher needs to act on this batch: either resume it (stepMatched, or a
// conversion match on an exit-on-conversion flow) and/or count its conversion once. Carries the
// fields needed to emit the `conversion` metric without re-reading the hogflow.
type MatchedJob = {
    id: string
    teamId: number
    functionId: string
    parentRunId: string | null
    stepMatched: boolean
    conversionMatched: boolean
    exitsOnConversion: boolean
    // Name, UUID and timestamp of the matched event, so the resume log can name it and link to it.
    eventName?: string
    eventUuid?: string
    eventTimestamp?: string
    // Distinct id + name/uuid of the event that satisfied the conversion goal, so the matcher can
    // emit the `$workflows_conversion` PostHog event for the converting person.
    conversionDistinctId?: string
    conversionEventName?: string
    conversionEventUuid?: string
}

type FilterGlobals = ReturnType<typeof convertToHogFunctionFilterGlobal>

// Emitted by Django when a run a workflow step dispatched ends; `origin_key` names the parked job.
export const WORKFLOW_STEP_RESUME_EVENT = '$workflow_step_resume'

// Wakes parked hogflow jobs when an event matches a `wait_until_condition` step
// or a workflow conversion goal.
export class CdpHogflowSubscriptionMatcherConsumer<
    TConfig extends PluginsServerConfig = PluginsServerConfig,
> extends CdpConsumerBase<TConfig> {
    protected name = 'CdpHogflowSubscriptionMatcherConsumer'
    protected kafkaConsumer: KafkaConsumerInterface
    // clickhouse_person carries non-event person-property changes (the precalculated topic only
    // emits a per-condition match boolean, not the full property set the wait bytecode needs).
    private personKafkaConsumer: KafkaConsumerInterface
    // cdp_internal_events carries CDP-generated signals (e.g. $insight_alert_firing,
    // $activity_log_entry_created) that bypass the public capture pipeline and never reach the
    // analytics events topic. (Email engagement events, by contrast, flow through capture to
    // clickhouse_events_json and are already covered by the events stream.)
    private internalEventsKafkaConsumer: KafkaConsumerInterface
    // clickhouse_person_distinct_id carries distinct_id → person repoints (version > 0). A merge moves the
    // merged-away person's distinct_ids onto the survivor here; we consume it to re-key parked waits onto
    // the survivor's id so the survivor's person/event updates can wake them.
    private personDistinctIdKafkaConsumer: KafkaConsumerInterface
    // realtime_only_events_json carries the $feature_flag_called events that ingestion stops writing to
    // clickhouse_events_json for a FLAG_EVALUATIONS_ONLY organization.
    private realtimeOnlyEventsKafkaConsumer?: KafkaConsumerInterface
    // cohort_membership_changed carries realtime cohort joins and leaves. Each one becomes a
    // `$cohort_membership_changed` event, so event waits and event conversion goals can match it.
    private cohortMembershipKafkaConsumer: KafkaConsumerInterface
    private cyclotronPool: Pool
    private watcherTeamsRefreshTimer: NodeJS.Timeout | null = null
    // Teams with at least one unexpired watcher. Empty until the first refresh, which start() awaits
    // before consuming, so the gate is never consulted against an unpopulated set.
    private teamsWithLiveWatchers: Set<number> = new Set()

    constructor(config: TConfig, deps: CdpConsumerBaseDeps) {
        super(config, deps)
        this.kafkaConsumer = createKafkaConsumer(
            {
                groupId: 'cdp-hogflow-subscription-matcher-consumer',
                topic: KAFKA_EVENTS_JSON,
            },
            startAtLatest
        )
        this.personKafkaConsumer = createKafkaConsumer(
            {
                groupId: 'cdp-hogflow-subscription-matcher-person-consumer',
                topic: KAFKA_PERSON,
            },
            startAtLatest
        )
        this.internalEventsKafkaConsumer = createKafkaConsumer(
            {
                groupId: 'cdp-hogflow-subscription-matcher-internal-events-consumer',
                topic: KAFKA_CDP_INTERNAL_EVENTS,
            },
            {
                ...startAtLatest,
                ...(config.CDP_INTERNAL_EVENTS_CONSUMER_METADATA_BROKER_LIST
                    ? { 'metadata.broker.list': config.CDP_INTERNAL_EVENTS_CONSUMER_METADATA_BROKER_LIST }
                    : {}),
            }
        )
        this.personDistinctIdKafkaConsumer = createKafkaConsumer(
            {
                groupId: 'cdp-hogflow-subscription-matcher-person-distinct-id-consumer',
                topic: KAFKA_PERSON_DISTINCT_ID,
            },
            startAtLatest
        )
        this.cohortMembershipKafkaConsumer = createKafkaConsumer(
            {
                groupId: 'cdp-hogflow-subscription-matcher-cohort-membership-consumer',
                topic: KAFKA_COHORT_MEMBERSHIP_CHANGED,
            },
            startAtLatest
        )
        if (config.CDP_HOGFLOW_SUBSCRIPTION_MATCHER_REALTIME_ONLY_EVENTS_TOPIC) {
            this.realtimeOnlyEventsKafkaConsumer = createKafkaConsumer(
                {
                    groupId: 'cdp-hogflow-subscription-matcher-realtime-only-events-consumer',
                    topic: config.CDP_HOGFLOW_SUBSCRIPTION_MATCHER_REALTIME_ONLY_EVENTS_TOPIC,
                },
                startAtLatest
            )
        }

        // The matcher does nothing but read/write cyclotron_jobs, so a missing connection
        // string means it would silently consume the event stream and wake nothing. Fail
        // loudly on startup instead of degrading into a healthy-looking no-op.
        if (!config.CYCLOTRON_NODE_DATABASE_URL) {
            throw new Error('CdpHogflowSubscriptionMatcherConsumer requires CYCLOTRON_NODE_DATABASE_URL')
        }
        this.cyclotronPool = new Pool({
            connectionString: config.CYCLOTRON_NODE_DATABASE_URL,
            max: config.CYCLOTRON_NODE_MAX_CONNECTIONS,
        })
    }

    public async processBatch(
        invocationGlobals: HogFunctionInvocationGlobals[],
        source: WakeSource = 'events'
    ): Promise<void> {
        if (!invocationGlobals.length) {
            return
        }
        try {
            await this.wakeMatchingWorkflows(invocationGlobals, source)
        } finally {
            // Flush any `conversion` metrics and `$workflows_conversion` events queued during matching.
            // Best-effort: a flush failure must not crash the batch (which would replay the event
            // offsets). flush() is a no-op when nothing was queued, so it's safe to call unconditionally.
            await instrumentFn({ key: 'cdp.background_task.monitoring_flush', sendException: false }, async () => {
                try {
                    await Promise.all([
                        this.hogFunctionMonitoringService.flush(),
                        this.invocationResultsService.capturedEventsService.flush(),
                    ])
                } catch (err) {
                    logger.error('⚠️', 'Failed to flush hogflow matcher app metrics/events', { err })
                    captureException(err)
                }
            })
        }
    }

    @instrumented('cdpHogflowSubscriptionMatcher.wakeMatchingWorkflows')
    private async wakeMatchingWorkflows(
        invocationGlobals: HogFunctionInvocationGlobals[],
        source: WakeSource = 'events'
    ): Promise<void> {
        const indexed = indexBatch(invocationGlobals)
        const { teamIds, distinctTeamIds, distinctIds, personTeamIds, personIds, byDistinctId, byPersonId } = indexed
        if (byDistinctId.size === 0 && byPersonId.size === 0) {
            return
        }

        // Compute filterGlobals once per event; the same event can match many candidates and many
        // watchers.
        const filterGlobalsByEvent = new Map<HogFunctionInvocationGlobals, FilterGlobals>()
        const filterGlobalsFor = (g: HogFunctionInvocationGlobals): FilterGlobals => {
            let fg = filterGlobalsByEvent.get(g)
            if (!fg) {
                fg = convertToHogFunctionFilterGlobal(g)
                filterGlobalsByEvent.set(g, fg)
            }
            return fg
        }

        // A cohort membership change carries no person properties. A filter that reads the person would see
        // an empty person, so a negative filter such as "is not set" would match. Such a change is judged
        // only by event entries that do not read the person.
        const eventsOnly = source === 'cohort_membership'

        // Claimed before the parked-job work and independently of it: a watcher's run may have
        // finished long ago, so gating this on the flow still having something parked would put back
        // exactly the blind spot watchers exist to remove.
        const conversionsCounted = await this.claimConversionWatchers(indexed, filterGlobalsFor, eventsOnly)
        counterHogflowMatcherConversionsCounted.inc(conversionsCounted)

        // Build the set of actionable flows from the in-memory hogflow cache (same pattern as
        // cdp-events). Only flows with a wait_until_condition step or an event-based conversion
        // goal can ever be woken; scoping to them keeps the function_id list in findParkedJobs
        // small and skips cyclotron entirely when a batch has none.
        const hogFlowsByTeam = await this.hogFlowManager.getHogFlowsForTeams(teamIds)
        const hogflows: Record<string, HogFlow> = {}
        for (const flows of Object.values(hogFlowsByTeam)) {
            for (const flow of flows) {
                if (hasWaitUntilOrConversion(flow)) {
                    hogflows[flow.id] = flow
                }
            }
        }

        if (Object.keys(hogflows).length === 0) {
            return
        }

        const candidates = await this.findParkedJobs(
            distinctTeamIds,
            distinctIds,
            personTeamIds,
            personIds,
            Object.keys(hogflows),
            source
        )
        if (candidates.length === 0) {
            return
        }
        counterHogflowMatcherCandidatesEvaluated.inc(candidates.length)

        const matchedJobs: MatchedJob[] = []
        for (const candidate of candidates) {
            const hogflow = hogflows[candidate.functionId]
            if (!hogflow) {
                continue
            }

            const candidateGlobals = collectCandidateGlobals(candidate, byDistinctId, byPersonId)
            if (candidateGlobals.length === 0) {
                continue
            }

            const parkedOn = candidate.actionId
                ? hogflow.actions.find((a: HogFlowAction) => a.id === candidate.actionId)
                : undefined
            const action =
                eventsOnly && parkedOn?.type === 'wait_until_condition'
                    ? {
                          ...parkedOn,
                          config: {
                              ...parkedOn.config,
                              condition: { filters: null },
                              events: parkedOn.config.events?.filter(
                                  (eventConfig) => !readsPerson(eventConfig.filters?.bytecode)
                              ),
                          },
                      }
                    : parkedOn

            // Any single matching event is enough. Stop early once both flags are set.
            let stepMatched = false
            let stepMatchedEventName: string | undefined
            let stepMatchedEventUuid: string | undefined
            let stepMatchedEventTimestamp: string | undefined
            let conversionMatched = false
            let conversionDistinctId: string | undefined
            let conversionEventName: string | undefined
            let conversionEventUuid: string | undefined
            for (const globals of candidateGlobals) {
                const filterGlobals = filterGlobalsFor(globals)
                if (!stepMatched && action?.type === 'wait_until_condition') {
                    if (
                        await matchesWaitUntilCondition(action, filterGlobals, {
                            hogFlowId: hogflow.id,
                            actionId: action.id,
                        })
                    ) {
                        stepMatched = true
                        stepMatchedEventName = globals.event.event
                        stepMatchedEventUuid = globals.event.uuid
                        stepMatchedEventTimestamp = globals.event.timestamp
                    }
                }
                if (!conversionMatched && (await this.evaluateConversionEvents(hogflow, filterGlobals, eventsOnly))) {
                    conversionMatched = true
                    // Remember who converted (and via which event) so we can emit $workflows_conversion.
                    conversionDistinctId = globals.event.distinct_id
                    conversionEventName = globals.event.event
                    conversionEventUuid = globals.event.uuid
                }
                if (stepMatched && conversionMatched) {
                    break
                }
            }

            // Collect a job when its wait step matched OR a conversion matched. Conversion matches are
            // collected even on measurement-only (non-exit) flows: processMatchedJobs reads the job's
            // state under FOR UPDATE to count the conversion exactly once per run (and surface it to
            // the executor for exit-on-conversion flows). The wake-vs-count-only decision is made there.
            if (stepMatched || conversionMatched) {
                matchedJobs.push({
                    id: candidate.id,
                    teamId: candidate.teamId,
                    functionId: candidate.functionId,
                    parentRunId: candidate.parentRunId,
                    stepMatched,
                    conversionMatched,
                    exitsOnConversion: exitsOnConversion(hogflow),
                    eventName: stepMatchedEventName,
                    eventUuid: stepMatchedEventUuid,
                    eventTimestamp: stepMatchedEventTimestamp,
                    conversionDistinctId,
                    conversionEventName,
                    conversionEventUuid,
                })
            }
        }

        if (matchedJobs.length === 0) {
            return
        }

        const woken = await this.processMatchedJobs(matchedJobs)
        counterHogflowMatcherJobsWoken.inc(woken)
        logger.info('⚡', 'Processed waiting workflows from event match', {
            evaluated: candidates.length,
            matched: matchedJobs.length,
            woken,
        })
    }

    private async evaluateConversionEvents(
        hogflow: HogFlow,
        filterGlobals: FilterGlobals,
        eventsOnly: boolean
    ): Promise<boolean> {
        const conversionEvents = hogflow.conversion?.events ?? []
        const context = { hogFlowId: hogflow.id }
        for (const eventConfig of conversionEvents) {
            if (!hasEventOrActionTarget(eventConfig)) {
                continue
            }
            if (eventsOnly && readsPerson(eventConfig.filters?.bytecode)) {
                continue
            }
            if (await runFilterBytecode(eventConfig.filters?.bytecode, filterGlobals, context)) {
                return true
            }
        }
        return false
    }

    private async findParkedJobs(
        distinctTeamIds: number[],
        distinctIds: string[],
        personTeamIds: number[],
        personIds: string[],
        functionIds: string[],
        source: WakeSource
    ): Promise<ParkedCandidate[]> {
        // Two index-friendly branches with UNION (dedupes rows that match both keys).
        // A single OR across distinct_id and person_id often forces Postgres into a
        // sequential scan; splitting lets each branch hit its own composite index.
        // Each branch correlates (team_id, id) as a tuple via `unnest(teams, ids)`, which
        // zips the parallel arrays row-wise — a job only matches when its team and its id
        // came from the SAME event. Filtering team_id and distinct_id independently with
        // ANY/ANY would match a job whose team and distinct_id came from two different
        // events in the batch (a cross-team false-positive candidate). The function_id
        // filter further scopes to flows the matcher can act on.
        //
        // We deliberately do NOT filter by queue_name: a parked wait can sit on a queue
        // other than 'hogflow'. When a step (e.g. email) routes the invocation to a
        // dedicated queue, the following wait parks on that queue, so a queue_name='hogflow'
        // filter would silently miss it. function_id already scopes to hogflow jobs, and
        // waking the job (scheduled = NOW()) lets whichever worker owns that queue resume it.
        const stopTimer = histogramHogflowMatcherFindParkedJobs.startTimer({ stream: source })
        let result
        try {
            result = await this.cyclotronPool.query(
                `SELECT id, team_id, function_id, parent_run_id, action_id, distinct_id, person_id
             FROM cyclotron_jobs
             WHERE status = 'available'
               AND scheduled > NOW()
               AND function_id = ANY($5::uuid[])
               AND (team_id, distinct_id) IN (SELECT * FROM unnest($1::int[], $2::text[]))
             UNION
             SELECT id, team_id, function_id, parent_run_id, action_id, distinct_id, person_id
             FROM cyclotron_jobs
             WHERE status = 'available'
               AND scheduled > NOW()
               AND function_id = ANY($5::uuid[])
               AND (team_id, person_id) IN (SELECT * FROM unnest($3::int[], $4::text[]))`,
                [distinctTeamIds, distinctIds, personTeamIds, personIds, functionIds]
            )
        } finally {
            stopTimer()
        }

        return result.rows.map((row) => ({
            id: row.id,
            teamId: row.team_id,
            functionId: row.function_id,
            parentRunId: row.parent_run_id,
            actionId: row.action_id,
            distinctId: row.distinct_id,
            personId: row.person_id,
        }))
    }

    // Only ever resumes runs. Conversions are counted by claimConversionWatchers, which is what keeps
    // a conversion countable after the run it belongs to has finished.
    private async processMatchedJobs(matched: MatchedJob[]): Promise<number> {
        if (matched.length === 0) {
            return 0
        }

        const ids = matched.map((m) => m.id)
        // The state read-modify-write must be atomic: two matcher instances acting on the same job
        // would otherwise both read the original state and the later UPDATE would drop the earlier
        // flag. SELECT ... FOR UPDATE inside a transaction serializes them. ORDER BY id keeps lock
        // acquisition order consistent across instances so concurrent batches can't deadlock.
        const client = await this.cyclotronPool.connect()
        try {
            await client.query('BEGIN')

            const stateRows = await client.query(
                `SELECT id, state FROM cyclotron_jobs
                 WHERE id = ANY($1::uuid[]) AND status = 'available'
                 ORDER BY id
                 FOR UPDATE`,
                [ids]
            )

            const matchedById = new Map(matched.map((m) => [m.id, m]))
            // Woken jobs get `scheduled = NOW()`; the rest get a state write that leaves `scheduled`
            // untouched, so a flag persists without pulling a parked job forward.
            const wakeUpdates: { id: string; state: Buffer }[] = []
            const stateOnlyUpdates: { id: string; state: Buffer }[] = []
            for (const row of stateRows.rows) {
                if (!row.state) {
                    continue
                }
                const m = matchedById.get(row.id)
                if (!m) {
                    continue
                }
                const outcome = applyMatchToState(row.state, m)
                if (!outcome) {
                    continue
                }
                if (outcome.wake) {
                    wakeUpdates.push({ id: row.id, state: outcome.state })
                } else {
                    stateOnlyUpdates.push({ id: row.id, state: outcome.state })
                }
            }

            let woken = 0
            if (wakeUpdates.length > 0) {
                const result = await client.query(
                    `UPDATE cyclotron_jobs cj
                     SET scheduled = NOW(), state = u.state
                     FROM (
                         SELECT unnest($1::uuid[]) AS id, unnest($2::bytea[]) AS state
                     ) u
                     WHERE cj.id = u.id AND cj.status = 'available'`,
                    [wakeUpdates.map((u) => u.id), wakeUpdates.map((u) => u.state)]
                )
                woken = result.rowCount ?? 0
            }
            if (stateOnlyUpdates.length > 0) {
                await client.query(
                    `UPDATE cyclotron_jobs cj
                     SET state = u.state
                     FROM (
                         SELECT unnest($1::uuid[]) AS id, unnest($2::bytea[]) AS state
                     ) u
                     WHERE cj.id = u.id AND cj.status = 'available'`,
                    [stateOnlyUpdates.map((u) => u.id), stateOnlyUpdates.map((u) => u.state)]
                )
            }
            await client.query('COMMIT')

            return woken
        } catch (err) {
            await client.query('ROLLBACK').catch(() => {})
            throw err
        } finally {
            client.release()
        }
    }

    /**
     * Counts conversions by claiming the watcher rows of runs whose goal this batch satisfied.
     *
     * Runs against watchers rather than parked jobs, which is what lets a conversion count after the
     * run has ended — by then the run's cyclotron job is gone. Deleting the watcher IS the claim, so
     * two matcher instances racing on the same run cannot both count it, and no lock or state parse
     * is involved. Never touches cyclotron_jobs: exiting early on conversion is the run's business,
     * handled by processMatchedJobs.
     */
    private async claimConversionWatchers(
        indexed: IndexedBatch,
        filterGlobalsFor: (globals: HogFunctionInvocationGlobals) => FilterGlobals,
        eventsOnly: boolean
    ): Promise<number> {
        const { distinctTeamIds, distinctIds, personTeamIds, personIds, byDistinctId, byPersonId } = indexed

        const stopTimer = histogramHogflowMatcherFindWatchers.startTimer()
        let found
        try {
            found = await this.cyclotronPool.query(
                `SELECT id, team_id, function_id, run_id, parent_run_id, distinct_id, person_id, flow_version, goal
                 FROM conversion_watchers
                 WHERE expires_at > NOW()
                   AND (team_id, distinct_id) IN (SELECT * FROM unnest($1::int[], $2::text[]))
                 UNION
                 SELECT id, team_id, function_id, run_id, parent_run_id, distinct_id, person_id, flow_version, goal
                 FROM conversion_watchers
                 WHERE expires_at > NOW()
                   AND (team_id, person_id) IN (SELECT * FROM unnest($3::int[], $4::text[]))`,
                [distinctTeamIds, distinctIds, personTeamIds, personIds]
            )
        } finally {
            stopTimer()
        }
        if (found.rows.length === 0) {
            return 0
        }
        counterHogflowMatcherWatchersEvaluated.inc(found.rows.length)

        const converted: {
            row: any
            distinctId?: string
            conversionType: 'property' | 'event'
            eventName?: string
            eventUuid?: string
        }[] = []
        for (const row of found.rows) {
            // A watcher with no usable goal can never convert. Skipping it keeps a malformed row from
            // throwing out of the whole batch, which also carries the unrelated wake path.
            if (!row.goal || typeof row.goal !== 'object') {
                counterHogflowMatcherWatchersSkipped.inc()
                continue
            }
            const candidateGlobals = collectCandidateGlobals(
                { teamId: row.team_id, distinctId: row.distinct_id, personId: row.person_id },
                byDistinctId,
                byPersonId
            )
            const goal: PinnedConversionGoal = eventsOnly
                ? { events: row.goal.events?.filter((bytecode: unknown) => !readsPerson(bytecode)) }
                : row.goal
            for (const globals of candidateGlobals) {
                const conversionType = await matchesPinnedGoal(goal, filterGlobalsFor(globals), row.function_id)
                if (conversionType) {
                    converted.push({
                        row,
                        distinctId: globals.event.distinct_id || row.distinct_id || undefined,
                        conversionType,
                        // Only an event goal is caused by the event that matched. A property goal is
                        // detected against person state carried by whatever event delivered the update
                        // (often the synthetic $person_updated, or an unrelated event), so stamping that
                        // event would attribute the conversion to something that did not cause it.
                        eventName: conversionType === 'event' ? globals.event.event : undefined,
                        eventUuid: conversionType === 'event' ? globals.event.uuid : undefined,
                    })
                    break
                }
            }
        }
        if (converted.length === 0) {
            return 0
        }

        // The delete is the claim: only rows this statement actually removed are counted, so a
        // concurrent instance that removed the same row first counts it and we do not.
        const claimed = await this.cyclotronPool.query(
            `DELETE FROM conversion_watchers WHERE id = ANY($1::uuid[]) RETURNING id`,
            [converted.map((c) => c.row.id)]
        )
        const claimedIds = new Set(claimed.rows.map((r) => r.id))

        for (const { row, distinctId, conversionType, eventName, eventUuid } of converted) {
            if (!claimedIds.has(row.id)) {
                continue
            }
            this.hogFunctionMonitoringService.queueAppMetric(
                {
                    team_id: row.team_id,
                    // Batch children attribute to the parent batch job, matching every other run metric.
                    app_source_id: row.parent_run_id ?? row.function_id,
                    instance_id: row.function_id,
                    metric_kind: 'other',
                    metric_name: 'conversion',
                    count: 1,
                    app_source_version:
                        row.flow_version !== null ? { id: row.function_id, version: row.flow_version } : undefined,
                },
                'hog_flow'
            )
            // Surfaced as a billable event too, so conversions can power insights and cohorts. The run
            // id is what joins it back to the enrollment that earned it.
            //
            // A person-triggered run (batch audience, manual person trigger) enrolls with no
            // distinct_id, and a person-property change carries none either, so there is nothing to
            // attribute a capture event to. The app metric still counts, but any consumer reading
            // conversions from the event stream will not see these — watch the counter.
            if (!distinctId) {
                counterHogflowMatcherConversionEventSkipped.inc()
            }
            if (distinctId) {
                await this.invocationResultsService.capturedEventsService.queueEvent({
                    team_id: row.team_id,
                    event: '$workflows_conversion',
                    distinct_id: distinctId,
                    timestamp: new Date().toISOString(),
                    properties: {
                        $workflow_id: row.function_id,
                        $workflow_run_id: row.run_id,
                        $workflow_version: row.flow_version ?? undefined,
                        $workflow_conversion_type: conversionType,
                        $workflow_conversion_event: eventName,
                        $workflow_conversion_event_uuid: eventUuid,
                    },
                })
            }
        }
        return claimedIds.size
    }

    /**
     * Whether a team's messages are worth the full parse. True when it has a flow the matcher can act
     * on, or when it still has live conversion watchers.
     *
     * The watcher half matters because the flow cache is active-only. Pausing or archiving a team's
     * only goal workflow, or removing its goal, would otherwise drop every message for that team, and
     * its already-enrolled watchers would expire uncounted — silently under-reporting, and defeating
     * the point of pinning the goal to the run in the first place.
     */
    private async teamIsActionable(teamId: number): Promise<boolean> {
        const teamHogFlows = await this.hogFlowManager.getHogFlowsForTeam(teamId)
        if (teamHogFlows.some(hasWaitUntilOrConversion)) {
            return true
        }
        return this.teamsWithLiveWatchers.has(teamId)
    }

    // One small query per refresh, returning at most one row per team that has ever enrolled a run on
    // a goal workflow. Kept as a whole-set replace so a team whose watchers have all expired drops out
    // without needing its own invalidation.
    private async refreshTeamsWithLiveWatchers(): Promise<void> {
        const result = await this.cyclotronPool.query(
            `SELECT DISTINCT team_id FROM conversion_watchers WHERE expires_at > NOW()`
        )
        this.teamsWithLiveWatchers = new Set(result.rows.map((row) => row.team_id))
        gaugeHogflowMatcherTeamsWithWatchers.set(this.teamsWithLiveWatchers.size)
    }

    @instrumented('cdpHogflowSubscriptionMatcher.parseKafkaMessages')
    public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
        return await this.parseMatcherBatch(messages, 'Error parsing message', (message) => {
            const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent
            // A job can be parked by distinct_id or person_id, so an event needs at least
            // one of them to match anything. Drop only events that carry neither.
            if (!clickHouseEvent.person_id && !clickHouseEvent.distinct_id) {
                counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
                return null
            }
            // Only the cohort membership topic says who joined or left a cohort. Anyone with the
            // project token can capture an event with this name, and it would count as a conversion.
            if (clickHouseEvent.event === COHORT_MEMBERSHIP_CHANGED_EVENT) {
                counterHogflowMatcherEventSkipped.labels({ reason: 'reserved_event_name' }).inc()
                return null
            }
            return {
                teamId: clickHouseEvent.team_id,
                toGlobals: (team) => convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL),
            }
        })
    }

    @instrumented('cdpHogflowSubscriptionMatcher.parseCohortMembershipMessages')
    public async _parseCohortMembershipBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
        return await this.parseMatcherBatch(messages, 'Error parsing cohort membership change', (message) => {
            const change = CohortMembershipChangeSchema.parse(parseJSON(message.value!.toString()))
            // Seed and reconcile changes restate membership the person already had. They are not a join
            // or a leave, so they must not satisfy a wait or a conversion goal.
            if (change.origin) {
                counterHogflowMatcherEventSkipped.labels({ reason: 'cohort_membership_backfill' }).inc()
                return null
            }
            return {
                teamId: change.team_id,
                toGlobals: (team) => convertCohortMembershipChangeToGlobals(change, team, this.config.SITE_URL),
            }
        })
    }

    @instrumented('cdpHogflowSubscriptionMatcher.parsePersonMessages')
    public async _parsePersonBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
        return await this.parseMatcherBatch(messages, 'Error parsing person message', (message) => {
            const data = parseJSON(message.value!.toString()) as ClickHousePerson
            // A deleted person can never satisfy a wait, so skip tombstones.
            if (data.is_deleted) {
                return null
            }
            // Parked waits are matched by person_id (the job's person.id), so a person
            // mutation needs an id to wake anything.
            if (!data.id) {
                counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
                return null
            }
            return {
                teamId: data.team_id,
                toGlobals: (team) => convertClickhousePersonToWakeGlobals(data, team, this.config.SITE_URL),
            }
        })
    }

    private async parseMatcherBatch(
        messages: Message[],
        errorLog: string,
        parse: ParseBatchOptions['parse']
    ): Promise<HogFunctionInvocationGlobals[]> {
        return await parseBatch(messages, {
            errorLog,
            parse,
            onInvalidTeamId: () => counterHogflowMatcherEventSkipped.labels({ reason: 'invalid_team_id' }).inc(),
            // Do not catch a lookup error here. A skipped event leaves the wait or conversion goal that it
            // should satisfy to time out.
            loadTeam: async (teamId) => {
                // The vast majority of messages belong to teams with no wait_until_condition step and
                // no conversion goal. Bail on those via the in-memory hogflow cache before paying for
                // getTeam + full globals conversion.
                if (!(await this.teamIsActionable(teamId))) {
                    counterHogflowMatcherEventSkipped.labels({ reason: 'no_actionable_flow' }).inc()
                    return null
                }
                const team = await this.deps.teamManager.getTeam(teamId)
                if (!team) {
                    counterHogflowMatcherEventSkipped.labels({ reason: 'no_team' }).inc()
                }
                return team
            },
        })
    }

    // Step resumes name their job, so they skip the person and team gates the event parser applies.
    public _splitStepResumes(messages: Message[]): { resumes: StepResume[]; rest: Message[] } {
        const resumes: StepResume[] = []
        const rest: Message[] = []
        for (const message of messages) {
            let parsed
            try {
                parsed = CdpInternalEventSchema.parse(parseJSON(message.value!.toString()))
            } catch {
                rest.push(message)
                continue
            }
            if (parsed.event.event !== WORKFLOW_STEP_RESUME_EVENT) {
                rest.push(message)
                continue
            }
            const props = WorkflowStepResumeSchema.safeParse(parsed.event.properties)
            const key = props.success ? parseWorkflowStepDispatchKey(props.data.origin_key) : null
            if (!props.success || !key) {
                counterStepResume.labels({ outcome: 'parse_error' }).inc()
                continue
            }
            resumes.push({ ...props.data, ...key })
        }
        return { resumes, rest }
    }

    @instrumented('cdpHogflowSubscriptionMatcher.processStepResumes')
    public async processStepResumes(resumes: StepResume[]): Promise<void> {
        await processStepResumes(this.cyclotronPool, resumes)
    }

    @instrumented('cdpHogflowSubscriptionMatcher.parseInternalEventMessages')
    public async _parseInternalEventsBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
        return await this.parseMatcherBatch(messages, 'Error parsing internal event message', (message) => {
            const parsed = CdpInternalEventSchema.parse(parseJSON(message.value!.toString()))
            if (!parsed.event.distinct_id && !parsed.person?.id) {
                counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
                return null
            }
            return {
                teamId: parsed.team_id,
                toGlobals: (team) =>
                    convertInternalEventToHogFunctionInvocationGlobals(parsed, team, this.config.SITE_URL),
            }
        })
    }

    // Synchronous (no getTeam/globals conversion — a repoint only carries ids), so unlike the other
    // parsers this isn't async or @instrumented.
    public _parsePersonDistinctIdBatch(messages: Message[]): PersonDistinctIdMove[] {
        const moves: PersonDistinctIdMove[] = []
        for (const message of messages) {
            const move = skipIfMalformed(
                'Error parsing person distinct id message',
                (): PersonDistinctIdMove | null => {
                    const data = parseJSON(message.value!.toString()) as ClickHousePersonDistinctId2
                    // version > 0 is a repoint (identify/merge). version 0 is the distinct_id's first mapping
                    // to a person, which a wait parked before the person resolved needs to acquire its anchor:
                    // person wakes are keyed on person_id alone, so a job left with a null anchor is
                    // unwakeable by any person-property change. processMoveBatch scopes version 0 to exactly
                    // those jobs, which is what keeps this off the insert firehose.
                    if (data.version === undefined || data.version === null || data.version < 0) {
                        return null
                    }
                    // A distinct_id being deleted can't wake anything, and must never re-point a wait at a
                    // person that is going away.
                    if (data.is_deleted) {
                        return null
                    }
                    if (!data.distinct_id || !data.person_id) {
                        counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
                        return null
                    }
                    if (!isValidTeamId(data.team_id)) {
                        counterHogflowMatcherEventSkipped.labels({ reason: 'invalid_team_id' }).inc()
                        return null
                    }
                    return {
                        teamId: data.team_id,
                        distinctId: data.distinct_id,
                        newPersonId: data.person_id,
                        version: data.version,
                    }
                }
            )
            if (move) {
                moves.push(move)
            }
        }
        return moves
    }

    // Point parked wait_until_condition jobs at the person their distinct_id now belongs to, for the two
    // ways that can change: a merge repointing it onto a survivor, or the distinct_id acquiring a person
    // for the first time. Either way the job's existing anchor can no longer be woken — person and event
    // wakes are keyed on person_id — so we rewrite the anchor (column + state.personId) and wake it.
    public async processMoveBatch(moves: PersonDistinctIdMove[]): Promise<void> {
        if (moves.length === 0) {
            return
        }

        // A repoint (version > 0) may rewrite any parked wait's anchor. A first mapping (version 0) may
        // only fill an anchor that is missing: it says "this distinct_id now has a person", which tells us
        // nothing about a job already anchored elsewhere, and the null-anchor scope is what keeps the
        // insert firehose off the whole flow set.
        const repoints = moves.filter((m) => m.version > 0)
        const firstMappings = moves.filter((m) => m.version === 0)

        if (repoints.length > 0) {
            await this.applyMoves(repoints, false)
        }
        // Watchers are keyed the same two ways parked jobs are, so a merge that repoints a distinct_id
        // leaves the watcher addressed to a person nothing will ever update again. A watcher can live
        // for the whole attribution window, so it is far more likely than a parked job to outlast a
        // merge, and an unreachable one silently stops counting.
        await this.rekeyWatchers(moves)
        if (firstMappings.length > 0) {
            counterHogflowMatcherFirstMapping.labels({ outcome: 'seen' }).inc(firstMappings.length)
            await this.applyMoves(firstMappings, true)
        }
    }

    private async rekeyWatchers(moves: PersonDistinctIdMove[]): Promise<void> {
        // Highest version wins per distinct_id: repoints are not Kafka-keyed, so a batch can carry a
        // chain (anon -> A -> B) out of order, and re-keying onto an intermediate person would leave
        // the watcher just as unreachable.
        const survivorByKey = new Map<string, { personId: string; version: number }>()
        for (const move of moves) {
            const key = `${move.teamId}:${move.distinctId}`
            const existing = survivorByKey.get(key)
            if (!existing || move.version > existing.version) {
                survivorByKey.set(key, { personId: move.newPersonId, version: move.version })
            }
        }
        const teamIds: number[] = []
        const distinctIds: string[] = []
        const personIds: string[] = []
        const versions: number[] = []
        for (const [key, survivor] of survivorByKey) {
            const [teamId, ...distinctIdParts] = key.split(':')
            teamIds.push(Number(teamId))
            distinctIds.push(distinctIdParts.join(':'))
            personIds.push(survivor.personId)
            versions.push(survivor.version)
        }

        // Let a re-key failure propagate, like applyMoves and the read/wake path do: it fails the batch
        // so the offset doesn't advance and the pod replays. The UPDATE is idempotent — a no-op once
        // person_id already matches — so replay is safe. Swallowing it here instead would strand every
        // watcher in the batch on its pre-merge person until it expires, silently under-counting.
        const result = await this.cyclotronPool.query(
            `UPDATE conversion_watchers w
             SET person_id = u.person_id, person_version = u.version
             FROM (
                 SELECT unnest($1::int[]) AS team_id, unnest($2::text[]) AS distinct_id, unnest($3::text[]) AS person_id, unnest($4::int[]) AS version
             ) u
             WHERE w.team_id = u.team_id AND w.distinct_id = u.distinct_id AND w.person_id IS DISTINCT FROM u.person_id
               -- Ordering within a batch is handled above; this is the cross-batch half. A repoint that
               -- arrives after a higher-versioned one would otherwise rewind the anchor to an
               -- intermediate person, leaving the watcher unreachable by person_id until it expires.
               AND u.version > w.person_version
               -- A first mapping (version 0) only says "this distinct_id now has a person"; it may fill
               -- an empty anchor but must not overwrite one a merge already set, mirroring the
               -- onlyNullAnchor scope applyMoves uses. Repoints (version > 0) may repoint any anchor.
               AND (u.version > 0 OR w.person_id IS NULL)`,
            [teamIds, distinctIds, personIds, versions]
        )
        counterHogflowMatcherWatchersRekeyed.inc(result.rowCount ?? 0)
    }

    private async applyMoves(moves: PersonDistinctIdMove[], onlyNullAnchor: boolean): Promise<void> {
        // Only flows with a wait_until_condition step can have a parked wait to re-key. Scope the
        // function_id filter to them so the cyclotron query stays index-friendly and skips repoints for
        // teams with no such flow.
        const teamIds = [...new Set(moves.map((m) => m.teamId))]
        const hogFlowsByTeam = await this.hogFlowManager.getHogFlowsForTeams(teamIds)
        const hogflows: Record<string, HogFlow> = {}
        for (const flows of Object.values(hogFlowsByTeam)) {
            for (const flow of flows) {
                if (flow.actions.some((a: HogFlowAction) => a.type === 'wait_until_condition')) {
                    hogflows[flow.id] = flow
                }
            }
        }
        const functionIds = Object.keys(hogflows)
        if (functionIds.length === 0) {
            return
        }

        // A batch can carry several repoints for the same distinct_id (chained merges anon → A → B).
        // Repoints aren't Kafka-keyed, so they may arrive out of order within the batch — keep the
        // highest-version survivor per key rather than the last array entry, or we could re-key onto an
        // intermediate person the survivor's person-stream updates would never wake.
        const newPersonByKey = new Map<string, { personId: string; version: number }>()
        for (const move of moves) {
            const key = `${move.teamId}:${move.distinctId}`
            const existing = newPersonByKey.get(key)
            if (!existing || move.version > existing.version) {
                newPersonByKey.set(key, { personId: move.newPersonId, version: move.version })
            }
        }
        const moveTeamIds = moves.map((m) => m.teamId)
        const moveDistinctIds = moves.map((m) => m.distinctId)

        // Scope the lock + state fetch to the wait_until_condition steps we can actually re-key, so jobs
        // of the same flow parked on unrelated steps (e.g. a delay) aren't locked or shipped over the wire.
        const waitActionIds = Object.values(hogflows).flatMap((flow) =>
            flow.actions.filter((a: HogFlowAction) => a.type === 'wait_until_condition').map((a) => a.id)
        )

        let updatedCount = 0
        const client = await this.cyclotronPool.connect()
        try {
            await client.query('BEGIN')
            // Lock the parked jobs for the repointed distinct_ids. ORDER BY id keeps lock order consistent
            // with processMatchedJobs so concurrent move/match batches can't deadlock.
            const rows = await client.query(
                `SELECT id, team_id, distinct_id, function_id, action_id, state
                 FROM cyclotron_jobs
                 WHERE status = 'available'
                   AND function_id = ANY($3::uuid[])
                   AND action_id = ANY($4::text[])
                   ${onlyNullAnchor ? 'AND person_id IS NULL' : ''}
                   AND (team_id, distinct_id) IN (SELECT * FROM unnest($1::int[], $2::text[]))
                 ORDER BY id
                 FOR UPDATE`,
                [moveTeamIds, moveDistinctIds, functionIds, waitActionIds]
            )

            const updates: { id: string; personId: string; state: Buffer }[] = []
            for (const row of rows.rows) {
                // Only re-key jobs currently parked ON a wait_until_condition step. A job of the same flow
                // could be parked in a delay; rewriting its person_id off a repoint would be wrong, and it
                // must never be pulled forward.
                const action = row.action_id
                    ? hogflows[row.function_id]?.actions.find((a: HogFlowAction) => a.id === row.action_id)
                    : undefined
                if (action?.type !== 'wait_until_condition') {
                    continue
                }
                const newPerson = newPersonByKey.get(`${row.team_id}:${row.distinct_id}`)
                if (!newPerson || !row.state) {
                    continue
                }
                const newState = rewriteStatePersonId(
                    row.state,
                    newPerson.personId,
                    newPerson.version,
                    row.id,
                    onlyNullAnchor
                )
                if (!newState) {
                    continue
                }
                updates.push({ id: row.id, personId: newPerson.personId, state: newState })
            }

            if (updates.length > 0) {
                // scheduled = NOW() wakes the parked wait on the repoint itself, closing the ordering
                // race where the person-property update lands before this re-key (the matcher would
                // otherwise miss the still-anon-keyed job and the wait would hang until its poll tick).
                // Safe now that the worker resolves a re-keyed job by survivor personId (personIdRepointed).
                const result = await client.query(
                    `UPDATE cyclotron_jobs cj
                     SET person_id = u.person_id, state = u.state, scheduled = NOW()
                     FROM (
                         SELECT unnest($1::uuid[]) AS id, unnest($2::text[]) AS person_id, unnest($3::bytea[]) AS state
                     ) u
                     WHERE cj.id = u.id AND cj.status = 'available'`,
                    [updates.map((u) => u.id), updates.map((u) => u.personId), updates.map((u) => u.state)]
                )
                updatedCount = result.rowCount ?? 0
            }
            await client.query('COMMIT')
            // Counted after COMMIT: an UPDATE that succeeds and then fails to commit is rolled back, so
            // counting before this would report anchors that no job actually carries.
            if (updatedCount > 0) {
                if (onlyNullAnchor) {
                    counterHogflowMatcherFirstMapping.labels({ outcome: 'filled' }).inc(updatedCount)
                } else {
                    counterHogflowMatcherJobsRekeyedOnMove.inc(updatedCount)
                }
            }
        } catch (err) {
            await client.query('ROLLBACK').catch(() => {})
            throw err
        } finally {
            client.release()
        }
    }

    public override async start(): Promise<void> {
        await super.start()
        // Awaited so the first batches are gated against a populated set rather than an empty one,
        // which would drop messages for teams whose only goal flow is paused. A failure here must not
        // stop the matcher starting: an empty set degrades the gate to its previous flow-only
        // behavior for one refresh interval, which is milder than refusing to consume at all.
        await this.refreshTeamsWithLiveWatchers().catch((err) => {
            logger.error('⚠️', 'Initial conversion-watcher team refresh failed; gating on flows alone', { err })
            captureException(err)
        })
        this.watcherTeamsRefreshTimer = setInterval(() => {
            void this.refreshTeamsWithLiveWatchers().catch((err) => {
                logger.error('⚠️', 'Failed to refresh teams with live conversion watchers', { err })
                captureException(err)
            })
        }, WATCHER_TEAMS_REFRESH_INTERVAL_MS)
        // Not work the process should stay alive for: periodic maintenance that the next start picks
        // up. Unref'd, a consumer that was started but never stopped cannot hold the event loop open —
        // which is what a leaked one does to a long single-process test run.
        this.watcherTeamsRefreshTimer.unref()
        // Surface failures to each kafka consumer so the offset doesn't advance past a batch we
        // couldn't match. The pod will crash and replay; the SELECT is read-only and the UPDATE
        // (with `status = 'available'` guards) is idempotent, so replay is safe. All the streams
        // funnel into the same input-agnostic wakeMatchingWorkflows via processBatch.
        await Promise.all([
            this.kafkaConsumer.connect(this.eventsBatchHandler('cdpHogflowSubscriptionMatcher.handleEachBatch')),
            this.personKafkaConsumer.connect(async (messages) => {
                return await instrumentFn('cdpHogflowSubscriptionMatcher.handlePersonBatch', async () => {
                    return { backgroundTask: this.processBatch(await this._parsePersonBatch(messages), 'person') }
                })
            }),
            this.internalEventsKafkaConsumer.connect(async (messages) => {
                return await instrumentFn('cdpHogflowSubscriptionMatcher.handleInternalEventsBatch', async () => {
                    const { resumes, rest } = this._splitStepResumes(messages)
                    const events = await this._parseInternalEventsBatch(rest)
                    return {
                        backgroundTask: Promise.all([
                            this.processBatch(events, 'internal_events'),
                            this.processStepResumes(resumes),
                        ]).then(() => undefined),
                    }
                })
            }),
            this.personDistinctIdKafkaConsumer.connect(async (messages) => {
                // Parsing repoints is synchronous, so wrap in a resolved promise to satisfy instrumentFn's
                // promise-returning callback without a no-op async body.
                return await instrumentFn('cdpHogflowSubscriptionMatcher.handlePersonDistinctIdBatch', () =>
                    Promise.resolve({
                        backgroundTask: this.processMoveBatch(this._parsePersonDistinctIdBatch(messages)),
                    })
                )
            }),
            this.realtimeOnlyEventsKafkaConsumer?.connect(
                this.eventsBatchHandler('cdpHogflowSubscriptionMatcher.handleRealtimeOnlyEventsBatch')
            ),
            this.cohortMembershipKafkaConsumer.connect(async (messages) => {
                return await instrumentFn('cdpHogflowSubscriptionMatcher.handleCohortMembershipBatch', async () => ({
                    backgroundTask: this.processBatch(
                        await this._parseCohortMembershipBatch(messages),
                        'cohort_membership'
                    ),
                }))
            }),
        ])
    }

    private eventsBatchHandler(
        instrumentKey: string
    ): (messages: Message[]) => Promise<{ backgroundTask: Promise<void> }> {
        return async (messages) =>
            await instrumentFn(instrumentKey, async () => ({
                backgroundTask: this.processBatch(await this._parseKafkaBatch(messages), 'events'),
            }))
    }

    public override async stop(): Promise<void> {
        logger.info('💤', `Stopping ${this.name}...`)
        if (this.watcherTeamsRefreshTimer) {
            clearInterval(this.watcherTeamsRefreshTimer)
        }
        await Promise.all([
            this.kafkaConsumer.disconnect(),
            this.personKafkaConsumer.disconnect(),
            this.internalEventsKafkaConsumer.disconnect(),
            this.personDistinctIdKafkaConsumer.disconnect(),
            this.realtimeOnlyEventsKafkaConsumer?.disconnect(),
            this.cohortMembershipKafkaConsumer.disconnect(),
        ])
        await this.cyclotronPool.end()
        await super.stop()
        logger.info('💤', `${this.name} stopped!`)
    }

    public isHealthy(): HealthCheckResult {
        // Unhealthy if any stream is unhealthy: with polling gone, a stalled stream means dropped wakes.
        const results = [
            this.kafkaConsumer.isHealthy(),
            this.personKafkaConsumer.isHealthy(),
            this.internalEventsKafkaConsumer.isHealthy(),
            this.personDistinctIdKafkaConsumer.isHealthy(),
            ...(this.realtimeOnlyEventsKafkaConsumer ? [this.realtimeOnlyEventsKafkaConsumer.isHealthy()] : []),
            this.cohortMembershipKafkaConsumer.isHealthy(),
        ]
        return results.find((r) => r.status !== 'ok') ?? results[0]
    }
}

// Person mutations arrive on clickhouse_person with the full current property set but no triggering
// analytics event. We synthesize a `$person_updated` event so wakeMatchingWorkflows can run a wait's
// property condition against the new person.properties. Event-name `events` entries never match this
// synthetic event name, which is correct: a person change should only satisfy property-based waits.
function convertClickhousePersonToWakeGlobals(
    data: ClickHousePerson,
    team: Team,
    siteUrl: string
): HogFunctionInvocationGlobals {
    const projectUrl = `${siteUrl}/project/${team.id}`
    const properties = parseJSON(data.properties) as Record<string, any>
    const person: CyclotronPerson = {
        id: data.id,
        properties,
        name: getPersonDisplayName(team, data.id, properties),
        url: `${projectUrl}/person/${encodeURIComponent(data.id)}`,
    }
    return {
        project: { id: team.id, name: team.name, url: projectUrl },
        event: {
            uuid: new UUIDT().toString(),
            event: '$person_updated',
            // Empty so indexBatch only keys this on person_id. A person mutation has no triggering
            // distinct_id, and using the person UUID here would add a spurious (team_id, distinct_id)
            // lookup that matches a job only if its distinct_id happened to equal the UUID.
            distinct_id: '',
            properties: {},
            timestamp: data.timestamp,
            url: person.url,
            elements_chain: '',
        },
        person,
    }
}

type IndexedBatch = {
    teamIds: number[]
    // Parallel (teamId, id) pair arrays for the correlated lookup (see findParkedJobs).
    distinctTeamIds: number[]
    distinctIds: string[]
    personTeamIds: number[]
    personIds: string[]
    byDistinctId: Map<string, HogFunctionInvocationGlobals[]>
    byPersonId: Map<string, HogFunctionInvocationGlobals[]>
}

// Whether a workflow exits when its conversion goal is met. Only then does the matcher wake a
// parked job on a conversion match (so it can exit early); otherwise the conversion goal is
// measurement-only and must not perturb the job's progression.
function exitsOnConversion(hogflow: HogFlow): boolean {
    return (
        hogflow.exit_condition === 'exit_on_conversion' ||
        hogflow.exit_condition === 'exit_on_trigger_not_matched_or_conversion'
    )
}

// Skip teams whose hogflows have nothing the matcher can act on: no wait_until_condition step, and
// no event-based conversion goal. Event conversions are evaluated regardless of exit condition so
// the `conversion` metric is tracked even for flows that don't exit on conversion — only the *wake*
// decision (in wakeMatchingWorkflows) still depends on exitsOnConversion.
function hasWaitUntilOrConversion(hogflow: HogFlow): boolean {
    if (hogflow.actions.some((a: HogFlowAction) => a.type === 'wait_until_condition')) {
        return true
    }
    // Property goals count as actionable: their watchers are evaluated against person-property
    // changes, and gating on `events` alone would skip the team before those watchers were ever read,
    // leaving every property-only goal permanently at zero conversions.
    if (hogflow.conversion?.filters?.length) {
        return true
    }
    const conversionEvents = hogflow.conversion?.events
    return Array.isArray(conversionEvents) && conversionEvents.length > 0
}

// Compiled filters read `person.properties.<key>` by pushing the string 'person' as the root of the field
// chain. A literal 'person' value in a filter also matches, which only drops that entry for membership changes.
function readsPerson(bytecode: unknown): boolean {
    return Array.isArray(bytecode) && bytecode.includes('person')
}

// A pinned goal converts when the property branch matches, or any event branch does. The two are
// independent detectors of the same goal, so they are OR'd — matching the UI, where a person who
// does either is converted.
async function matchesPinnedGoal(
    goal: PinnedConversionGoal,
    filterGlobals: FilterGlobals,
    hogFlowId: string
): Promise<'property' | 'event' | null> {
    const context = { hogFlowId }
    if (goal.properties?.length && (await runFilterBytecode(goal.properties, filterGlobals, context))) {
        return 'property'
    }
    for (const bytecode of goal.events ?? []) {
        if (await runFilterBytecode(bytecode, filterGlobals, context)) {
            return 'event'
        }
    }
    return null
}

// Single pass over the batch: dedup distinct/person ids, collect team ids,
// and bucket every event under all the keys it could match a candidate by.
function indexBatch(invocationGlobals: HogFunctionInvocationGlobals[]): IndexedBatch {
    const teamIds = new Set<number>()
    const distinctTeamIds: number[] = []
    const distinctIds: string[] = []
    const personTeamIds: number[] = []
    const personIds: string[] = []
    const byDistinctId = new Map<string, HogFunctionInvocationGlobals[]>()
    const byPersonId = new Map<string, HogFunctionInvocationGlobals[]>()

    for (const globals of invocationGlobals) {
        const teamId = globals.project.id
        teamIds.add(teamId)

        const distinctId = globals.event.distinct_id
        if (distinctId) {
            const key = `${teamId}:${distinctId}`
            if (!byDistinctId.has(key)) {
                distinctTeamIds.push(teamId)
                distinctIds.push(distinctId)
            }
            pushToMap(byDistinctId, key, globals)
        }
        const personId = globals.person?.id
        if (personId) {
            const key = `${teamId}:${personId}`
            if (!byPersonId.has(key)) {
                personTeamIds.push(teamId)
                personIds.push(personId)
            }
            pushToMap(byPersonId, key, globals)
        }
    }

    return {
        teamIds: [...teamIds],
        distinctTeamIds,
        distinctIds,
        personTeamIds,
        personIds,
        byDistinctId,
        byPersonId,
    }
}

function pushToMap<K, V>(map: Map<K, V[]>, key: K, value: V): void {
    const existing = map.get(key)
    if (existing) {
        existing.push(value)
    } else {
        map.set(key, [value])
    }
}

function collectCandidateGlobals(
    candidate: Pick<ParkedCandidate, 'teamId' | 'distinctId' | 'personId'>,
    byDistinctId: Map<string, HogFunctionInvocationGlobals[]>,
    byPersonId: Map<string, HogFunctionInvocationGlobals[]>
): HogFunctionInvocationGlobals[] {
    const seen = new Set<HogFunctionInvocationGlobals>()
    if (candidate.distinctId) {
        for (const g of byDistinctId.get(`${candidate.teamId}:${candidate.distinctId}`) ?? []) {
            seen.add(g)
        }
    }
    if (candidate.personId) {
        for (const g of byPersonId.get(`${candidate.teamId}:${candidate.personId}`) ?? []) {
            seen.add(g)
        }
    }
    return [...seen]
}

// Point the persisted personId at the merge survivor so the worker's re-resolution — which falls back
// to state.personId when the job has no distinct_id — lands on the survivor. Returns the new state
// buffer, or null if the state can't be parsed (leave the row untouched).
function rewriteStatePersonId(
    stateBuffer: Buffer,
    newPersonId: string,
    newVersion: number,
    jobId: string,
    fillingNullAnchor = false
): Buffer | null {
    try {
        const parsed = parseJSON(stateBuffer.toString('utf-8'))
        // Reject a repoint no newer than the one already applied. Repoints aren't Kafka-keyed, so a
        // delayed lower-version move (anon → A, v2) can arrive in a later batch than a higher one already
        // applied (anon → B, v3); without this watermark it would rewind the wait onto the obsolete
        // person A, and B's person-stream updates would no longer address the parked job.
        // A first mapping is exempt: it carries version 0, which no watermark can beat, and there is no
        // anchor to rewind — the caller has already scoped it to jobs with none.
        const appliedVersion = parsed.state?.personIdRepointVersion ?? 0
        if (!fillingNullAnchor && newVersion <= appliedVersion) {
            return null
        }
        // Flag the re-key so the worker resolves the person by this survivor personId, not the repointed
        // distinct_id whose ~1min cache still points at the pre-merge person (see personIdRepointed).
        parsed.state = {
            ...parsed.state,
            personId: newPersonId,
            personIdRepointed: true,
            personIdRepointVersion: newVersion,
        }
        // Mark the wake so the wait handler knows the matcher woke this job rather than a polling
        // re-check. currentAction is always a wait_until_condition here (re-key scope). The two causes
        // stay separate flags: counterHogflowRekeyWake judges whether waking on a merge is wasted churn,
        // so folding first-mapping fills into it would blend two causes into one ratio.
        if (parsed.state.currentAction) {
            parsed.state.currentAction = fillingNullAnchor
                ? { ...parsed.state.currentAction, anchorWake: true }
                : { ...parsed.state.currentAction, rekeyWake: true }
        }
        return Buffer.from(JSON.stringify(parsed))
    } catch (err) {
        logger.warn('Failed to parse state during distinct_id-move re-key', { jobId, err })
        return null
    }
}

type MatchOutcome = { state: Buffer; wake: boolean }

// Applies a batch match to a parked job's state. Returns the new state plus whether the job should
// be woken (`scheduled = NOW()`) and whether its conversion should be counted this run. Returns null
// when nothing changed (e.g. the conversion was already counted and there's no wake to apply).
function applyMatchToState(stateBuffer: Buffer, m: MatchedJob): MatchOutcome | null {
    try {
        const parsed = parseJSON(stateBuffer.toString('utf-8'))
        const updatedState: HogFlowInvocationContext = { ...parsed.state }

        let changed = false
        let wake = false

        // A matched wait_until_condition step resumes the job, tagged so the resume reads as an event
        // match rather than a timeout.
        if (m.stepMatched) {
            if (updatedState.currentAction) {
                updatedState.currentAction = {
                    ...updatedState.currentAction,
                    eventMatched: true,
                    eventMatchedEvent: m.eventName,
                    eventMatchedEventUuid: m.eventUuid,
                    eventMatchedEventTimestamp: m.eventTimestamp,
                }
                changed = true
                wake = true
            } else {
                // A parked wait_until_condition job should always carry its current action. Without it
                // we cannot tag the wake as an event match - skip the flag and continue, since the
                // conversion handling above is independent of currentAction and may still apply.
                logger.warn('Skipping eventMatched: no currentAction in state', { jobId: m.id })
            }
        }

        // Only exit-on-conversion flows resume on a conversion match, so shouldExitEarly can exit. For
        // any other exit condition the goal is measurement-only and the run must not be rescheduled,
        // or we'd pull a parked job (e.g. one in a delay) forward. Counting happens on the watcher
        // either way, so a measurement-only flow needs no write here at all.
        if (m.conversionMatched && m.exitsOnConversion) {
            updatedState.conversionMatched = true
            changed = true
            wake = true
        }

        if (!changed) {
            return null
        }
        parsed.state = updatedState
        return { state: Buffer.from(JSON.stringify(parsed)), wake }
    } catch (err) {
        logger.warn('Failed to parse state during match', { jobId: m.id, err })
        return null
    }
}