Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

2.9 KB
Raw Blame
import { Gauge } from 'prom-client'

import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'

import { WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT, WarehouseSourceWebhooksOutput } from '../../outputs/outputs'
import { CyclotronJobInvocationResult, WarehouseWebhookPayload } from '../../types'

const warehouseWebhookPendingPayloads = new Gauge({
    name: 'cdp_warehouse_webhook_pending_payloads',
    help: 'Number of warehouse webhook payloads queued and waiting to be flushed to Kafka.',
})

/**
 * Collects and flushes warehouse source webhook payloads produced by hog function
 * invocations. Lifecycle mirrors `HogFunctionMonitoringService`: callers push
 * payloads in via `queueInvocationResults` (or `queue` directly) and trigger a
 * batch emit via `flush()`.
 */
export class WarehouseWebhooksService {
    private queuedPayloads: WarehouseWebhookPayload[] = []

    constructor(private outputs: IngestionOutputs<WarehouseSourceWebhooksOutput>) {}

    queue(payloads: WarehouseWebhookPayload[]): void {
        if (payloads.length === 0) {
            return
        }
        for (const payload of payloads) {
            this.queuedPayloads.push(payload)
        }
        warehouseWebhookPendingPayloads.set(this.queuedPayloads.length)
    }

    queueInvocationResults(results: CyclotronJobInvocationResult[]): void {
        for (const result of results) {
            if (result.warehouseWebhookPayloads && result.warehouseWebhookPayloads.length > 0) {
                this.queue(result.warehouseWebhookPayloads)
            }
        }
    }

    // Required by ResultSink. This sink holds nothing of its own: it writes through the shared
    // Kafka outputs, which the server disconnects separately on shutdown.
    async stop(): Promise<void> {
        return Promise.resolve()
    }

    async flush(): Promise<void> {
        const payloads = this.queuedPayloads
        this.queuedPayloads = []
        warehouseWebhookPendingPayloads.set(0)

        if (payloads.length === 0) {
            return
        }

        await Promise.all(
            payloads.map((payload) =>
                this.outputs
                    .produce(WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT, {
                        key: Buffer.from(`${payload.team_id}:${payload.schema_id}`),
                        value: Buffer.from(
                            JSON.stringify({
                                schema_id: payload.schema_id,
                                team_id: payload.team_id,
                                payload: JSON.stringify(payload.payload),
                            })
                        ),
                    })
                    .catch((error) => {
                        logger.error('Error producing warehouse webhook payload', { error })
                        captureException(error)
                    })
            )
        )
    }
}