Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.3 KB
Raw Blame
import { z } from 'zod'

import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
import { UUID, UUIDT } from '~/common/utils/utils'

import { RawClickHouseEvent, Team } from '../../types'
import {
    CyclotronJobInvocationHogFunction,
    CyclotronJobInvocationResult,
    HogFunctionInvocationGlobals,
    HogFunctionType,
} from '../types'
import { convertToHogFunctionInvocationGlobals } from '../utils'
import { dualWrite } from '../utils/dual-store'
import { HogConfigurationError } from '../utils/hog-error-classification'
import { createInputsFailureResult, createInvocation } from '../utils/invocation-utils'
import { HogExecutorAsyncService } from './hog-executor-async.service'
import { InvocationResultsService } from './invocation-results.service'
import { GroupsManagerService } from './managers/groups-manager.service'
import { HogFunctionManagerService } from './managers/hog-function-manager.service'
import { HogWatcherService } from './monitoring/hog-watcher.service'

// TODO: This might be too strict so we need to validate that it matches well what we would expect to get from batch exports
const batchExportRequestBodySchema = z.object({
    clickhouse_event: z.object({
        uuid: z.string(),
        event: z.string(),
        team_id: z.number(),
        distinct_id: z.string(),
        person_id: z.string().optional(),
        timestamp: z.string(),
        captured_at: z.string().nullish(),
        properties: z.string().optional(),
        elements_chain: z.string().default(''),
        person_properties: z.string().optional(),
    }),
    invocation_id: z.guid().optional(),
})

export class BatchExportHogFunctionService {
    private promiseScheduler: PromiseScheduler

    constructor(
        private siteUrl: string,
        private teamManager: TeamManager,
        private groupsManager: GroupsManagerService,
        private hogFunctionManager: HogFunctionManagerService,
        private hogExecutorAsync: HogExecutorAsyncService,
        private hogWatcher: HogWatcherService,
        private invocationResultsService: InvocationResultsService,
        private hogWatcherMirror: HogWatcherService
    ) {
        this.promiseScheduler = new PromiseScheduler()
    }

    async execute(
        params: { team_id: string; hog_function_id: string },
        body: unknown
    ): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
        const parsed = batchExportRequestBodySchema.safeParse(body)
        if (!parsed.success) {
            throw new ParseError('Invalid request body: ' + parsed.error.message)
        }

        const { clickhouse_event, invocation_id } = parsed.data
        const invocationId = invocation_id ? new UUID(invocation_id) : new UUIDT()

        let team: Team | null
        try {
            team = await this.teamManager.getTeam(parseInt(params.team_id))
        } catch {
            throw new ParseError('Invalid team_id: ' + params.team_id)
        }
        if (!team) {
            throw new NotFoundError('Missing team with id: ' + params.team_id)
        }

        const hogFunction = await this.hogFunctionManager.getHogFunction(params.hog_function_id)
        if (!hogFunction) {
            throw new NotFoundError('Missing hog function with id: ' + params.hog_function_id)
        }
        if (hogFunction.team_id !== team.id || !hogFunction.batch_export_id) {
            throw new NotFoundError('Missing hog function with id: ' + params.hog_function_id)
        }

        const globals = this.buildRequestGlobals(clickhouse_event as RawClickHouseEvent, hogFunction, team)
        await this.groupsManager.addGroupsToGlobals(globals)

        let result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>
        try {
            const globalsWithInputs = await this.hogExecutorAsync.hogExecutor.buildInputsWithGlobals(
                hogFunction,
                globals
            )
            const invocation = createInvocation(globalsWithInputs, hogFunction)
            invocation.id = invocationId.toString()

            result = await this.hogExecutorAsync.executeWithAsyncFunctions(invocation, { maxFetchRetries: 0 }) // Retries are handled by the batch export service
        } catch (error) {
            // A 500 makes the batch export retry forever, so it never reaches its error threshold.
            if (!(error instanceof HogConfigurationError)) {
                throw error
            }
            result = createInputsFailureResult(hogFunction, globals, error, invocationId.toString())
        }

        // TODO: Follow up - we might want to more accuratelt link an execution to the fact it came from a batch export
        // We have the parent_id but that overrides the function id which is not always what we want
        // Likely after v0 we will want to add an extra field or concept depending on whether it is a backfill vs a standard run

        void this.promiseScheduler.schedule(
            Promise.all([
                this.invocationResultsService.queueInvocationResultsAndFlush([result]),
                dualWrite(
                    'hog-watcher.observeResultsBuffered',
                    () => this.hogWatcher.observeResultsBuffered(result),
                    () => this.hogWatcherMirror.observeResultsBuffered(result)
                ),
            ])
        )

        return result
    }

    private buildRequestGlobals(
        event: RawClickHouseEvent,
        hogFunction: HogFunctionType,
        team: Team
    ): HogFunctionInvocationGlobals {
        const globals = convertToHogFunctionInvocationGlobals(event, team, this.siteUrl)
        const projectUrl = `${this.siteUrl}/project/${team.id}`

        return {
            ...globals,
            source: {
                id: hogFunction.id,
                name: hogFunction.name ?? `Hog function: ${hogFunction.id}`,
                url: `${projectUrl}/functions/${hogFunction.id}`,
            },
        }
    }

    public async stop(): Promise<void> {
        await this.promiseScheduler.waitForAllSettled()
    }
}

export class NotFoundError extends Error {
    constructor(message: string) {
        super(message)
        this.name = 'NotFoundError'
    }
}

export class ParseError extends Error {
    constructor(message: string) {
        super(message)
        this.name = 'ParseError'
    }
}