:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { z } from 'zod'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
import { UUID, UUIDT } from '~/common/utils/utils'
import { RawClickHouseEvent, Team } from '../../types'
import {
CyclotronJobInvocationHogFunction,
CyclotronJobInvocationResult,
HogFunctionInvocationGlobals,
HogFunctionType,
} from '../types'
import { convertToHogFunctionInvocationGlobals } from '../utils'
import { dualWrite } from '../utils/dual-store'
import { HogConfigurationError } from '../utils/hog-error-classification'
import { createInputsFailureResult, createInvocation } from '../utils/invocation-utils'
import { HogExecutorAsyncService } from './hog-executor-async.service'
import { InvocationResultsService } from './invocation-results.service'
import { GroupsManagerService } from './managers/groups-manager.service'
import { HogFunctionManagerService } from './managers/hog-function-manager.service'
import { HogWatcherService } from './monitoring/hog-watcher.service'
// TODO: This might be too strict so we need to validate that it matches well what we would expect to get from batch exports
const batchExportRequestBodySchema = z.object({
clickhouse_event: z.object({
uuid: z.string(),
event: z.string(),
team_id: z.number(),
distinct_id: z.string(),
person_id: z.string().optional(),
timestamp: z.string(),
captured_at: z.string().nullish(),
properties: z.string().optional(),
elements_chain: z.string().default(''),
person_properties: z.string().optional(),
}),
invocation_id: z.guid().optional(),
})
export class BatchExportHogFunctionService {
private promiseScheduler: PromiseScheduler
constructor(
private siteUrl: string,
private teamManager: TeamManager,
private groupsManager: GroupsManagerService,
private hogFunctionManager: HogFunctionManagerService,
private hogExecutorAsync: HogExecutorAsyncService,
private hogWatcher: HogWatcherService,
private invocationResultsService: InvocationResultsService,
private hogWatcherMirror: HogWatcherService
) {
this.promiseScheduler = new PromiseScheduler()
}
async execute(
params: { team_id: string; hog_function_id: string },
body: unknown
): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
const parsed = batchExportRequestBodySchema.safeParse(body)
if (!parsed.success) {
throw new ParseError('Invalid request body: ' + parsed.error.message)
}
const { clickhouse_event, invocation_id } = parsed.data
const invocationId = invocation_id ? new UUID(invocation_id) : new UUIDT()
let team: Team | null
try {
team = await this.teamManager.getTeam(parseInt(params.team_id))
} catch {
throw new ParseError('Invalid team_id: ' + params.team_id)
}
if (!team) {
throw new NotFoundError('Missing team with id: ' + params.team_id)
}
const hogFunction = await this.hogFunctionManager.getHogFunction(params.hog_function_id)
if (!hogFunction) {
throw new NotFoundError('Missing hog function with id: ' + params.hog_function_id)
}
if (hogFunction.team_id !== team.id || !hogFunction.batch_export_id) {
throw new NotFoundError('Missing hog function with id: ' + params.hog_function_id)
}
const globals = this.buildRequestGlobals(clickhouse_event as RawClickHouseEvent, hogFunction, team)
await this.groupsManager.addGroupsToGlobals(globals)
let result: CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>
try {
const globalsWithInputs = await this.hogExecutorAsync.hogExecutor.buildInputsWithGlobals(
hogFunction,
globals
)
const invocation = createInvocation(globalsWithInputs, hogFunction)
invocation.id = invocationId.toString()
result = await this.hogExecutorAsync.executeWithAsyncFunctions(invocation, { maxFetchRetries: 0 }) // Retries are handled by the batch export service
} catch (error) {
// A 500 makes the batch export retry forever, so it never reaches its error threshold.
if (!(error instanceof HogConfigurationError)) {
throw error
}
result = createInputsFailureResult(hogFunction, globals, error, invocationId.toString())
}
// TODO: Follow up - we might want to more accuratelt link an execution to the fact it came from a batch export
// We have the parent_id but that overrides the function id which is not always what we want
// Likely after v0 we will want to add an extra field or concept depending on whether it is a backfill vs a standard run
void this.promiseScheduler.schedule(
Promise.all([
this.invocationResultsService.queueInvocationResultsAndFlush([result]),
dualWrite(
'hog-watcher.observeResultsBuffered',
() => this.hogWatcher.observeResultsBuffered(result),
() => this.hogWatcherMirror.observeResultsBuffered(result)
),
])
)
return result
}
private buildRequestGlobals(
event: RawClickHouseEvent,
hogFunction: HogFunctionType,
team: Team
): HogFunctionInvocationGlobals {
const globals = convertToHogFunctionInvocationGlobals(event, team, this.siteUrl)
const projectUrl = `${this.siteUrl}/project/${team.id}`
return {
...globals,
source: {
id: hogFunction.id,
name: hogFunction.name ?? `Hog function: ${hogFunction.id}`,
url: `${projectUrl}/functions/${hogFunction.id}`,
},
}
}
public async stop(): Promise<void> {
await this.promiseScheduler.waitForAllSettled()
}
}
export class NotFoundError extends Error {
constructor(message: string) {
super(message)
this.name = 'NotFoundError'
}
}
export class ParseError extends Error {
constructor(message: string) {
super(message)
this.name = 'ParseError'
}
}