:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'
import { mutatePostIngestionEventWithElementsList } from '~/common/utils/event'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { PubSub } from '~/common/utils/pubsub'
import { TeamManager } from '~/common/utils/team-manager'
import { clickHouseTimestampSecondPrecisionToISO, clickHouseTimestampToISO } from '~/common/utils/utils'
import { Action, Hook, HookPayload, PostIngestionEvent, RawClickHouseEvent, RawKafkaEvent, Team } from '../../types'
import { assertTeamId, skipIfMalformed } from '../consumers/message-parsing'
import { ActionManager } from '../legacy-webhooks/action-manager'
import { ActionMatcher } from '../legacy-webhooks/action-matcher'
import { cdpTrackedFetch } from '../utils/cdp-fetch'
export class LegacyWebhookService {
protected actionManager: ActionManager
protected actionMatcher: ActionMatcher
constructor(
private postgres: PostgresRouter,
private teamManager: TeamManager,
pubSub: PubSub
) {
this.actionManager = new ActionManager(postgres, pubSub)
this.actionMatcher = new ActionMatcher(this.actionManager)
}
public async processEvent(event: PostIngestionEvent) {
const actionMatches = this.actionMatcher.match(event)
if (!actionMatches.length) {
return
}
await this.fireWebhooks(event, actionMatches)
}
private async fireWebhooks(event: PostIngestionEvent, actionMatches: Action[]): Promise<void> {
const team = await this.teamManager.getTeam(event.teamId)
if (!team) {
return
}
if (await this.teamManager.hasAvailableFeature(team.id, 'zapier')) {
const restHooks = actionMatches.flatMap((action) => action.hooks.map((hook) => ({ hook, action })))
if (restHooks.length > 0) {
const restHookRequests = restHooks.map(({ hook }) => this.postWebhook(event, team, hook))
await Promise.all(restHookRequests)
}
}
}
private async postWebhook(event: PostIngestionEvent, team: Team, hook: Hook): Promise<void> {
if (!hook.target) {
return
}
const body: HookPayload = {
hook: { id: hook.id, event: hook.event, target: hook.target },
data: convertToHookPayload(event),
}
logger.debug('⚠️', `Firing webhook ${hook.target} for team ${team.id}`)
const response = await cdpTrackedFetch({
url: hook.target,
fetchParams: {
method: 'POST',
body: JSON.stringify(body, null, 4),
headers: { 'Content-Type': 'application/json' },
},
templateId: 'legacy-zapier-webhook',
teamId: team.id,
})
if (response.fetchError) {
throw response.fetchError
}
if (response.fetchResponse?.status === 410) {
await this.deleteRestHook(hook.id)
}
}
private async deleteRestHook(hookId: Hook['id']): Promise<void> {
await this.postgres.query(
PostgresUse.COMMON_WRITE,
`DELETE FROM ee_hook WHERE id = $1`,
[hookId],
'deleteRestHook'
)
}
public async parseBatch(messages: Message[]): Promise<PostIngestionEvent[]> {
const events: PostIngestionEvent[] = []
await Promise.all(
messages.map(async (message) => {
const clickHouseEvent = skipIfMalformed('Error parsing message', () =>
assertTeamId(parseJSON(message.value!.toString()) as RawClickHouseEvent)
)
if (!clickHouseEvent) {
return
}
// Outside skipIfMalformed: a lookup error fails the batch, so the consumer reads the batch again.
if (
!this.actionMatcher.hasWebhooks(clickHouseEvent.team_id) ||
!(await this.teamManager.hasAvailableFeature(clickHouseEvent.team_id, 'zapier'))
) {
return
}
const event = skipIfMalformed('Error parsing message', () =>
convertToPostIngestionEvent(clickHouseEvent)
)
if (event) {
events.push(event)
}
})
)
return events
}
public processEvents(events: PostIngestionEvent[]): { backgroundTask: Promise<any> } {
return { backgroundTask: Promise.all(events.map((event) => this.processEvent(event))) }
}
public async start(): Promise<void> {
await this.actionManager.start()
}
public async stop(): Promise<void> {
logger.info('💤', 'Stopping consumer...')
await this.actionManager.stop()
}
}
function convertToPostIngestionEvent(event: RawKafkaEvent): PostIngestionEvent {
const properties = event.properties ? parseJSON(event.properties) : {}
if (event.elements_chain) {
properties['$elements_chain'] = event.elements_chain
}
return {
eventUuid: event.uuid,
event: event.event!,
teamId: event.team_id,
projectId: event.project_id,
distinctId: event.distinct_id,
properties,
timestamp: clickHouseTimestampToISO(event.timestamp),
elementsList: undefined,
person_id: event.person_id,
person_created_at: event.person_created_at
? clickHouseTimestampSecondPrecisionToISO(event.person_created_at)
: null,
person_properties: event.person_properties ? parseJSON(event.person_properties) : {},
}
}
function convertToHookPayload(event: PostIngestionEvent): HookPayload['data'] {
// It is only at this point that we need the elements list for the full event
// NOTE: It is possible that nobody uses it in which case we could remove this for performance but
// currently we have no way of being sure so we keep it in
mutatePostIngestionEventWithElementsList(event)
return {
eventUuid: event.eventUuid,
event: event.event,
teamId: event.teamId,
distinctId: event.distinctId,
properties: event.properties,
timestamp: event.timestamp,
elementsList: event.elementsList,
person: {
uuid: event.person_id!,
properties: event.person_properties,
created_at: event.person_created_at,
},
}
}