Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

16.6 KB
Raw Blame
import { Histogram } from 'prom-client'
import { ReadableStream } from 'stream/web'

import { parseJSON } from '~/common/utils/json-parse'
import { FetchOptions, FetchResponse, Response } from '~/common/utils/request'

import { LegacyPluginLogger } from '../legacy-plugins/types'
import { SEGMENT_DESTINATIONS_BY_ID } from '../segment/segment-templates'
import { CyclotronJobInvocationHogFunction, CyclotronJobInvocationResult } from '../types'
import { destinationE2eLagMsSummary } from '../utils'
import {
    CDP_TEST_ID,
    createAddLogFunction,
    getSensitiveValues,
    isSegmentPluginHogFunction,
    redactError,
    redactSensitiveValues,
} from '../utils'
import {
    CdpFetchConfig,
    cdpTrackedFetch,
    getNextRetryTime,
    isFetchResponseRetriable,
    readFetchResponseText,
} from '../utils/cdp-fetch'
import { createInvocationResult } from '../utils/invocation-utils'

const pluginExecutionDuration = new Histogram({
    name: 'cdp_segment_execution_duration_ms',
    help: 'Processing time and success status of plugins',
    // We have a timeout so we don't need to worry about much more than that
    buckets: [0, 10, 20, 50, 100, 200],
})

class SegmentFetchError extends Error {
    constructor(message?: string) {
        super(message)
    }
}

export type SegmentPluginMeta = {
    config: Record<string, any>
    global: Record<string, any>
    logger: LegacyPluginLogger
}

// The module doesn't export this so we redeclare it here
export interface ModifiedResponse<T = unknown> extends Omit<Response, 'headers'> {
    content: string
    data: unknown extends T ? undefined | unknown : T
    headers: Headers & {
        toJSON: () => Record<string, string>
    }
}

// Guards against recursing into a cyclic or pathologically deep payload
const MAX_OMIT_EMPTY_VALUES_DEPTH = 10

const isPlainObject = (value: unknown): value is Record<string, unknown> => {
    if (typeof value !== 'object' || value === null || Array.isArray(value)) {
        return false
    }
    const prototype = Object.getPrototypeOf(value)
    return prototype === Object.prototype || prototype === null
}

/**
 * Recursively drops keys whose value is null, undefined or an empty string.
 * Only plain objects are traversed so things like Date instances survive intact.
 */
const omitEmptyValues = (value: unknown, depth = 0): unknown => {
    if (depth >= MAX_OMIT_EMPTY_VALUES_DEPTH) {
        return value
    }

    if (Array.isArray(value)) {
        return value.map((item) => omitEmptyValues(item, depth + 1))
    }

    if (!isPlainObject(value)) {
        return value
    }

    const result: Record<string, unknown> = {}
    for (const [key, item] of Object.entries(value)) {
        if (item === null || item === undefined || item === '') {
            continue
        }
        result[key] = omitEmptyValues(item, depth + 1)
    }
    return result
}

const convertFetchResponse = <Data = unknown>(response: FetchResponse, text: string): ModifiedResponse<Data> => {
    const headers = new Headers() as ModifiedResponse['headers']
    Object.entries(response.headers).forEach(([key, value]) => {
        headers.set(key, value)
    })

    headers.toJSON = () => {
        return Object.fromEntries(headers.entries())
    }

    let json = undefined as Data

    try {
        json = parseJSON(text) as Data
    } catch {}

    const modifiedResponse: ModifiedResponse<Data> = {
        ...response,
        data: json,
        content: text,
        ok: response.status >= 200 && response.status < 300,
        redirected: response.status === 301 || response.status === 302,
        statusText: 'Status: ' + response.status,
        type: 'default',
        url: 'url',
        headers,
        body: new ReadableStream({
            start: (controller) => {
                controller.enqueue(new TextEncoder().encode(text))
                controller.close()
            },
        }),
        // NOTE: The majority of items below aren't used but we need to simulate their response type
        clone: () => modifiedResponse as unknown as Response,
        bodyUsed: false,
        arrayBuffer: () => {
            throw new Error('Not implemented')
        },
        bytes: () => {
            throw new Error('Not implemented')
        },
        textStream: () => {
            throw new Error('Not implemented')
        },
        blob: () => {
            throw new Error('Not implemented')
        },
        formData: () => {
            throw new Error('Not implemented')
        },
        json: () => response.json(),
    }

    return modifiedResponse
}

/**
 * NOTE: This is a consumer to take care of segment plugins.
 */

export class SegmentDestinationExecutorService {
    constructor(private serverConfig: CdpFetchConfig) {}

    public async execute(
        invocation: CyclotronJobInvocationHogFunction
    ): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
        const result = createInvocationResult<CyclotronJobInvocationHogFunction>(invocation)
        // Debug logs dump the resolved inputs and request options, which carry integration secrets and
        // credential headers. The logs reach the test API response and stored function logs.
        const sensitiveValues = getSensitiveValues(invocation.hogFunction, invocation.state.globals.inputs ?? {})
        const addLog = createAddLogFunction(result.logs, sensitiveValues)

        // Upsert the tries count on the metadata
        const metadata = (invocation.queueMetadata as { tries: number }) || { tries: 0 }
        metadata.tries = metadata.tries + 1
        result.invocation.queueMetadata = metadata

        // Indicates if a retry is possible. Once we have peformed 1 successful non-GET request, we can't retry.
        let retriesPossible = true

        const segmentDestinationId = isSegmentPluginHogFunction(invocation.hogFunction)
            ? invocation.hogFunction.template_id
            : null

        try {
            const segmentDestination = segmentDestinationId ? SEGMENT_DESTINATIONS_BY_ID[segmentDestinationId] : null

            if (!segmentDestination) {
                throw new Error(`Segment destination ${segmentDestinationId} not found`)
            }

            const isTestFunction = invocation.hogFunction.name.includes(CDP_TEST_ID)
            const start = performance.now()

            // All segment options are done as inputs
            const config = invocation.state.globals.inputs

            if (config.debug_mode) {
                addLog('debug', 'config', config)
            }

            const action = segmentDestination.destination.actions[config.internal_partner_action]

            if (!action) {
                throw new Error(`Action ${config.internal_partner_action} not found`)
            }

            await action.perform(
                // @ts-expect-error can't figure out unknown extends Data
                async (endpoint, options) => {
                    if (config.debug_mode) {
                        addLog('debug', 'endpoint', endpoint)
                    }
                    if (config.debug_mode) {
                        addLog('debug', 'options', options)
                    }
                    const requestExtension = segmentDestination.destination.extendRequest?.({
                        settings: config,
                        auth: config as any,
                        payload: config,
                    })
                    if (config.debug_mode) {
                        addLog('debug', 'requestExtension', requestExtension)
                    }
                    const headers: Record<string, string> = {
                        ...options?.headers,
                        ...requestExtension?.headers,
                    }
                    if (config.debug_mode) {
                        addLog('debug', 'headers', headers)
                    }

                    let body: string | undefined = undefined
                    if (options?.json) {
                        let jsonData = options.json

                        // For the pipedrive activities endpoint, we need to omit the id field if it's null or an empty string
                        // https://devcommunity.pipedrive.com/t/getting-bad-request-for-all-post-and-put-at-activities/345
                        if (
                            endpoint.endsWith('.pipedrive.com/api/v1/activities') &&
                            jsonData &&
                            typeof jsonData === 'object' &&
                            'id' in jsonData
                        ) {
                            const { id, ...rest } = jsonData as any
                            if (id === null || id === '') {
                                jsonData = rest
                            }
                        }

                        if (config.internal_omit_empty_values) {
                            jsonData = omitEmptyValues(jsonData) as typeof jsonData
                        }

                        body = JSON.stringify(jsonData)
                        headers['Content-Type'] = 'application/json'
                    } else if (options?.body && options.body instanceof URLSearchParams) {
                        body = options.body.toString()
                        headers['Content-Type'] = 'application/x-www-form-urlencoded'
                    } else if (options?.body && typeof options.body === 'string') {
                        body = options.body
                        headers['Content-Type'] = 'application/json'
                    }

                    const params = new URLSearchParams()
                    if (options?.searchParams && typeof options.searchParams === 'object') {
                        Object.entries(options.searchParams as Record<string, string>).forEach(([key, value]) =>
                            params.append(key, value)
                        )
                    }
                    if (requestExtension?.searchParams && typeof requestExtension.searchParams === 'object') {
                        Object.entries(requestExtension.searchParams as Record<string, string>).forEach(
                            ([key, value]) => params.append(key, value)
                        )
                    }

                    const method = options?.method?.toUpperCase() ?? 'GET'

                    const fetchOptions: FetchOptions = {
                        method,
                        headers,
                        body,
                    }

                    result.metrics!.push({
                        team_id: invocation.hogFunction.team_id,
                        app_source_id: invocation.hogFunction.id,
                        metric_kind: 'other',
                        metric_name: 'fetch',
                        count: 1,
                    })

                    if (isTestFunction && options?.method?.toUpperCase() !== 'GET') {
                        // For testing we mock out all non-GET requests
                        addLog('info', 'Fetch called but mocked due to test function', {
                            url: endpoint,
                            options: fetchOptions,
                        })
                        // Simulate a mini bit of fetch delay
                        await new Promise((resolve) => setTimeout(resolve, 200))
                        return convertFetchResponse(
                            {
                                status: 200,
                                headers: {},
                            } as FetchResponse,
                            JSON.stringify({
                                status: 'OK',
                                message: 'Test function',
                            })
                        )
                    }

                    if (config.debug_mode) {
                        addLog('debug', 'fetchOptions', fetchOptions)
                    }

                    const url = `${endpoint}${params.toString() ? '?' + params.toString() : ''}`
                    const { fetchError, fetchResponse } = await cdpTrackedFetch({
                        url,
                        fetchParams: fetchOptions,
                        templateId: invocation.hogFunction.template_id ?? '',
                        teamId: invocation.teamId,
                        hogFunctionId: invocation.hogFunction.id,
                    })

                    const fetchResponseText = await readFetchResponseText(fetchResponse)

                    if (fetchError || !fetchResponse || fetchResponse.status >= 400) {
                        if (
                            !(
                                retriesPossible &&
                                isFetchResponseRetriable(fetchResponse, fetchError) &&
                                metadata.tries < this.serverConfig.CDP_FETCH_RETRIES
                            )
                        ) {
                            retriesPossible = false
                        }
                        // Only the copies that get surfaced are masked. The destination's own code
                        // still receives the raw body below, so it can branch on what came back.
                        const reportableResponseText = redactSensitiveValues(
                            fetchResponseText ?? 'unknown',
                            sensitiveValues
                        )

                        addLog(
                            'warn',
                            `HTTP request failed with status ${
                                fetchResponse?.status
                            } (${reportableResponseText}). ${retriesPossible ? 'Scheduling retry...' : ''}`
                        )

                        // If it's retriable and we have retries left, we can trigger a retry, otherwise we just pass through to the function
                        if (retriesPossible || (options?.throwHttpErrors ?? true)) {
                            throw new SegmentFetchError(
                                `Error executing function on event ${
                                    invocation.state.globals.event.uuid
                                }: Request failed with status ${fetchResponse?.status} (${reportableResponseText})`
                            )
                        }
                    }

                    if (method !== 'GET') {
                        // If we have got to this point with anything other than a GET request, we can't retry for the risk of duplicating data
                        // as retries apply to the entire invocation, not just the http request.
                        retriesPossible = false
                    }

                    if (!fetchResponse) {
                        throw new Error('HTTP request failed')
                    }

                    const convertedResponse = convertFetchResponse(fetchResponse, fetchResponseText)
                    if (config.debug_mode) {
                        addLog(
                            'debug',
                            'convertedResponse',
                            convertedResponse.status,
                            convertedResponse.data,
                            convertedResponse.content,
                            convertedResponse.body
                        )
                    }
                    return convertedResponse
                },
                {
                    payload: config,
                    settings: config,
                }
            )

            addLog('info', `Function completed in ${performance.now() - start}ms.`)

            pluginExecutionDuration.observe(performance.now() - start)
            if (result.finished) {
                const capturedAt = invocation.state.globals.event?.captured_at
                if (capturedAt) {
                    const e2eLagMs = Date.now() - new Date(capturedAt).getTime()
                    destinationE2eLagMsSummary.observe(e2eLagMs)
                }
            }
        } catch (e) {
            // A destination's own error can quote a credential. The error reaches the test API response
            // and the stored invocation result.
            e = redactError(e, sensitiveValues)
            if (e instanceof SegmentFetchError) {
                if (retriesPossible) {
                    // We have retries left so we can trigger a retry
                    result.finished = false
                    result.invocation.queuePriority = metadata.tries
                    result.invocation.queueScheduledAt = getNextRetryTime(
                        this.serverConfig.CDP_FETCH_BACKOFF_BASE_MS,
                        this.serverConfig.CDP_FETCH_BACKOFF_MAX_MS,
                        metadata.tries
                    )
                    return result
                }
            }

            result.error = e

            addLog('error', `Function failed: ${e.message}`)
        }

        return result
    }
}