Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

13.0 KB
Raw Blame
/* eslint-disable @typescript-eslint/no-var-requires */
// The native addon is built from `src/lib.rs` and copied to `index.node` at the package root.
const native = require('../index.node')

export interface AllowListsInput {
    /** Words kept verbatim by the text scrubber (ASCII-case-insensitive). */
    text: string[]
    /** URL path segments/params kept verbatim by the URL scrubber. */
    url: string[]
}

/** Per emitted JSONL line, in line order. */
export interface AnonymizeEventMeta {
    /** The event's `timestamp` (epoch ms; can be fractional). */
    ts: number
    /** Bitmask of the FLAG_* bits in `snapshot.rs` (mirrored as PRE_SERIALIZED_FLAG_* in the consumer). */
    flags: number
    /** Post-scrub `hrefFrom(event)` (`data.href` / `data.payload.href`, trimmed), when present. */
    href?: string
    jsonLd?: { rootTypes: string[]; fullSnapshotTimestamp?: number }
}

/** One collected original image: `offset..offset+len` in {@link AnonymizeKafkaPayloadResult.images}. */
export interface AnonymizeImageEntry {
    /** First 22 base64url chars of `HMAC-SHA256(contentKey, bytes)` (`hashImageBytes` in content-ref.ts). */
    hash: string
    offset: number
    len: number
}

/** One collected remote image URL, ready for the fetch lane. */
export interface AnonymizeUrlEntry {
    /** First 22 base64url chars of `HMAC-SHA256(urlKey, dedupUrl)`, where the dedup URL is the
     *  canonical URL minus its volatile parameters. The namespaced ref attribute ends with this. */
    hash: string
    /** The canonical URL with every parameter intact — what the fetcher requests. A signed URL only
     *  works in this form, which is why it is not the value the hash was taken over. */
    url: string
    /** The host the request goes to. robots.txt and the connection limit are scoped to this. */
    host: string
    /** The registrable domain of `host`. The fetch topic keys on this, so every URL of one operator
     *  lands on one partition and one pod holds its rate budget without a distributed lock. */
    domain: string
}

export interface AnonymizeImageSourceCount {
    source: 'css' | 'html'
    property: string
    kind: 'inline' | 'url'
    count: number
}

/** Envelope + per-event metadata parsed from {@link AnonymizeKafkaPayloadResult.meta}. */
export interface AnonymizeMeta {
    distinctId: string
    /** Raw `$session_id` — normalization stays in TS. */
    sessionId: string
    /** `$window_id ?? ''`. */
    windowId: string
    snapshotSource: string | null
    snapshotLibrary: string | null
    /** Min/max valid-event timestamps (epoch ms). */
    startTs: number
    endTs: number
    /** rrweb/console@1 plugin events by level. */
    consoleLogCount: number
    consoleWarnCount: number
    consoleErrorCount: number
    jsonLdEventCount: number
    events: AnonymizeEventMeta[]
    /** Collected original images (hash-sorted); present only when the collection lane was enabled and images were collected. */
    images?: AnonymizeImageEntry[]
    /** Collected remote image URLs (hash-sorted); present only when the URL lane was enabled and URLs were collected. */
    urls?: AnonymizeUrlEntry[]
    /** Collected ref occurrences by bounded replay location, property, and inline or URL lane. */
    imageSources?: AnonymizeImageSourceCount[]
    /** Counts by reason for the URLs the collector refused. Absent when it refused none. */
    urlDeclines?: { reason: string; count: number }[]
    scrubDrops?: { reason: string; count: number }[]
}

/**
 * Phase timings for one {@link anonymizeKafkaPayload} call, reported on success and failure alike
 * (including contained panics). All offsets are monotonic nanoseconds from the moment the addon
 * was invoked on the JS thread; a `null` boundary means the phase was never reached.
 */
export interface AnonymizeTimings {
    /** Threadpool pickup — this offset IS the libuv queue wait. */
    taskStartNs: number | null
    decompressStartNs: number | null
    decompressEndNs: number | null
    scrubStartNs: number | null
    scrubEndNs: number | null
    /** Accumulated cv de/recompression time across all events in the message. */
    cvTotalNs: number
    cvCount: number
    /**
     * The op in flight when processing stopped: `done` on success, else the phase or op
     * (`queued` | `decompress` | `scrub` | `cv` | `serialize_meta`) that was running.
     */
    lastOp: string
}

export interface AnonymizeKafkaPayloadResult {
    /** True if the message could not be anonymized — the caller must drop or DLQ it (fail-closed). */
    failed: boolean
    /**
     * Failure classification when `failed`, matching the TS parse step's dlq/drop reasons:
     * `invalid_json` | `invalid_message_payload` | `received_non_snapshot_message` |
     * `message_contained_no_valid_rrweb_events` | `anonymize_failed`.
     */
    reason: string | null
    /** Failure detail when `failed`, else `null`. */
    error: string | null
    /** Scrubbed JSONL block lines (`["<windowId>",<event>]\n` per valid event), ready to write. */
    lines: Buffer | null
    /** JSON-serialized {@link AnonymizeMeta}. */
    meta: string | null
    /**
     * Which implementation produced the output (differential-tested identical). `tree` means the
     * whole-message parse fallback fired; the label is an A/B / fallback-rate signal.
     */
    route: 'stream' | 'tree' | null
    /** Phase timings; present on success and failure alike. `null` only if serialization failed. */
    timings: AnonymizeTimings | null
    /** Original bytes of the collected images, concatenated in `meta.images` order; null when none. */
    images: Buffer | null
    dedupedImageCount?: number
    dedupedUrlCount?: number
    /** Distinct registrable domains among the collected URLs, counted before the dedup drop. */
    collectedUrlDomainCount?: number
}

export interface RefDedupCacheStats {
    entries: number
    evictions: number
    /** Sampled misses on a ref that a cache of twice the capacity would still have held. */
    wouldHit: number
    wouldMiss: number
}

/** The entries live outside the V8 heap, so a full cache adds nothing for a major GC to mark. A capacity of 0 turns dedup off. */
export class RefDedupCache {
    /** The addon's handle. Only this package reads it. */
    readonly nativeHandle: unknown

    constructor(capacity: number) {
        this.nativeHandle = native.refDedupCacheNew(capacity)
    }

    claimRefs(refs: string[]): boolean[] {
        return native.refDedupCacheClaimRefs(this.nativeHandle, refs)
    }

    releaseRefs(refs: string[]): void {
        native.refDedupCacheReleaseRefs(this.nativeHandle, refs)
    }

    claimTransportUrls(refs: string[], urls: string[], timeBucket: number): boolean[] {
        return native.refDedupCacheClaimTransportUrls(this.nativeHandle, refs, urls, timeBucket)
    }

    releaseTransportUrls(refs: string[], urls: string[], timeBucket: number): void {
        native.refDedupCacheReleaseTransportUrls(this.nativeHandle, refs, urls, timeBucket)
    }

    stats(): RefDedupCacheStats {
        return native.refDedupCacheStats(this.nativeHandle)
    }
}

/**
 * The produce lanes' caches, consulted inside the anonymize call without marking. An image or URL
 * whose ref the cache holds is left out of the result, so it never reaches the JS heap. The caller
 * still claims what comes back before it produces.
 */
export interface ProducedRefDedup {
    images?: RefDedupCache
    urls?: RefDedupCache
    urlTimeBucket?: number
}

/** Initialize the process-wide allow lists. Call once at startup before {@link anonymizeKafkaPayload}. */
export function initAnonymizer(allow: AllowListsInput): void {
    native.initAnonymizer(JSON.stringify(allow))
}

/**
 * Anonymize a replay Kafka payload (`{"distinct_id": ..., "data": "<event json>"}`). Rust owns the
 * decompression (lz4 via the `content-encoding` header, gzip via magic bytes), the parse, the
 * scrub, and the serialize; only the raw bytes cross the FFI boundary. CPU work — including the
 * decompression — runs off the Node event loop.
 *
 * `cv` payloads re-emit as zstd; the reader dispatches on magic bytes.
 *
 * Non-empty `teamId` + `contentKey` enable image collection using the raw team ID and per-team
 * content HMAC key. The master secret stays with the caller. Inlined images are replaced
 * with `image:<teamId>:<hash>` refs (hash = keyed HMAC of the bytes), and the original bytes come
 * back in `images`/`meta.images` for the caller to produce to the scrub topic. Without them every
 * inlined image is blanked.
 *
 * `urlKey` enables the URL-collection lane independently. It is the global URL HMAC key. A remote
 * image's `src` keeps the media placeholder, a namespaced sibling attribute carries its ref, and
 * its original URL comes back in `meta.urls` for the caller to hand to the fetch lane.
 * `referenceNamespace` scopes URL refs as `imageurl:<namespace>:<hash>`; omitting it produces
 * `imageurl:<hash>`. For v2, pass `v2:<raw team id>:<YYYY-MM>` as both `teamId` and `referenceNamespace`.
 *
 * The two lanes are independent: either, both, or neither. Only `contentKey` needs `teamId`.
 *
 * `producedRefDedup` leaves out the collected images and URLs that an earlier message already produced.
 *
 * `maxImageBytes` caps the decoded size of a collected image; a bigger one is blanked.
 * Omitted, the crate default applies.
 */
export async function anonymizeKafkaPayload(
    payload: Buffer,
    contentEncoding?: string | null,
    teamId?: string | null,
    contentKey?: string | null,
    urlKey?: string | null,
    referenceNamespace?: string | null,
    producedRefDedup?: ProducedRefDedup,
    maxImageBytes?: number | null
): Promise<AnonymizeKafkaPayloadResult> {
    const result = await native.anonymizeKafkaPayload(
        payload,
        contentEncoding ?? undefined,
        teamId ?? undefined,
        contentKey ?? undefined,
        urlKey ?? undefined,
        referenceNamespace ?? undefined,
        producedRefDedup?.images?.nativeHandle,
        producedRefDedup?.urls?.nativeHandle,
        producedRefDedup?.urlTimeBucket,
        maxImageBytes ?? undefined
    )
    // Timings are best-effort telemetry: a malformed timings blob must never fail the message.
    let timings: AnonymizeTimings | null = null
    if (typeof result.timings === 'string') {
        try {
            timings = JSON.parse(result.timings)
        } catch {
            timings = null
        }
    }
    return { ...result, timings }
}

/**
 * The politeness unit for a host: the registrable domain, or the host itself when it has none.
 *
 * The fetch lane rate limits by this value and the fetch topic keys on it, so both must get the
 * same answer from one public suffix list, which is why the value comes from the Rust crate. The
 * private section of that list keeps `user.github.io` and `d111.cloudfront.net` out of a shared
 * budget with the other tenants of the same provider.
 *
 * An IP literal has no registrable domain and comes back unchanged, because the address is the
 * operator.
 */
export function politenessKey(host: string): string {
    return native.politenessKey(host)
}

/**
 * Whether the fetch lane may send a request to a host.
 *
 * The collector applies this rule before a URL reaches the topic. A redirect target has not been
 * through it, so the fetcher calls the same function rather than deriving a second answer.
 *
 * It refuses a private or reserved address, a single-label name, and a name under a suffix that
 * resolves only inside one network, which is the split-horizon DNS case.
 */
export function isPublicHost(host: string): boolean {
    return native.isPublicHost(host)
}

export interface CanonicalUrl {
    fetch: string
    dedup: string
    host: string
    domain: string
}

/** The labels of the Rust `Decline` enum. The fetch lane reports them as metric reasons. */
export type UrlPolicyDecline =
    | 'too_long'
    | 'not_absolute'
    | 'bad_scheme'
    | 'bad_port'
    | 'no_host'
    | 'non_public_host'
    | 'credential'
    | 'invalid_query'
    | 'tracking_beacon'

/**
 * The canonical forms of a URL the policy accepts, or the rule that refused it. `unwanted` is true
 * when the URL is well formed and safe but nobody wants it fetched, so a queue consumer drops only
 * that job instead of rejecting the record that carries it.
 */
export type UrlPolicyVerdict =
    | { ok: true; url: CanonicalUrl }
    | { ok: false; decline: UrlPolicyDecline; unwanted: boolean }

export function tryCanonicalizeUrl(url: string): UrlPolicyVerdict {
    const result = native.tryCanonicalizeUrl(url)
    if (typeof result.decline === 'string') {
        return {
            ok: false,
            decline: result.decline,
            unwanted: result.unwanted === true,
        }
    }
    return { ok: true, url: result }
}

export function canonicalizeUrl(url: string): CanonicalUrl | null {
    const verdict = tryCanonicalizeUrl(url)
    return verdict.ok ? verdict.url : null
}

/** The same bytes as `zlib.brotliCompress` at this quality with a size hint of `data.length`. */
export function compressBrotli(data: Buffer, quality: number): Promise<Buffer> {
    return native.compressBrotli(data, quality)
}