:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
/* eslint-disable @typescript-eslint/no-var-requires */
// The native addon is built from `src/lib.rs` and copied to `index.node` at the package root.
const native = require('../index.node')
export interface AllowListsInput {
/** Words kept verbatim by the text scrubber (ASCII-case-insensitive). */
text: string[]
/** URL path segments/params kept verbatim by the URL scrubber. */
url: string[]
}
/** Per emitted JSONL line, in line order. */
export interface AnonymizeEventMeta {
/** The event's `timestamp` (epoch ms; can be fractional). */
ts: number
/** Bitmask of the FLAG_* bits in `snapshot.rs` (mirrored as PRE_SERIALIZED_FLAG_* in the consumer). */
flags: number
/** Post-scrub `hrefFrom(event)` (`data.href` / `data.payload.href`, trimmed), when present. */
href?: string
jsonLd?: { rootTypes: string[]; fullSnapshotTimestamp?: number }
}
/** One collected original image: `offset..offset+len` in {@link AnonymizeKafkaPayloadResult.images}. */
export interface AnonymizeImageEntry {
/** First 22 base64url chars of `HMAC-SHA256(contentKey, bytes)` (`hashImageBytes` in content-ref.ts). */
hash: string
offset: number
len: number
}
/** One collected remote image URL, ready for the fetch lane. */
export interface AnonymizeUrlEntry {
/** First 22 base64url chars of `HMAC-SHA256(urlKey, dedupUrl)`, where the dedup URL is the
* canonical URL minus its volatile parameters. The namespaced ref attribute ends with this. */
hash: string
/** The canonical URL with every parameter intact — what the fetcher requests. A signed URL only
* works in this form, which is why it is not the value the hash was taken over. */
url: string
/** The host the request goes to. robots.txt and the connection limit are scoped to this. */
host: string
/** The registrable domain of `host`. The fetch topic keys on this, so every URL of one operator
* lands on one partition and one pod holds its rate budget without a distributed lock. */
domain: string
}
export interface AnonymizeImageSourceCount {
source: 'css' | 'html'
property: string
kind: 'inline' | 'url'
count: number
}
/** Envelope + per-event metadata parsed from {@link AnonymizeKafkaPayloadResult.meta}. */
export interface AnonymizeMeta {
distinctId: string
/** Raw `$session_id` — normalization stays in TS. */
sessionId: string
/** `$window_id ?? ''`. */
windowId: string
snapshotSource: string | null
snapshotLibrary: string | null
/** Min/max valid-event timestamps (epoch ms). */
startTs: number
endTs: number
/** rrweb/console@1 plugin events by level. */
consoleLogCount: number
consoleWarnCount: number
consoleErrorCount: number
jsonLdEventCount: number
events: AnonymizeEventMeta[]
/** Collected original images (hash-sorted); present only when the collection lane was enabled and images were collected. */
images?: AnonymizeImageEntry[]
/** Collected remote image URLs (hash-sorted); present only when the URL lane was enabled and URLs were collected. */
urls?: AnonymizeUrlEntry[]
/** Collected ref occurrences by bounded replay location, property, and inline or URL lane. */
imageSources?: AnonymizeImageSourceCount[]
/** Counts by reason for the URLs the collector refused. Absent when it refused none. */
urlDeclines?: { reason: string; count: number }[]
scrubDrops?: { reason: string; count: number }[]
}
/**
* Phase timings for one {@link anonymizeKafkaPayload} call, reported on success and failure alike
* (including contained panics). All offsets are monotonic nanoseconds from the moment the addon
* was invoked on the JS thread; a `null` boundary means the phase was never reached.
*/
export interface AnonymizeTimings {
/** Threadpool pickup — this offset IS the libuv queue wait. */
taskStartNs: number | null
decompressStartNs: number | null
decompressEndNs: number | null
scrubStartNs: number | null
scrubEndNs: number | null
/** Accumulated cv de/recompression time across all events in the message. */
cvTotalNs: number
cvCount: number
/**
* The op in flight when processing stopped: `done` on success, else the phase or op
* (`queued` | `decompress` | `scrub` | `cv` | `serialize_meta`) that was running.
*/
lastOp: string
}
export interface AnonymizeKafkaPayloadResult {
/** True if the message could not be anonymized — the caller must drop or DLQ it (fail-closed). */
failed: boolean
/**
* Failure classification when `failed`, matching the TS parse step's dlq/drop reasons:
* `invalid_json` | `invalid_message_payload` | `received_non_snapshot_message` |
* `message_contained_no_valid_rrweb_events` | `anonymize_failed`.
*/
reason: string | null
/** Failure detail when `failed`, else `null`. */
error: string | null
/** Scrubbed JSONL block lines (`["<windowId>",<event>]\n` per valid event), ready to write. */
lines: Buffer | null
/** JSON-serialized {@link AnonymizeMeta}. */
meta: string | null
/**
* Which implementation produced the output (differential-tested identical). `tree` means the
* whole-message parse fallback fired; the label is an A/B / fallback-rate signal.
*/
route: 'stream' | 'tree' | null
/** Phase timings; present on success and failure alike. `null` only if serialization failed. */
timings: AnonymizeTimings | null
/** Original bytes of the collected images, concatenated in `meta.images` order; null when none. */
images: Buffer | null
dedupedImageCount?: number
dedupedUrlCount?: number
/** Distinct registrable domains among the collected URLs, counted before the dedup drop. */
collectedUrlDomainCount?: number
}
export interface RefDedupCacheStats {
entries: number
evictions: number
/** Sampled misses on a ref that a cache of twice the capacity would still have held. */
wouldHit: number
wouldMiss: number
}
/** The entries live outside the V8 heap, so a full cache adds nothing for a major GC to mark. A capacity of 0 turns dedup off. */
export class RefDedupCache {
/** The addon's handle. Only this package reads it. */
readonly nativeHandle: unknown
constructor(capacity: number) {
this.nativeHandle = native.refDedupCacheNew(capacity)
}
claimRefs(refs: string[]): boolean[] {
return native.refDedupCacheClaimRefs(this.nativeHandle, refs)
}
releaseRefs(refs: string[]): void {
native.refDedupCacheReleaseRefs(this.nativeHandle, refs)
}
claimTransportUrls(refs: string[], urls: string[], timeBucket: number): boolean[] {
return native.refDedupCacheClaimTransportUrls(this.nativeHandle, refs, urls, timeBucket)
}
releaseTransportUrls(refs: string[], urls: string[], timeBucket: number): void {
native.refDedupCacheReleaseTransportUrls(this.nativeHandle, refs, urls, timeBucket)
}
stats(): RefDedupCacheStats {
return native.refDedupCacheStats(this.nativeHandle)
}
}
/**
* The produce lanes' caches, consulted inside the anonymize call without marking. An image or URL
* whose ref the cache holds is left out of the result, so it never reaches the JS heap. The caller
* still claims what comes back before it produces.
*/
export interface ProducedRefDedup {
images?: RefDedupCache
urls?: RefDedupCache
urlTimeBucket?: number
}
/** Initialize the process-wide allow lists. Call once at startup before {@link anonymizeKafkaPayload}. */
export function initAnonymizer(allow: AllowListsInput): void {
native.initAnonymizer(JSON.stringify(allow))
}
/**
* Anonymize a replay Kafka payload (`{"distinct_id": ..., "data": "<event json>"}`). Rust owns the
* decompression (lz4 via the `content-encoding` header, gzip via magic bytes), the parse, the
* scrub, and the serialize; only the raw bytes cross the FFI boundary. CPU work — including the
* decompression — runs off the Node event loop.
*
* `cv` payloads re-emit as zstd; the reader dispatches on magic bytes.
*
* Non-empty `teamId` + `contentKey` enable image collection using the raw team ID and per-team
* content HMAC key. The master secret stays with the caller. Inlined images are replaced
* with `image:<teamId>:<hash>` refs (hash = keyed HMAC of the bytes), and the original bytes come
* back in `images`/`meta.images` for the caller to produce to the scrub topic. Without them every
* inlined image is blanked.
*
* `urlKey` enables the URL-collection lane independently. It is the global URL HMAC key. A remote
* image's `src` keeps the media placeholder, a namespaced sibling attribute carries its ref, and
* its original URL comes back in `meta.urls` for the caller to hand to the fetch lane.
* `referenceNamespace` scopes URL refs as `imageurl:<namespace>:<hash>`; omitting it produces
* `imageurl:<hash>`. For v2, pass `v2:<raw team id>:<YYYY-MM>` as both `teamId` and `referenceNamespace`.
*
* The two lanes are independent: either, both, or neither. Only `contentKey` needs `teamId`.
*
* `producedRefDedup` leaves out the collected images and URLs that an earlier message already produced.
*
* `maxImageBytes` caps the decoded size of a collected image; a bigger one is blanked.
* Omitted, the crate default applies.
*/
export async function anonymizeKafkaPayload(
payload: Buffer,
contentEncoding?: string | null,
teamId?: string | null,
contentKey?: string | null,
urlKey?: string | null,
referenceNamespace?: string | null,
producedRefDedup?: ProducedRefDedup,
maxImageBytes?: number | null
): Promise<AnonymizeKafkaPayloadResult> {
const result = await native.anonymizeKafkaPayload(
payload,
contentEncoding ?? undefined,
teamId ?? undefined,
contentKey ?? undefined,
urlKey ?? undefined,
referenceNamespace ?? undefined,
producedRefDedup?.images?.nativeHandle,
producedRefDedup?.urls?.nativeHandle,
producedRefDedup?.urlTimeBucket,
maxImageBytes ?? undefined
)
// Timings are best-effort telemetry: a malformed timings blob must never fail the message.
let timings: AnonymizeTimings | null = null
if (typeof result.timings === 'string') {
try {
timings = JSON.parse(result.timings)
} catch {
timings = null
}
}
return { ...result, timings }
}
/**
* The politeness unit for a host: the registrable domain, or the host itself when it has none.
*
* The fetch lane rate limits by this value and the fetch topic keys on it, so both must get the
* same answer from one public suffix list, which is why the value comes from the Rust crate. The
* private section of that list keeps `user.github.io` and `d111.cloudfront.net` out of a shared
* budget with the other tenants of the same provider.
*
* An IP literal has no registrable domain and comes back unchanged, because the address is the
* operator.
*/
export function politenessKey(host: string): string {
return native.politenessKey(host)
}
/**
* Whether the fetch lane may send a request to a host.
*
* The collector applies this rule before a URL reaches the topic. A redirect target has not been
* through it, so the fetcher calls the same function rather than deriving a second answer.
*
* It refuses a private or reserved address, a single-label name, and a name under a suffix that
* resolves only inside one network, which is the split-horizon DNS case.
*/
export function isPublicHost(host: string): boolean {
return native.isPublicHost(host)
}
export interface CanonicalUrl {
fetch: string
dedup: string
host: string
domain: string
}
/** The labels of the Rust `Decline` enum. The fetch lane reports them as metric reasons. */
export type UrlPolicyDecline =
| 'too_long'
| 'not_absolute'
| 'bad_scheme'
| 'bad_port'
| 'no_host'
| 'non_public_host'
| 'credential'
| 'invalid_query'
| 'tracking_beacon'
/**
* The canonical forms of a URL the policy accepts, or the rule that refused it. `unwanted` is true
* when the URL is well formed and safe but nobody wants it fetched, so a queue consumer drops only
* that job instead of rejecting the record that carries it.
*/
export type UrlPolicyVerdict =
| { ok: true; url: CanonicalUrl }
| { ok: false; decline: UrlPolicyDecline; unwanted: boolean }
export function tryCanonicalizeUrl(url: string): UrlPolicyVerdict {
const result = native.tryCanonicalizeUrl(url)
if (typeof result.decline === 'string') {
return {
ok: false,
decline: result.decline,
unwanted: result.unwanted === true,
}
}
return { ok: true, url: result }
}
export function canonicalizeUrl(url: string): CanonicalUrl | null {
const verdict = tryCanonicalizeUrl(url)
return verdict.ok ? verdict.url : null
}
/** The same bytes as `zlib.brotliCompress` at this quality with a size hint of `data.length`. */
export function compressBrotli(data: Buffer, quality: number): Promise<Buffer> {
return native.compressBrotli(data, quality)
}