:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Counter, Gauge } from 'prom-client'
import { logger } from '~/common/utils/logger'
import { SerializedKafkaMessage } from './types'
const outOfOrderMessages = new Counter({
name: 'ingestion_api_out_of_order_messages_total',
help: 'Messages fed to the pipeline at or below their routing key’s last seen offset, outside a marked replay — a per-key ordering violation',
})
const replayedMessages = new Counter({
name: 'ingestion_api_replayed_messages_total',
help: 'Messages at or below their routing key’s last seen offset inside a marked replay request — expected at-least-once redelivery, not a violation',
})
const sentinelKeys = new Gauge({
name: 'ingestion_api_order_sentinel_keys',
help: 'Routing keys currently tracked by the feed-order sentinel',
})
const sentinelEvictions = new Counter({
name: 'ingestion_api_order_sentinel_evictions_total',
help: 'Routing keys evicted from the feed-order sentinel at capacity (their next appearance rebaselines unchecked)',
})
export interface FeedOrderCheckResult {
outOfOrder: number
replayed: number
}
interface KeyEntry {
consumerId: string
offset: number
}
/**
* Verifies the pipeline's per-key ordering invariant at its measurement point:
* the feed. The grouping stage processes each routing key's messages strictly
* in feed order, so "fed in Kafka offset order per key" is exactly "processed
* in order per key". Call `check` synchronously immediately before
* `pipeline.feed()` — the shared event loop then makes check order identical
* to feed order even with concurrent streams.
*
* The Rust consumer stamps each request with its process incarnation
* (`consumer_id`) and a `replay` flag (a deferred-flush re-route).
* An offset regression within one incarnation is an ordering violation unless
* the request is a replay (at-least-once redelivery of un-ACKed messages). A
* changed incarnation (consumer restart or partition handoff) legitimately
* replays uncommitted offsets, so the key rebaselines instead of firing.
*
* State is a bounded LRU: at capacity the least-recently-seen key is dropped
* and its next appearance rebaselines unchecked (sampling, not a guarantee
* gap under normal cardinality).
*/
export class FeedOrderSentinel {
private entries = new Map<string, KeyEntry>()
constructor(private readonly maxKeys: number = 200_000) {}
check(messages: SerializedKafkaMessage[], consumerId: string, replay: boolean): FeedOrderCheckResult {
const result: FeedOrderCheckResult = { outOfOrder: 0, replayed: 0 }
for (const message of messages) {
if (!message.key) {
// Unkeyed: the consumer routes these individually under a
// synthetic unique key, so there is no per-key order to check.
continue
}
const key = `${message.topic}/${message.partition}/${message.key}`
const entry = this.entries.get(key)
if (entry && entry.consumerId === consumerId) {
if (message.offset <= entry.offset) {
if (replay) {
result.replayed++
} else {
result.outOfOrder++
logger.warn('⚠️', 'Out-of-order message fed to ingestion pipeline', {
key,
offset: message.offset,
lastSeenOffset: entry.offset,
consumerId,
})
}
} else {
entry.offset = message.offset
}
// Refresh LRU recency (Map preserves insertion order).
this.entries.delete(key)
this.entries.set(key, entry)
} else {
// New key, or a new consumer incarnation: rebaseline.
this.entries.delete(key)
this.entries.set(key, { consumerId, offset: message.offset })
}
}
let evicted = 0
while (this.entries.size > this.maxKeys) {
const oldest = this.entries.keys().next().value
if (oldest === undefined) {
break
}
this.entries.delete(oldest)
evicted++
}
if (result.outOfOrder > 0) {
outOfOrderMessages.inc(result.outOfOrder)
}
if (result.replayed > 0) {
replayedMessages.inc(result.replayed)
}
if (evicted > 0) {
sentinelEvictions.inc(evicted)
}
sentinelKeys.set(this.entries.size)
return result
}
get size(): number {
return this.entries.size
}
}