:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import avro from 'avsc'
import type { RedisClientPipeline, RedisV2 } from '~/common/redis/redis-v2'
import { type LogRecord, decodeLogRecords, encodeLogRecords } from '~/logs/log-record-avro'
import { type DropStats, runPipelineStages } from '~/logs/pipeline/log-processing-pipeline'
import { compileRuleSet } from './compile-rules'
import type { CompiledRuleSet } from './evaluate'
import { LogsSamplingService } from './logs-sampling.service'
const LOG_RECORD_AVRO = avro.Type.forSchema({
type: 'record',
name: 'LogRecord',
fields: [
{ name: 'uuid', type: ['null', 'string'] },
{ name: 'trace_id', type: ['null', 'bytes'] },
{ name: 'span_id', type: ['null', 'bytes'] },
{ name: 'trace_flags', type: ['null', 'int'] },
{ name: 'timestamp', type: ['null', 'long'] },
{ name: 'observed_timestamp', type: ['null', 'long'] },
{ name: 'body', type: ['null', 'string'] },
{ name: 'severity_text', type: ['null', 'string'] },
{ name: 'severity_number', type: ['null', 'int'] },
{ name: 'service_name', type: ['null', 'string'] },
{ name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },
{ name: 'instrumentation_scope', type: ['null', 'string'] },
{ name: 'event_name', type: ['null', 'string'] },
{ name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },
{ name: 'bytes_uncompressed', type: ['null', 'long'] },
],
})
function baseLog(uuid: string, serviceName: string, bytesUncompressed: number | null = null): LogRecord {
return {
uuid,
trace_id: null,
span_id: null,
trace_flags: null,
timestamp: 1_700_000_000_000_000,
observed_timestamp: 1_700_000_000_000_000,
body: 'x',
severity_text: 'info',
severity_number: 9,
service_name: serviceName,
resource_attributes: null,
instrumentation_scope: null,
event_name: null,
attributes: null,
bytes_uncompressed: bytesUncompressed,
}
}
describe('LogsSamplingService', () => {
// Decodes the buffer and runs the sampling stage over the records, mirroring what the pipeline
// does around it. Returns the drop accounting plus the survivors so assertions read as before.
async function sampleBuffer(
service: LogsSamplingService,
buffer: Buffer,
ruleSet: CompiledRuleSet,
teamId?: number,
headerBytesUncompressed = 0
): Promise<{ kept: LogRecord[]; allDropped: boolean } & DropStats> {
const [, , records] = await decodeLogRecords(buffer)
const { kept, stats } = await runPipelineStages(records, [
service.makeSamplingStage(ruleSet, teamId, headerBytesUncompressed),
])
return { ...stats, kept, allDropped: kept.length === 0 }
}
it('batches rate_limit lines and drops when tokensBefore is below pending count', async () => {
const ruleSet = compileRuleSet([
{
id: 'rl-1',
rule_type: 'rate_limit',
scope_service: 'api',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { logs_per_second: 100, burst_logs: 1000 },
},
])
const buffer = await encodeLogRecords(LOG_RECORD_AVRO, 'zstandard', [
baseLog('a', 'api', 100),
baseLog('b', 'api', 100),
baseLog('c', 'api', 100),
])
const mockRedis: RedisV2 = {
useClient: jest.fn(() => Promise.resolve(null)),
usePipeline: jest.fn((_opts, cb) => {
const pipeline = { checkRateLimitV3: jest.fn() } as unknown as RedisClientPipeline
cb(pipeline)
const pipelineResult: [Error | null, any][] = [[null, [2, 0] as const]]
return Promise.resolve(pipelineResult)
}),
}
const service = new LogsSamplingService(mockRedis, 60)
const result = await sampleBuffer(service, buffer, ruleSet, 99)
expect(result.recordsDropped).toBe(1)
expect(result.recordsDroppedByRuleId.get('rl-1')).toBe(1)
// Each row is fixtured at 100 bytes; the one dropped row contributes 100.
expect(result.bytesDropped).toBe(100)
expect(result.bytesDroppedByRuleId.get('rl-1')).toBe(100)
expect(result.allDropped).toBe(false)
expect(result.kept).toHaveLength(2)
})
it('attributes per-row bytes to the dropping rule and contributes 0 for null bytes_uncompressed', async () => {
const ruleSet = compileRuleSet([
{
id: 'rl-1',
rule_type: 'rate_limit',
scope_service: 'api',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { logs_per_second: 100, burst_logs: 1000 },
},
])
// 3 rows: 250 bytes, 750 bytes, null (old-producer message).
// Token budget of 1 → first row admitted, the other two dropped.
const buffer = await encodeLogRecords(LOG_RECORD_AVRO, 'zstandard', [
baseLog('a', 'api', 250),
baseLog('b', 'api', 750),
baseLog('c', 'api', null),
])
const mockRedis: RedisV2 = {
useClient: jest.fn(() => Promise.resolve(null)),
usePipeline: jest.fn((_opts, cb) => {
const pipeline = { checkRateLimitV3: jest.fn() } as unknown as RedisClientPipeline
cb(pipeline)
const pipelineResult: [Error | null, any][] = [[null, [1, 0] as const]]
return Promise.resolve(pipelineResult)
}),
}
const service = new LogsSamplingService(mockRedis, 60)
const result = await sampleBuffer(service, buffer, ruleSet, 99)
expect(result.recordsDropped).toBe(2)
// Only the row with a populated bytes_uncompressed contributes to the byte sum;
// the null row falls back to 0 (in-flight transition behavior).
expect(result.bytesDropped).toBe(750)
expect(result.bytesDroppedByRuleId.get('rl-1')).toBe(750)
})
it('KB-mode admits records while accumulated bytes fit the budget', async () => {
const ruleSet = compileRuleSet([
{
id: 'rl-kb',
rule_type: 'rate_limit',
scope_service: 'api',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { kb_per_second: 1, burst_kb: 2 },
},
])
// 3 rows of 300, 400, 500 bytes. Budget = 1024 bytes (1 KB).
// Walk: 300 ≤ 1024 (admit), 300+400 ≤ 1024 (admit), 700+500 > 1024 (drop).
const buffer = await encodeLogRecords(LOG_RECORD_AVRO, 'zstandard', [
baseLog('a', 'api', 300),
baseLog('b', 'api', 400),
{ ...baseLog('c', 'api', 500), attributes: { k: 'vv' }, event_name: 'evt' },
])
const mockRedis: RedisV2 = {
useClient: jest.fn(() => Promise.resolve(null)),
usePipeline: jest.fn((_opts, cb) => {
const pipeline = { checkRateLimitV3: jest.fn() } as unknown as RedisClientPipeline
cb(pipeline)
const pipelineResult: [Error | null, any][] = [[null, [1024, 0] as const]]
return Promise.resolve(pipelineResult)
}),
}
const service = new LogsSamplingService(mockRedis, 60)
const result = await sampleBuffer(service, buffer, ruleSet, 99)
expect(result.recordsDropped).toBe(1)
expect(result.recordsDroppedByRuleId.get('rl-kb')).toBe(1)
expect(result.bytesDropped).toBe(500)
expect(result.bytesDroppedByRuleId.get('rl-kb')).toBe(500)
// Billing pro-rate weights are sender-content bytes, independent of the per-row
// bytes_uncompressed field used for rate limiting: rows a,b = body 'x' + severity 'info' (5 each);
// dropped row c = 5 + attrs 'k'+'vv'(3) + 'evt'(3) = 11.
expect(result.contentBytesTotal).toBe(21)
expect(result.contentBytesDropped).toBe(11)
expect(result.kept).toHaveLength(2)
})
it('KB-mode treats null bytes_uncompressed as zero-cost (in-flight producer rollout)', async () => {
const ruleSet = compileRuleSet([
{
id: 'rl-kb',
rule_type: 'rate_limit',
scope_service: 'api',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { kb_per_second: 1, burst_kb: 2 },
},
])
// Rows: 200, null (old-producer message), 1000. Budget = 1024.
// Walk: 200 (admit), 200+0 = 200 (admit), 200+1000 > 1024 (drop).
const buffer = await encodeLogRecords(LOG_RECORD_AVRO, 'zstandard', [
baseLog('a', 'api', 200),
baseLog('b', 'api', null),
baseLog('c', 'api', 1000),
])
const mockRedis: RedisV2 = {
useClient: jest.fn(() => Promise.resolve(null)),
usePipeline: jest.fn((_opts, cb) => {
const pipeline = { checkRateLimitV3: jest.fn() } as unknown as RedisClientPipeline
cb(pipeline)
const pipelineResult: [Error | null, any][] = [[null, [1024, 0] as const]]
return Promise.resolve(pipelineResult)
}),
}
const service = new LogsSamplingService(mockRedis, 60)
const result = await sampleBuffer(service, buffer, ruleSet, 99)
expect(result.recordsDropped).toBe(1)
expect(result.bytesDropped).toBe(1000)
expect(result.bytesDroppedByRuleId.get('rl-kb')).toBe(1000)
// Content weights don't depend on the per-row bytes_uncompressed field at all:
// the null-field row still weighs its body and severity (5 each, 3 rows, 1 dropped).
expect(result.contentBytesTotal).toBe(15)
expect(result.contentBytesDropped).toBe(5)
expect(result.kept).toHaveLength(2)
})
it('KB-mode meters each row against its pro-rata share of the batch header, not per-row bytes_uncompressed', async () => {
const ruleSet = compileRuleSet([
{
id: 'rl-kb',
rule_type: 'rate_limit',
scope_service: 'api',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { kb_per_second: 1, burst_kb: 2 },
},
])
// Per-row bytes_uncompressed is inflated (900 each) because shared batch data is
// re-counted on every row; content bytes (body only here) are 2/4/6 → total 12.
// Header = 24 → pro-rata scale 2 → per-row cost 4/8/12. Budget = 12:
// 4 (admit), 4+8=12 (admit), 12+12=24 > 12 (drop row c). Metering on the raw
// bytes_uncompressed sum (900 each) would have dropped every row.
const buffer = await encodeLogRecords(LOG_RECORD_AVRO, 'zstandard', [
{ ...baseLog('a', 'api', 900), body: 'aa', severity_text: null },
{ ...baseLog('b', 'api', 900), body: 'bbbb', severity_text: null },
{ ...baseLog('c', 'api', 900), body: 'cccccc', severity_text: null },
])
const checkRateLimitV3 = jest.fn()
const mockRedis: RedisV2 = {
useClient: jest.fn(() => Promise.resolve(null)),
usePipeline: jest.fn((_opts, cb) => {
cb({ checkRateLimitV3 } as unknown as RedisClientPipeline)
const pipelineResult: [Error | null, any][] = [[null, [12, 0] as const]]
return Promise.resolve(pipelineResult)
}),
}
const service = new LogsSamplingService(mockRedis, 60)
const result = await sampleBuffer(service, buffer, ruleSet, 99, 24)
// The batch is metered at the header pro-rata total (= header bytes), not Σ bytes_uncompressed.
// Args are [key, now, cost, bucketSize, refillRate, ttl]; cost is index 2.
expect(checkRateLimitV3.mock.calls[0]![2]).toBe(24)
expect(result.recordsDropped).toBe(1)
expect(result.recordsDroppedByRuleId.get('rl-kb')).toBe(1)
// The dropped-bytes metric still reports the row's real bytes_uncompressed.
expect(result.bytesDropped).toBe(900)
expect(result.kept).toHaveLength(2)
})
it('fail-open keeps all rate_limit lines when Redis pipeline returns null', async () => {
const ruleSet = compileRuleSet([
{
id: 'rl-1',
rule_type: 'rate_limit',
scope_service: 'api',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { logs_per_second: 10 },
},
])
const buffer = await encodeLogRecords(LOG_RECORD_AVRO, 'zstandard', [baseLog('a', 'api'), baseLog('b', 'api')])
const mockRedis: RedisV2 = {
useClient: jest.fn(() => Promise.resolve(null)),
usePipeline: jest.fn(() => Promise.resolve(null)),
}
const service = new LogsSamplingService(mockRedis, 60)
const result = await sampleBuffer(service, buffer, ruleSet, 1)
expect(result.recordsDropped).toBe(0)
expect(result.recordsDroppedByRuleId.size).toBe(0)
expect(result.bytesDropped).toBe(0)
expect(result.bytesDroppedByRuleId.size).toBe(0)
expect(result.kept).toHaveLength(2)
})
})