Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(metrics): shadow per-point billable bytes and report metrics usage (#128192)

Ben White4fac7ce81b60
diff --git a/nodejs/src/common/usage-ingestion/index.ts b/nodejs/src/common/usage-ingestion/index.ts
index 204fb02f95e7..25b910426d30 100644
--- a/nodejs/src/common/usage-ingestion/index.ts
+++ b/nodejs/src/common/usage-ingestion/index.ts
@@ -22,6 +22,7 @@ export type UsageReportSite =
| 'surveys'
| 'logs'
| 'apm_traces'
+ | 'metrics'
| 'session_replay'
| 'enhanced_persons'
@@ -33,6 +34,7 @@ const PRODUCER_IDS = {
surveys: 'ingestion',
logs: 'logs',
apm_traces: 'apm-traces',
+ metrics: 'metrics',
session_replay: 'session-replay',
enhanced_persons: 'ingestion',
} as const
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.test.ts
index eea4b50ef73d..a32a8e0e66fe 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.test.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.test.ts
@@ -4,6 +4,7 @@ import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { APP_METRICS_OUTPUT, DLQ_OUTPUT } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { MessageSizeTooLarge } from '~/common/utils/db/error'
import { parseJSON } from '~/common/utils/json-parse'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
@@ -42,6 +43,7 @@ describe('MetricsIngestionPipeline', () => {
const createMessage = (token: string | null, recordCount: number): Message => {
const headers: Record<string, string> = {
bytes_uncompressed: String(recordCount * 100),
+ bytes_uncompressed_records: String(recordCount * 250),
record_count: String(recordCount),
created_at: new Date().toISOString(),
}
@@ -64,7 +66,11 @@ describe('MetricsIngestionPipeline', () => {
const pipeline = createMetricsIngestionPipeline(config)
const usage = await runMetricsIngestionPipeline(pipeline, messages)
await promiseScheduler.waitForAll()
- await emitMetricsUsage(config.outputs, usage)
+ await emitMetricsUsage(
+ config.outputs,
+ new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),
+ usage
+ )
}
const producedTo = (topic: string): ProducedMessage[] =>
@@ -167,10 +173,12 @@ describe('MetricsIngestionPipeline', () => {
[1, 'bytes_received', 300],
[1, 'records_received', 3],
[1, 'bytes_ingested', 300],
+ [1, 'bytes_ingested_records', 750],
[1, 'records_ingested', 3],
[2, 'bytes_received', 100],
[2, 'records_received', 1],
[2, 'bytes_ingested', 100],
+ [2, 'bytes_ingested_records', 250],
[2, 'records_ingested', 1],
[3, 'bytes_received', 400],
[3, 'records_received', 4],
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts
index 369d2fa50ba4..6e1cc21c062c 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts
@@ -1,6 +1,8 @@
import { Message } from 'node-rdkafka'
import { KafkaConsumerV2 } from '~/common/kafka/consumer/consumer-v2'
+import { UsageIngestionClient } from '~/common/usage-ingestion/client'
+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { DependencyUnavailableError, MessageSizeTooLarge } from '~/common/utils/db/error'
import { createMockIngestionOutputs } from '~/tests/helpers/mock-ingestion-outputs'
import { createTestTeam } from '~/tests/helpers/team'
@@ -30,15 +32,22 @@ describe('MetricsPipelineConsumer', () => {
let outputs: ReturnType<typeof createMockIngestionOutputs>
let teamManager: { getTeam: jest.Mock; getTeamByToken: jest.Mock; setTeamsIngestedData: jest.Mock }
let consumer: MetricsPipelineConsumer
+ let usageClient: { ingest: jest.Mock }
const message: Message = {
value: Buffer.from('opaque avro packet'),
key: null,
- headers: [{ token: Buffer.from(team.api_token) }, { record_count: Buffer.from('1') }],
+ headers: [
+ { token: Buffer.from(team.api_token) },
+ { record_count: Buffer.from('1') },
+ { bytes_uncompressed: Buffer.from('100') },
+ { bytes_uncompressed_records: Buffer.from('400') },
+ ],
topic: 'metrics_ingestion',
partition: 0,
offset: 0,
size: 0,
+ timestamp: 1_700_000_000_000,
}
beforeEach(() => {
@@ -48,10 +57,15 @@ describe('MetricsPipelineConsumer', () => {
getTeamByToken: jest.fn().mockResolvedValue(team),
setTeamsIngestedData: jest.fn().mockResolvedValue(undefined),
}
+ usageClient = { ingest: jest.fn().mockResolvedValue(undefined) }
consumer = new MetricsPipelineConsumer(getDefaultMetricsIngestionConsumerConfig(), {
teamManager,
quotaLimiting: { isTeamTokenQuotaLimited: jest.fn().mockResolvedValue(false) },
outputs,
+ usageBatch: new UsageRecordBatch(usageClient as unknown as UsageIngestionClient, {
+ unit: 'bytes',
+ isTeamEnabled: () => true,
+ }),
} as unknown as MetricsPipelineConsumerDeps)
})
@@ -68,12 +82,33 @@ describe('MetricsPipelineConsumer', () => {
expect(outputs.queueMessages).not.toHaveBeenCalled()
expect(teamManager.setTeamsIngestedData).not.toHaveBeenCalled()
+ expect(usageClient.ingest).not.toHaveBeenCalled()
ack()
await backgroundTask
expect(outputs.produce).toHaveBeenCalledTimes(1)
expect(outputs.queueMessages).toHaveBeenCalled()
expect(teamManager.setTeamsIngestedData).toHaveBeenCalledWith([team.id])
+ expect(usageClient.ingest).toHaveBeenCalledTimes(1)
+ expect(usageClient.ingest.mock.calls[0][0]).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ recordId: 'metrics_ingestion:0:0',
+ timestampMs: 1_700_000_000_000,
+ teamId: team.id,
+ usageKey: 'metrics_bytes',
+ unit: 'bytes',
+ quantity: 100,
+ }),
+ expect.objectContaining({
+ teamId: team.id,
+ usageKey: 'metrics_bytes_records',
+ unit: 'bytes',
+ quantity: 400,
+ }),
+ expect.objectContaining({ teamId: team.id, usageKey: 'metrics_records', unit: 'records', quantity: 1 }),
+ ])
+ )
})
it('runs on consumer-v2, which stores no offsets when the background task rejects', () => {
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts
index bee8ebc90881..1898f0d80d88 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts
@@ -4,6 +4,7 @@ import { KafkaConsumerInterface } from '~/common/kafka/consumer'
import { KafkaConsumerV2 } from '~/common/kafka/consumer/consumer-v2'
import { QuotaLimiting } from '~/common/services/quota-limiting.service'
import { instrumentFn } from '~/common/tracing/tracing-utils'
+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { logger } from '~/common/utils/logger'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
@@ -25,6 +26,7 @@ export interface MetricsPipelineConsumerDeps {
teamManager: TeamManager
quotaLimiting: QuotaLimiting
outputs: MetricsIngestionOutputs
+ usageBatch: UsageRecordBatch
}
/**
@@ -59,6 +61,7 @@ export class MetricsPipelineConsumer {
private promiseScheduler: FailureLatchingPromiseScheduler
private pipeline: MetricsIngestionPipeline
private outputs: MetricsIngestionOutputs
+ private usageBatch: UsageRecordBatch
private teamManager: TeamManager
constructor(config: MetricsIngestionConsumerConfig, deps: MetricsPipelineConsumerDeps) {
@@ -69,6 +72,7 @@ export class MetricsPipelineConsumer {
topic: config.METRICS_INGESTION_CONSUMER_CONSUME_TOPIC,
})
this.outputs = deps.outputs
+ this.usageBatch = deps.usageBatch
this.teamManager = deps.teamManager
this.promiseScheduler = new FailureLatchingPromiseScheduler()
this.pipeline = createMetricsIngestionPipeline({
@@ -121,7 +125,7 @@ export class MetricsPipelineConsumer {
await this.teamManager.setTeamsIngestedData(
[...usage.entries()].filter(([, stats]) => stats.recordsAllowed > 0).map(([teamId]) => teamId)
)
- await emitMetricsUsage(this.outputs, usage)
+ await emitMetricsUsage(this.outputs, this.usageBatch, usage)
}),
}
}
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-steps.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-steps.test.ts
index 3e9d44851a24..edd901b1ada7 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-steps.test.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-steps.test.ts
@@ -1,6 +1,7 @@
import { Message } from 'node-rdkafka'
import { AppMetricsOutput, DLQ_OUTPUT, DlqOutput } from '~/common/outputs'
+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { DependencyUnavailableError, MessageSizeTooLarge } from '~/common/utils/db/error'
import { isDevEnv } from '~/common/utils/env-utils'
import { logger } from '~/common/utils/logger'
@@ -73,6 +74,7 @@ describe('metrics ingestion steps', () => {
kafkaHeaders: { token: 'tok', bytes_uncompressed: '120', record_count: '3' },
token: 'tok',
bytesUncompressed: 120,
+ bytesUncompressedRecords: 0,
bytesCompressed: 0,
recordCount: 3,
})
@@ -169,6 +171,7 @@ describe('metrics ingestion steps', () => {
teamId,
message: createTestMessage(),
bytesUncompressed,
+ bytesUncompressedRecords: 0,
bytesCompressed: 0,
recordCount: 1,
usage,
@@ -329,7 +332,13 @@ describe('metrics ingestion steps', () => {
usage.recordReceived(1, 100, 2)
outputs.queueMessages.mockRejectedValueOnce(new Error('broker down'))
- await expect(emitMetricsUsage(outputs, usage)).resolves.toBeUndefined()
+ await expect(
+ emitMetricsUsage(
+ outputs,
+ new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),
+ usage
+ )
+ ).resolves.toBeUndefined()
expect(logger.error).toHaveBeenCalledWith(
'🔴',
'Failed to emit usage metrics - billing data may be lost',
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-usage-steps.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-usage-steps.ts
index 52ba8db1e188..b8cb01c7f0de 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-usage-steps.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-usage-steps.ts
@@ -1,6 +1,7 @@
import { AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'
+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { logger } from '~/common/utils/logger'
import { BeforeBatchStep } from '~/ingestion/framework/batching-pipeline'
import { ok } from '~/ingestion/framework/results'
@@ -46,15 +47,30 @@ export function createRecordMetricsReceivedStep<T extends RecordMetricsReceivedI
}
}
-const USAGE_METRIC_NAMES: [keyof UsageStats, string][] = [
+export const USAGE_METRIC_NAMES: [keyof UsageStats, string][] = [
['bytesReceived', 'bytes_received'],
['recordsReceived', 'records_received'],
['bytesAllowed', 'bytes_ingested'],
+ ['bytesAllowedRecords', 'bytes_ingested_records'],
['recordsAllowed', 'records_ingested'],
['bytesDropped', 'bytes_dropped'],
['recordsDropped', 'records_dropped'],
]
+const USAGE_RECORD_KEYS: ['bytes' | 'bytesRecords' | 'records', string, string][] = [
+ ['bytes', 'metrics_bytes', 'bytes'],
+ ['bytesRecords', 'metrics_bytes_records', 'bytes'],
+ ['records', 'metrics_records', 'records'],
+]
+
+export function addMetricsUsageRecords(usageBatch: UsageRecordBatch, usage: MetricsUsageAccumulator): void {
+ for (const message of usage.allowedMessages()) {
+ for (const [key, usageKey, unit] of USAGE_RECORD_KEYS) {
+ usageBatch.add(message.teamId, usageKey, message.recordId, message[key], unit, message.timestampMs)
+ }
+ }
+}
+
function incrementUsageCounters(usage: MetricsUsageAccumulator): void {
let bytesReceived = 0
let recordsReceived = 0
@@ -112,10 +128,17 @@ async function emitUsageRows(
/** Never rejects. A failed write of the billing rows is logged. */
export async function emitMetricsUsage(
outputs: IngestionOutputs<AppMetricsOutput>,
+ usageBatch: UsageRecordBatch,
usage: MetricsUsageAccumulator
): Promise<void> {
incrementUsageCounters(usage)
- await emitUsageRows(outputs, usage)
+ addMetricsUsageRecords(usageBatch, usage)
+ await Promise.all([
+ emitUsageRows(outputs, usage),
+ usageBatch.flush().catch((error) => {
+ logger.error('🔴', 'Failed to emit usage records - billing data may be lost', { error })
+ }),
+ ])
}
export function createKeepMetricsUsageStep<T extends { batchContext: MetricsUsageBatchContext }>(): ProcessingStep<
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts
index 5505805fa415..9a8b2c9285db 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts
@@ -1,7 +1,10 @@
+import { Message } from 'node-rdkafka'
+
export type UsageStats = {
bytesReceived: number
recordsReceived: number
bytesAllowed: number
+ bytesAllowedRecords: number
recordsAllowed: number
bytesDropped: number
recordsDropped: number
@@ -11,6 +14,7 @@ const DEFAULT_USAGE_STATS: UsageStats = {
bytesReceived: 0,
recordsReceived: 0,
bytesAllowed: 0,
+ bytesAllowedRecords: 0,
recordsAllowed: 0,
bytesDropped: 0,
recordsDropped: 0,
@@ -18,8 +22,23 @@ const DEFAULT_USAGE_STATS: UsageStats = {
type UsageStatsByTeam = Map<number, UsageStats>
+export interface AllowedMetricsMessage {
+ teamId: number
+ recordId: string
+ timestampMs?: number
+ bytes: number
+ bytesRecords: number
+ records: number
+}
+
+/** Stable across Kafka replays. Usage-ingestion dedups on the record ID and the day of the timestamp. */
+export function kafkaMessageUsageIdentity(message: Message): { recordId: string; timestampMs?: number } {
+ return { recordId: `${message.topic}:${message.partition}:${message.offset}`, timestampMs: message.timestamp }
+}
+
export class MetricsUsageAccumulator {
private readonly stats: UsageStatsByTeam = new Map()
+ private readonly allowed: AllowedMetricsMessage[] = []
recordReceived(teamId: number, bytes: number, records: number): void {
const stats = this.statsFor(teamId)
@@ -27,10 +46,22 @@ export class MetricsUsageAccumulator {
stats.recordsReceived += records
}
- recordAllowed(teamId: number, bytes: number, records: number): void {
+ recordAllowed(
+ teamId: number,
+ identity: { recordId: string; timestampMs?: number },
+ bytes: number,
+ records: number,
+ bytesRecords: number
+ ): void {
const stats = this.statsFor(teamId)
stats.bytesAllowed += bytes
+ stats.bytesAllowedRecords += bytesRecords
stats.recordsAllowed += records
+ this.allowed.push({ teamId, ...identity, bytes, bytesRecords, records })
+ }
+
+ allowedMessages(): readonly AllowedMetricsMessage[] {
+ return this.allowed
}
recordDropped(teamId: number, bytes: number, records: number): void {
diff --git a/nodejs/src/ingestion/pipelines/metrics/parse-metrics-headers-step.ts b/nodejs/src/ingestion/pipelines/metrics/parse-metrics-headers-step.ts
index 1be329a97fba..8f36d95f6d0e 100644
--- a/nodejs/src/ingestion/pipelines/metrics/parse-metrics-headers-step.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/parse-metrics-headers-step.ts
@@ -11,11 +11,12 @@ export interface MetricsHeaders {
kafkaHeaders: Record<string, string>
token: string
bytesUncompressed: number
+ bytesUncompressedRecords: number
bytesCompressed: number
recordCount: number
}
-const SIZE_HEADERS = ['bytes_uncompressed', 'bytes_compressed', 'record_count'] as const
+const SIZE_HEADERS = ['bytes_uncompressed', 'bytes_uncompressed_records', 'bytes_compressed', 'record_count'] as const
/** A missing header counts as 0. A negative value makes the Prometheus counters throw, so only non-negative safe integers are valid. */
function parseSizeHeader(value: string | undefined): number | null {
@@ -39,17 +40,30 @@ export function createParseMetricsHeadersStep<T extends { message: Message }>():
metricMessageDroppedCounter.inc({ reason: 'missing_token', team_id: 'unknown' })
return Promise.resolve(drop('missing_token'))
}
- const [bytesUncompressed, bytesCompressed, recordCount] = SIZE_HEADERS.map((name) =>
- parseSizeHeader(headers[name])
+ const [bytesUncompressed, bytesUncompressedRecords, bytesCompressed, recordCount] = SIZE_HEADERS.map(
+ (name) => parseSizeHeader(headers[name])
)
- if (bytesUncompressed === null || bytesCompressed === null || recordCount === null) {
+ if (
+ bytesUncompressed === null ||
+ bytesUncompressedRecords === null ||
+ bytesCompressed === null ||
+ recordCount === null
+ ) {
const invalid = SIZE_HEADERS.filter((name) => parseSizeHeader(headers[name]) === null)
logger.error('invalid_size_header', { headers: invalid })
metricMessageDroppedCounter.inc({ reason: 'invalid_size_header', team_id: 'unknown' })
return Promise.resolve(drop('invalid_size_header'))
}
return Promise.resolve(
- ok({ ...input, kafkaHeaders: headers, token, bytesUncompressed, bytesCompressed, recordCount })
+ ok({
+ ...input,
+ kafkaHeaders: headers,
+ token,
+ bytesUncompressed,
+ bytesUncompressedRecords,
+ bytesCompressed,
+ recordCount,
+ })
)
} catch (e) {
logger.error('Error parsing message', e)
diff --git a/nodejs/src/ingestion/pipelines/metrics/rate-limit-metrics-step.ts b/nodejs/src/ingestion/pipelines/metrics/rate-limit-metrics-step.ts
index 8acf2f654654..0ccbcdbf1ae3 100644
--- a/nodejs/src/ingestion/pipelines/metrics/rate-limit-metrics-step.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/rate-limit-metrics-step.ts
@@ -2,7 +2,7 @@ import { ChunkProcessingStep } from '~/ingestion/framework/builders'
import { drop, ok } from '~/ingestion/framework/results'
import { metricMessageDroppedCounter } from './metrics'
-import { MetricsUsageAccumulator } from './metrics-usage'
+import { MetricsUsageAccumulator, kafkaMessageUsageIdentity } from './metrics-usage'
import { MetricsRateLimiterService } from './services/metrics-rate-limiter.service'
import { MetricsIngestionMessage } from './types'
@@ -20,7 +20,13 @@ export function createRateLimitMetricsStep<T extends MetricsIngestionMessage & {
metricMessageDroppedCounter.inc({ reason: 'rate_limited', team_id: value.teamId.toString() })
return drop<T>('rate_limited')
}
- value.usage.recordAllowed(value.teamId, value.bytesUncompressed, value.recordCount)
+ value.usage.recordAllowed(
+ value.teamId,
+ kafkaMessageUsageIdentity(value.message),
+ value.bytesUncompressed,
+ value.recordCount,
+ value.bytesUncompressedRecords
+ )
return ok(value)
})
}
diff --git a/nodejs/src/ingestion/pipelines/metrics/types.ts b/nodejs/src/ingestion/pipelines/metrics/types.ts
index c9b8f99ad246..b0d58eb47ddc 100644
--- a/nodejs/src/ingestion/pipelines/metrics/types.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/types.ts
@@ -7,6 +7,8 @@ export type MetricsIngestionMessage = {
teamId: number
message: Message
bytesUncompressed: number
+ /** Sum of per-point sizes from the `bytes_uncompressed_records` header. 0 for batches from older producers. */
+ bytesUncompressedRecords: number
bytesCompressed: number
recordCount: number
}
diff --git a/nodejs/src/servers/ingestion-metrics-server.ts b/nodejs/src/servers/ingestion-metrics-server.ts
index 80af1f9f606e..c7e8cbda0823 100644
--- a/nodejs/src/servers/ingestion-metrics-server.ts
+++ b/nodejs/src/servers/ingestion-metrics-server.ts
@@ -2,6 +2,8 @@ import { defaultConfig, overrideConfigWithEnv } from '~/common/config/config'
import { createPosthogRedisConnectionConfig } from '~/common/config/redis-pools'
import { KafkaProducerRegistry } from '~/common/outputs/kafka-producer-registry'
import { QuotaLimiting } from '~/common/services/quota-limiting.service'
+import { UsageIngestionConfig, createUsageIngestionClient, usageReportTeamMatcher } from '~/common/usage-ingestion'
+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { PostgresRouter } from '~/common/utils/db/postgres'
import { createRedisPoolFromConfig } from '~/common/utils/db/redis'
import { logger } from '~/common/utils/logger'
@@ -47,6 +49,7 @@ export type IngestionMetricsServerConfig = BaseServerConfig &
KafkaBrokerConfig &
DatabaseConnectionConfig &
RedisConnectionsConfig &
+ UsageIngestionConfig &
Pick<CommonConfig, 'LOG_LEVEL' | 'PLUGIN_SERVER_MODE' | 'CLOUD_DEPLOYMENT' | 'HEALTHCHECK_MAX_STALE_SECONDS'>
export class IngestionMetricsServer implements NodeServer {
@@ -109,7 +112,16 @@ export class IngestionMetricsServer implements NodeServer {
const serviceLoaders: (() => Promise<PluginServerService>)[] = []
serviceLoaders.push(async () => {
- const consumer = new MetricsPipelineConsumer(this.config, { teamManager, quotaLimiting, outputs })
+ const usageBatch = new UsageRecordBatch(createUsageIngestionClient(this.config, 'metrics'), {
+ unit: 'bytes',
+ isTeamEnabled: usageReportTeamMatcher(this.config),
+ })
+ const consumer = new MetricsPipelineConsumer(this.config, {
+ teamManager,
+ quotaLimiting,
+ outputs,
+ usageBatch,
+ })
await consumer.start()
return consumer.service
})
diff --git a/rust/capture-logs/src/kafka.rs b/rust/capture-logs/src/kafka.rs
index ebf0549d3d78..ed969d438972 100644
--- a/rust/capture-logs/src/kafka.rs
+++ b/rust/capture-logs/src/kafka.rs
@@ -1,6 +1,6 @@
use crate::avro_schema::AVRO_SCHEMA;
use crate::log_record::{min_kafka_log_row_timestamp_micros, sum_kafka_log_row_bytes, KafkaLogRow};
-use crate::metric_record::KafkaMetricRow;
+use crate::metric_record::{sum_kafka_metric_row_bytes, KafkaMetricRow};
use crate::metrics_avro_schema::METRICS_AVRO_SCHEMA;
use crate::trace_record::KafkaTraceRow;
use crate::traces_avro_schema::TRACES_AVRO_SCHEMA;
@@ -589,6 +589,11 @@ impl KafkaSink {
counter!("capture_metrics_timestamps_overridden").increment(timestamps_overridden);
}
+ let records_uncompressed_bytes = sum_kafka_metric_row_bytes(&rows);
+ counter!("capture_metrics_bytes_uncompressed_payload").increment(uncompressed_bytes);
+ counter!("capture_metrics_bytes_uncompressed_records")
+ .increment(records_uncompressed_bytes);
+
self.write_avro_batch(
&self.metrics_producer,
&self.metrics_topic,
@@ -596,7 +601,7 @@ impl KafkaSink {
token,
&rows,
uncompressed_bytes,
- None,
+ Some(records_uncompressed_bytes),
timestamps_overridden,
None,
None,
diff --git a/rust/capture-logs/src/metric_record.rs b/rust/capture-logs/src/metric_record.rs
index eab62d41e929..bf75b9a16688 100644
--- a/rust/capture-logs/src/metric_record.rs
+++ b/rust/capture-logs/src/metric_record.rs
@@ -55,6 +55,23 @@ pub struct KafkaMetricRow {
pub retention_days: Option<i32>,
}
+const POINT_BYTES: u64 = 24;
+const LABEL_BYTES: u64 = 8;
+const HISTOGRAM_BUCKET_BYTES: u64 = 8;
+
+/// Billable size of one point: 24 bytes, plus 8 bytes for each label and each histogram bucket.
+/// The size uses counts, not string lengths, so customers can predict it from what they send.
+/// The wire format, batch size and name lengths do not change it.
+pub fn compute_kafka_metric_row_bytes(row: &KafkaMetricRow) -> u64 {
+ let labels = (row.resource_attributes.len() + row.attributes.len()) as u64;
+ let buckets = row.histogram_counts.len() as u64;
+ POINT_BYTES + LABEL_BYTES * labels + HISTOGRAM_BUCKET_BYTES * buckets
+}
+
+pub fn sum_kafka_metric_row_bytes(rows: &[KafkaMetricRow]) -> u64 {
+ rows.iter().map(compute_kafka_metric_row_bytes).sum()
+}
+
/// Flatten an OTEL Metric into one or more KafkaMetricRow records.
/// Each data point in the metric becomes a separate row.
pub fn flatten_metric(
@@ -586,6 +603,30 @@ mod tests {
row
}
+ #[test]
+ fn test_row_bytes_count_points_labels_and_buckets_not_lengths() {
+ let mut row = build_test_row(&[]);
+ row.metric_name = "rpc.latency".to_string();
+ row.histogram_bounds = vec![1.0, 5.0];
+ row.histogram_counts = vec![1, 2, 3];
+ row.resource_attributes = HashMap::from([("host".to_string(), "web-1".to_string())]);
+ row.attributes = HashMap::from([("route".to_string(), "/api".to_string())]);
+
+ assert_eq!(compute_kafka_metric_row_bytes(&row), 24 + 2 * 8 + 3 * 8);
+
+ row.metric_name = "m".repeat(200);
+ row.instrumentation_scope = "s".repeat(200);
+ row.unit = "u".repeat(200);
+ row.attributes = HashMap::from([("r".repeat(100), "v".repeat(100))]);
+ assert_eq!(compute_kafka_metric_row_bytes(&row), 24 + 2 * 8 + 3 * 8);
+
+ // The second row's overridden timestamp adds a `$originalTimestamp` label.
+ assert_eq!(
+ sum_kafka_metric_row_bytes(&[row, build_test_row(&[])]),
+ (24 + 2 * 8 + 3 * 8) + (24 + 8)
+ );
+ }
+
#[test]
fn test_has_labels_and_retention_days_serialise_into_avro_payload() {
use crate::metrics_avro_schema::METRICS_AVRO_SCHEMA;