:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(metrics): shadow per-point billable bytes and report metrics usage (#128192)
Ben White4fac7ce81b60
diff --git a/nodejs/src/common/usage-ingestion/index.ts b/nodejs/src/common/usage-ingestion/index.tsindex 204fb02f95e7..25b910426d30 100644--- a/nodejs/src/common/usage-ingestion/index.ts+++ b/nodejs/src/common/usage-ingestion/index.ts@@ -22,6 +22,7 @@ export type UsageReportSite =| 'surveys'| 'logs'| 'apm_traces'+ | 'metrics'| 'session_replay'| 'enhanced_persons'@@ -33,6 +34,7 @@ const PRODUCER_IDS = {surveys: 'ingestion',logs: 'logs',apm_traces: 'apm-traces',+ metrics: 'metrics',session_replay: 'session-replay',enhanced_persons: 'ingestion',} as constdiff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.test.tsindex eea4b50ef73d..a32a8e0e66fe 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.test.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.test.ts@@ -4,6 +4,7 @@ import { KafkaProducerWrapper } from '~/common/kafka/producer'import { APP_METRICS_OUTPUT, DLQ_OUTPUT } from '~/common/outputs'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'import { MessageSizeTooLarge } from '~/common/utils/db/error'import { parseJSON } from '~/common/utils/json-parse'import { PromiseScheduler } from '~/common/utils/promise-scheduler'@@ -42,6 +43,7 @@ describe('MetricsIngestionPipeline', () => {const createMessage = (token: string | null, recordCount: number): Message => {const headers: Record<string, string> = {bytes_uncompressed: String(recordCount * 100),+ bytes_uncompressed_records: String(recordCount * 250),record_count: String(recordCount),created_at: new Date().toISOString(),}@@ -64,7 +66,11 @@ describe('MetricsIngestionPipeline', () => {const pipeline = createMetricsIngestionPipeline(config)const usage = await runMetricsIngestionPipeline(pipeline, messages)await promiseScheduler.waitForAll()- await emitMetricsUsage(config.outputs, usage)+ await emitMetricsUsage(+ config.outputs,+ new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),+ usage+ )}const producedTo = (topic: string): ProducedMessage[] =>@@ -167,10 +173,12 @@ describe('MetricsIngestionPipeline', () => {[1, 'bytes_received', 300],[1, 'records_received', 3],[1, 'bytes_ingested', 300],+ [1, 'bytes_ingested_records', 750],[1, 'records_ingested', 3],[2, 'bytes_received', 100],[2, 'records_received', 1],[2, 'bytes_ingested', 100],+ [2, 'bytes_ingested_records', 250],[2, 'records_ingested', 1],[3, 'bytes_received', 400],[3, 'records_received', 4],diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.tsindex 369d2fa50ba4..6e1cc21c062c 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts@@ -1,6 +1,8 @@import { Message } from 'node-rdkafka'import { KafkaConsumerV2 } from '~/common/kafka/consumer/consumer-v2'+import { UsageIngestionClient } from '~/common/usage-ingestion/client'+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'import { DependencyUnavailableError, MessageSizeTooLarge } from '~/common/utils/db/error'import { createMockIngestionOutputs } from '~/tests/helpers/mock-ingestion-outputs'import { createTestTeam } from '~/tests/helpers/team'@@ -30,15 +32,22 @@ describe('MetricsPipelineConsumer', () => {let outputs: ReturnType<typeof createMockIngestionOutputs>let teamManager: { getTeam: jest.Mock; getTeamByToken: jest.Mock; setTeamsIngestedData: jest.Mock }let consumer: MetricsPipelineConsumer+ let usageClient: { ingest: jest.Mock }const message: Message = {value: Buffer.from('opaque avro packet'),key: null,- headers: [{ token: Buffer.from(team.api_token) }, { record_count: Buffer.from('1') }],+ headers: [+ { token: Buffer.from(team.api_token) },+ { record_count: Buffer.from('1') },+ { bytes_uncompressed: Buffer.from('100') },+ { bytes_uncompressed_records: Buffer.from('400') },+ ],topic: 'metrics_ingestion',partition: 0,offset: 0,size: 0,+ timestamp: 1_700_000_000_000,}beforeEach(() => {@@ -48,10 +57,15 @@ describe('MetricsPipelineConsumer', () => {getTeamByToken: jest.fn().mockResolvedValue(team),setTeamsIngestedData: jest.fn().mockResolvedValue(undefined),}+ usageClient = { ingest: jest.fn().mockResolvedValue(undefined) }consumer = new MetricsPipelineConsumer(getDefaultMetricsIngestionConsumerConfig(), {teamManager,quotaLimiting: { isTeamTokenQuotaLimited: jest.fn().mockResolvedValue(false) },outputs,+ usageBatch: new UsageRecordBatch(usageClient as unknown as UsageIngestionClient, {+ unit: 'bytes',+ isTeamEnabled: () => true,+ }),} as unknown as MetricsPipelineConsumerDeps)})@@ -68,12 +82,33 @@ describe('MetricsPipelineConsumer', () => {expect(outputs.queueMessages).not.toHaveBeenCalled()expect(teamManager.setTeamsIngestedData).not.toHaveBeenCalled()+ expect(usageClient.ingest).not.toHaveBeenCalled()ack()await backgroundTaskexpect(outputs.produce).toHaveBeenCalledTimes(1)expect(outputs.queueMessages).toHaveBeenCalled()expect(teamManager.setTeamsIngestedData).toHaveBeenCalledWith([team.id])+ expect(usageClient.ingest).toHaveBeenCalledTimes(1)+ expect(usageClient.ingest.mock.calls[0][0]).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ recordId: 'metrics_ingestion:0:0',+ timestampMs: 1_700_000_000_000,+ teamId: team.id,+ usageKey: 'metrics_bytes',+ unit: 'bytes',+ quantity: 100,+ }),+ expect.objectContaining({+ teamId: team.id,+ usageKey: 'metrics_bytes_records',+ unit: 'bytes',+ quantity: 400,+ }),+ expect.objectContaining({ teamId: team.id, usageKey: 'metrics_records', unit: 'records', quantity: 1 }),+ ])+ )})it('runs on consumer-v2, which stores no offsets when the background task rejects', () => {diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.tsindex bee8ebc90881..1898f0d80d88 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts@@ -4,6 +4,7 @@ import { KafkaConsumerInterface } from '~/common/kafka/consumer'import { KafkaConsumerV2 } from '~/common/kafka/consumer/consumer-v2'import { QuotaLimiting } from '~/common/services/quota-limiting.service'import { instrumentFn } from '~/common/tracing/tracing-utils'+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'import { logger } from '~/common/utils/logger'import { PromiseScheduler } from '~/common/utils/promise-scheduler'import { TeamManager } from '~/common/utils/team-manager'@@ -25,6 +26,7 @@ export interface MetricsPipelineConsumerDeps {teamManager: TeamManagerquotaLimiting: QuotaLimitingoutputs: MetricsIngestionOutputs+ usageBatch: UsageRecordBatch}/**@@ -59,6 +61,7 @@ export class MetricsPipelineConsumer {private promiseScheduler: FailureLatchingPromiseSchedulerprivate pipeline: MetricsIngestionPipelineprivate outputs: MetricsIngestionOutputs+ private usageBatch: UsageRecordBatchprivate teamManager: TeamManagerconstructor(config: MetricsIngestionConsumerConfig, deps: MetricsPipelineConsumerDeps) {@@ -69,6 +72,7 @@ export class MetricsPipelineConsumer {topic: config.METRICS_INGESTION_CONSUMER_CONSUME_TOPIC,})this.outputs = deps.outputs+ this.usageBatch = deps.usageBatchthis.teamManager = deps.teamManagerthis.promiseScheduler = new FailureLatchingPromiseScheduler()this.pipeline = createMetricsIngestionPipeline({@@ -121,7 +125,7 @@ export class MetricsPipelineConsumer {await this.teamManager.setTeamsIngestedData([...usage.entries()].filter(([, stats]) => stats.recordsAllowed > 0).map(([teamId]) => teamId))- await emitMetricsUsage(this.outputs, usage)+ await emitMetricsUsage(this.outputs, this.usageBatch, usage)}),}}diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-steps.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-steps.test.tsindex 3e9d44851a24..edd901b1ada7 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-steps.test.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-steps.test.ts@@ -1,6 +1,7 @@import { Message } from 'node-rdkafka'import { AppMetricsOutput, DLQ_OUTPUT, DlqOutput } from '~/common/outputs'+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'import { DependencyUnavailableError, MessageSizeTooLarge } from '~/common/utils/db/error'import { isDevEnv } from '~/common/utils/env-utils'import { logger } from '~/common/utils/logger'@@ -73,6 +74,7 @@ describe('metrics ingestion steps', () => {kafkaHeaders: { token: 'tok', bytes_uncompressed: '120', record_count: '3' },token: 'tok',bytesUncompressed: 120,+ bytesUncompressedRecords: 0,bytesCompressed: 0,recordCount: 3,})@@ -169,6 +171,7 @@ describe('metrics ingestion steps', () => {teamId,message: createTestMessage(),bytesUncompressed,+ bytesUncompressedRecords: 0,bytesCompressed: 0,recordCount: 1,usage,@@ -329,7 +332,13 @@ describe('metrics ingestion steps', () => {usage.recordReceived(1, 100, 2)outputs.queueMessages.mockRejectedValueOnce(new Error('broker down'))- await expect(emitMetricsUsage(outputs, usage)).resolves.toBeUndefined()+ await expect(+ emitMetricsUsage(+ outputs,+ new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),+ usage+ )+ ).resolves.toBeUndefined()expect(logger.error).toHaveBeenCalledWith('🔴','Failed to emit usage metrics - billing data may be lost',diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-usage-steps.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-usage-steps.tsindex 52ba8db1e188..b8cb01c7f0de 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-usage-steps.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-usage-steps.ts@@ -1,6 +1,7 @@import { AppMetricsOutput } from '~/common/outputs'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'import { logger } from '~/common/utils/logger'import { BeforeBatchStep } from '~/ingestion/framework/batching-pipeline'import { ok } from '~/ingestion/framework/results'@@ -46,15 +47,30 @@ export function createRecordMetricsReceivedStep<T extends RecordMetricsReceivedI}}-const USAGE_METRIC_NAMES: [keyof UsageStats, string][] = [+export const USAGE_METRIC_NAMES: [keyof UsageStats, string][] = [['bytesReceived', 'bytes_received'],['recordsReceived', 'records_received'],['bytesAllowed', 'bytes_ingested'],+ ['bytesAllowedRecords', 'bytes_ingested_records'],['recordsAllowed', 'records_ingested'],['bytesDropped', 'bytes_dropped'],['recordsDropped', 'records_dropped'],]+const USAGE_RECORD_KEYS: ['bytes' | 'bytesRecords' | 'records', string, string][] = [+ ['bytes', 'metrics_bytes', 'bytes'],+ ['bytesRecords', 'metrics_bytes_records', 'bytes'],+ ['records', 'metrics_records', 'records'],+]++export function addMetricsUsageRecords(usageBatch: UsageRecordBatch, usage: MetricsUsageAccumulator): void {+ for (const message of usage.allowedMessages()) {+ for (const [key, usageKey, unit] of USAGE_RECORD_KEYS) {+ usageBatch.add(message.teamId, usageKey, message.recordId, message[key], unit, message.timestampMs)+ }+ }+}+function incrementUsageCounters(usage: MetricsUsageAccumulator): void {let bytesReceived = 0let recordsReceived = 0@@ -112,10 +128,17 @@ async function emitUsageRows(/** Never rejects. A failed write of the billing rows is logged. */export async function emitMetricsUsage(outputs: IngestionOutputs<AppMetricsOutput>,+ usageBatch: UsageRecordBatch,usage: MetricsUsageAccumulator): Promise<void> {incrementUsageCounters(usage)- await emitUsageRows(outputs, usage)+ addMetricsUsageRecords(usageBatch, usage)+ await Promise.all([+ emitUsageRows(outputs, usage),+ usageBatch.flush().catch((error) => {+ logger.error('🔴', 'Failed to emit usage records - billing data may be lost', { error })+ }),+ ])}export function createKeepMetricsUsageStep<T extends { batchContext: MetricsUsageBatchContext }>(): ProcessingStep<diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-usage.tsindex 5505805fa415..9a8b2c9285db 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-usage.ts@@ -1,7 +1,10 @@+import { Message } from 'node-rdkafka'+export type UsageStats = {bytesReceived: numberrecordsReceived: numberbytesAllowed: number+ bytesAllowedRecords: numberrecordsAllowed: numberbytesDropped: numberrecordsDropped: number@@ -11,6 +14,7 @@ const DEFAULT_USAGE_STATS: UsageStats = {bytesReceived: 0,recordsReceived: 0,bytesAllowed: 0,+ bytesAllowedRecords: 0,recordsAllowed: 0,bytesDropped: 0,recordsDropped: 0,@@ -18,8 +22,23 @@ const DEFAULT_USAGE_STATS: UsageStats = {type UsageStatsByTeam = Map<number, UsageStats>+export interface AllowedMetricsMessage {+ teamId: number+ recordId: string+ timestampMs?: number+ bytes: number+ bytesRecords: number+ records: number+}++/** Stable across Kafka replays. Usage-ingestion dedups on the record ID and the day of the timestamp. */+export function kafkaMessageUsageIdentity(message: Message): { recordId: string; timestampMs?: number } {+ return { recordId: `${message.topic}:${message.partition}:${message.offset}`, timestampMs: message.timestamp }+}+export class MetricsUsageAccumulator {private readonly stats: UsageStatsByTeam = new Map()+ private readonly allowed: AllowedMetricsMessage[] = []recordReceived(teamId: number, bytes: number, records: number): void {const stats = this.statsFor(teamId)@@ -27,10 +46,22 @@ export class MetricsUsageAccumulator {stats.recordsReceived += records}- recordAllowed(teamId: number, bytes: number, records: number): void {+ recordAllowed(+ teamId: number,+ identity: { recordId: string; timestampMs?: number },+ bytes: number,+ records: number,+ bytesRecords: number+ ): void {const stats = this.statsFor(teamId)stats.bytesAllowed += bytes+ stats.bytesAllowedRecords += bytesRecordsstats.recordsAllowed += records+ this.allowed.push({ teamId, ...identity, bytes, bytesRecords, records })+ }++ allowedMessages(): readonly AllowedMetricsMessage[] {+ return this.allowed}recordDropped(teamId: number, bytes: number, records: number): void {diff --git a/nodejs/src/ingestion/pipelines/metrics/parse-metrics-headers-step.ts b/nodejs/src/ingestion/pipelines/metrics/parse-metrics-headers-step.tsindex 1be329a97fba..8f36d95f6d0e 100644--- a/nodejs/src/ingestion/pipelines/metrics/parse-metrics-headers-step.ts+++ b/nodejs/src/ingestion/pipelines/metrics/parse-metrics-headers-step.ts@@ -11,11 +11,12 @@ export interface MetricsHeaders {kafkaHeaders: Record<string, string>token: stringbytesUncompressed: number+ bytesUncompressedRecords: numberbytesCompressed: numberrecordCount: number}-const SIZE_HEADERS = ['bytes_uncompressed', 'bytes_compressed', 'record_count'] as const+const SIZE_HEADERS = ['bytes_uncompressed', 'bytes_uncompressed_records', 'bytes_compressed', 'record_count'] as const/** A missing header counts as 0. A negative value makes the Prometheus counters throw, so only non-negative safe integers are valid. */function parseSizeHeader(value: string | undefined): number | null {@@ -39,17 +40,30 @@ export function createParseMetricsHeadersStep<T extends { message: Message }>():metricMessageDroppedCounter.inc({ reason: 'missing_token', team_id: 'unknown' })return Promise.resolve(drop('missing_token'))}- const [bytesUncompressed, bytesCompressed, recordCount] = SIZE_HEADERS.map((name) =>- parseSizeHeader(headers[name])+ const [bytesUncompressed, bytesUncompressedRecords, bytesCompressed, recordCount] = SIZE_HEADERS.map(+ (name) => parseSizeHeader(headers[name]))- if (bytesUncompressed === null || bytesCompressed === null || recordCount === null) {+ if (+ bytesUncompressed === null ||+ bytesUncompressedRecords === null ||+ bytesCompressed === null ||+ recordCount === null+ ) {const invalid = SIZE_HEADERS.filter((name) => parseSizeHeader(headers[name]) === null)logger.error('invalid_size_header', { headers: invalid })metricMessageDroppedCounter.inc({ reason: 'invalid_size_header', team_id: 'unknown' })return Promise.resolve(drop('invalid_size_header'))}return Promise.resolve(- ok({ ...input, kafkaHeaders: headers, token, bytesUncompressed, bytesCompressed, recordCount })+ ok({+ ...input,+ kafkaHeaders: headers,+ token,+ bytesUncompressed,+ bytesUncompressedRecords,+ bytesCompressed,+ recordCount,+ }))} catch (e) {logger.error('Error parsing message', e)diff --git a/nodejs/src/ingestion/pipelines/metrics/rate-limit-metrics-step.ts b/nodejs/src/ingestion/pipelines/metrics/rate-limit-metrics-step.tsindex 8acf2f654654..0ccbcdbf1ae3 100644--- a/nodejs/src/ingestion/pipelines/metrics/rate-limit-metrics-step.ts+++ b/nodejs/src/ingestion/pipelines/metrics/rate-limit-metrics-step.ts@@ -2,7 +2,7 @@ import { ChunkProcessingStep } from '~/ingestion/framework/builders'import { drop, ok } from '~/ingestion/framework/results'import { metricMessageDroppedCounter } from './metrics'-import { MetricsUsageAccumulator } from './metrics-usage'+import { MetricsUsageAccumulator, kafkaMessageUsageIdentity } from './metrics-usage'import { MetricsRateLimiterService } from './services/metrics-rate-limiter.service'import { MetricsIngestionMessage } from './types'@@ -20,7 +20,13 @@ export function createRateLimitMetricsStep<T extends MetricsIngestionMessage & {metricMessageDroppedCounter.inc({ reason: 'rate_limited', team_id: value.teamId.toString() })return drop<T>('rate_limited')}- value.usage.recordAllowed(value.teamId, value.bytesUncompressed, value.recordCount)+ value.usage.recordAllowed(+ value.teamId,+ kafkaMessageUsageIdentity(value.message),+ value.bytesUncompressed,+ value.recordCount,+ value.bytesUncompressedRecords+ )return ok(value)})}diff --git a/nodejs/src/ingestion/pipelines/metrics/types.ts b/nodejs/src/ingestion/pipelines/metrics/types.tsindex c9b8f99ad246..b0d58eb47ddc 100644--- a/nodejs/src/ingestion/pipelines/metrics/types.ts+++ b/nodejs/src/ingestion/pipelines/metrics/types.ts@@ -7,6 +7,8 @@ export type MetricsIngestionMessage = {teamId: numbermessage: MessagebytesUncompressed: number+ /** Sum of per-point sizes from the `bytes_uncompressed_records` header. 0 for batches from older producers. */+ bytesUncompressedRecords: numberbytesCompressed: numberrecordCount: number}diff --git a/nodejs/src/servers/ingestion-metrics-server.ts b/nodejs/src/servers/ingestion-metrics-server.tsindex 80af1f9f606e..c7e8cbda0823 100644--- a/nodejs/src/servers/ingestion-metrics-server.ts+++ b/nodejs/src/servers/ingestion-metrics-server.ts@@ -2,6 +2,8 @@ import { defaultConfig, overrideConfigWithEnv } from '~/common/config/config'import { createPosthogRedisConnectionConfig } from '~/common/config/redis-pools'import { KafkaProducerRegistry } from '~/common/outputs/kafka-producer-registry'import { QuotaLimiting } from '~/common/services/quota-limiting.service'+import { UsageIngestionConfig, createUsageIngestionClient, usageReportTeamMatcher } from '~/common/usage-ingestion'+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'import { PostgresRouter } from '~/common/utils/db/postgres'import { createRedisPoolFromConfig } from '~/common/utils/db/redis'import { logger } from '~/common/utils/logger'@@ -47,6 +49,7 @@ export type IngestionMetricsServerConfig = BaseServerConfig &KafkaBrokerConfig &DatabaseConnectionConfig &RedisConnectionsConfig &+ UsageIngestionConfig &Pick<CommonConfig, 'LOG_LEVEL' | 'PLUGIN_SERVER_MODE' | 'CLOUD_DEPLOYMENT' | 'HEALTHCHECK_MAX_STALE_SECONDS'>export class IngestionMetricsServer implements NodeServer {@@ -109,7 +112,16 @@ export class IngestionMetricsServer implements NodeServer {const serviceLoaders: (() => Promise<PluginServerService>)[] = []serviceLoaders.push(async () => {- const consumer = new MetricsPipelineConsumer(this.config, { teamManager, quotaLimiting, outputs })+ const usageBatch = new UsageRecordBatch(createUsageIngestionClient(this.config, 'metrics'), {+ unit: 'bytes',+ isTeamEnabled: usageReportTeamMatcher(this.config),+ })+ const consumer = new MetricsPipelineConsumer(this.config, {+ teamManager,+ quotaLimiting,+ outputs,+ usageBatch,+ })await consumer.start()return consumer.service})diff --git a/rust/capture-logs/src/kafka.rs b/rust/capture-logs/src/kafka.rsindex ebf0549d3d78..ed969d438972 100644--- a/rust/capture-logs/src/kafka.rs+++ b/rust/capture-logs/src/kafka.rs@@ -1,6 +1,6 @@use crate::avro_schema::AVRO_SCHEMA;use crate::log_record::{min_kafka_log_row_timestamp_micros, sum_kafka_log_row_bytes, KafkaLogRow};-use crate::metric_record::KafkaMetricRow;+use crate::metric_record::{sum_kafka_metric_row_bytes, KafkaMetricRow};use crate::metrics_avro_schema::METRICS_AVRO_SCHEMA;use crate::trace_record::KafkaTraceRow;use crate::traces_avro_schema::TRACES_AVRO_SCHEMA;@@ -589,6 +589,11 @@ impl KafkaSink {counter!("capture_metrics_timestamps_overridden").increment(timestamps_overridden);}+ let records_uncompressed_bytes = sum_kafka_metric_row_bytes(&rows);+ counter!("capture_metrics_bytes_uncompressed_payload").increment(uncompressed_bytes);+ counter!("capture_metrics_bytes_uncompressed_records")+ .increment(records_uncompressed_bytes);+self.write_avro_batch(&self.metrics_producer,&self.metrics_topic,@@ -596,7 +601,7 @@ impl KafkaSink {token,&rows,uncompressed_bytes,- None,+ Some(records_uncompressed_bytes),timestamps_overridden,None,None,diff --git a/rust/capture-logs/src/metric_record.rs b/rust/capture-logs/src/metric_record.rsindex eab62d41e929..bf75b9a16688 100644--- a/rust/capture-logs/src/metric_record.rs+++ b/rust/capture-logs/src/metric_record.rs@@ -55,6 +55,23 @@ pub struct KafkaMetricRow {pub retention_days: Option<i32>,}+const POINT_BYTES: u64 = 24;+const LABEL_BYTES: u64 = 8;+const HISTOGRAM_BUCKET_BYTES: u64 = 8;++/// Billable size of one point: 24 bytes, plus 8 bytes for each label and each histogram bucket.+/// The size uses counts, not string lengths, so customers can predict it from what they send.+/// The wire format, batch size and name lengths do not change it.+pub fn compute_kafka_metric_row_bytes(row: &KafkaMetricRow) -> u64 {+ let labels = (row.resource_attributes.len() + row.attributes.len()) as u64;+ let buckets = row.histogram_counts.len() as u64;+ POINT_BYTES + LABEL_BYTES * labels + HISTOGRAM_BUCKET_BYTES * buckets+}++pub fn sum_kafka_metric_row_bytes(rows: &[KafkaMetricRow]) -> u64 {+ rows.iter().map(compute_kafka_metric_row_bytes).sum()+}+/// Flatten an OTEL Metric into one or more KafkaMetricRow records./// Each data point in the metric becomes a separate row.pub fn flatten_metric(@@ -586,6 +603,30 @@ mod tests {row}+ #[test]+ fn test_row_bytes_count_points_labels_and_buckets_not_lengths() {+ let mut row = build_test_row(&[]);+ row.metric_name = "rpc.latency".to_string();+ row.histogram_bounds = vec![1.0, 5.0];+ row.histogram_counts = vec![1, 2, 3];+ row.resource_attributes = HashMap::from([("host".to_string(), "web-1".to_string())]);+ row.attributes = HashMap::from([("route".to_string(), "/api".to_string())]);++ assert_eq!(compute_kafka_metric_row_bytes(&row), 24 + 2 * 8 + 3 * 8);++ row.metric_name = "m".repeat(200);+ row.instrumentation_scope = "s".repeat(200);+ row.unit = "u".repeat(200);+ row.attributes = HashMap::from([("r".repeat(100), "v".repeat(100))]);+ assert_eq!(compute_kafka_metric_row_bytes(&row), 24 + 2 * 8 + 3 * 8);++ // The second row's overridden timestamp adds a `$originalTimestamp` label.+ assert_eq!(+ sum_kafka_metric_row_bytes(&[row, build_test_row(&[])]),+ (24 + 2 * 8 + 3 * 8) + (24 + 8)+ );+ }+#[test]fn test_has_labels_and_retention_days_serialise_into_avro_payload() {use crate::metrics_avro_schema::METRICS_AVRO_SCHEMA;