OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M16-005): Turn batching ON by default per ACCEPTED ADR-023: BATCH_SIZE=25, BATCH_LINGER_MS=5, plus the two follow-ups M16-002 could not reach
ralphe004e19960bf
diff --git a/.env.example b/.env.exampleindex 69dde0b0d..2646db748 100644--- a/.env.example+++ b/.env.example@@ -44,6 +44,15 @@ API_PORT=3333# ingestion (stop producers, let consumers fully catch up) and recreate the# topic — see self-hosting/redpanda/bootstrap.template.yaml.# KAFKA_EVENTS_TOPIC_PARTITIONS="24"+# Producer throughput knobs (ADR-023; docs/KAFKA_PRODUCER_OPTIONS.md).+# How many events are accumulated into one Kafka send(); 1 turns batching off.+# KAFKA_PRODUCER_BATCH_SIZE="25"+# How long a partly filled batch waits for company, in ms; added to the+# /track response time when the batch is not already full.+# KAFKA_PRODUCER_BATCH_LINGER_MS="5"+# How many produce requests may be outstanding at once; raising it buys ~2-3%+# and risks reordering after a broker hiccup, so leave it at 1.+# KAFKA_PRODUCER_MAX_IN_FLIGHT="1"# Local dev with docker-compose:# KAFKA_BROKERS="localhost:19092"diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.tsindex 2a884e80c..30d0005f0 100644--- a/apps/api/src/config/env.ts+++ b/apps/api/src/config/env.ts@@ -29,11 +29,6 @@import type { CoreConfig } from '@openpanel/core';import { queues } from '@openpanel/core';-// The producer's throughput knobs are shaped by the module that applies them-// (ADR-022 R8: a module's `*.constants.ts` is the file an app may import), so-// a renamed field fails to compile on both sides instead of going quietly-// unread. See packages/core/src/modules/ingest/src/producer-tuning.ts.-import type { KafkaProducerTuning } from '@openpanel/core/modules/ingest/ingest.constants';import { z } from 'zod';export const ROLE_VALUES = ['api', 'worker', 'all'] as const;@@ -65,11 +60,18 @@ const DEFAULT_KAFKA_CONNECTION_TIMEOUT_MS = 2000;const DEFAULT_KAFKA_PRODUCER_RETRIES = 2;const DEFAULT_KAFKA_PRODUCER_INITIAL_RETRY_MS = 100;const DEFAULT_KAFKA_PRODUCER_MAX_RETRY_MS = 1000;-/** One produce round-trip at a time — today's behaviour (M16-002 option 4). */+/**+ * One produce round-trip at a time. ADR-023 rejects raising this: measured at+ * +2-3%, i.e. noise, for a documented reordering hazard.+ */const DEFAULT_KAFKA_PRODUCER_MAX_IN_FLIGHT = 1;-/** Messages per `send()`. 1 = batching off (M16-002 option 2). */-const DEFAULT_KAFKA_PRODUCER_BATCH_SIZE = 1;-/** A partial batch's maximum wait. Inert while the batch size is 1. */+/** Messages per `send()`. ADR-023: batching on; the win is flat above 25. */+const DEFAULT_KAFKA_PRODUCER_BATCH_SIZE = 25;+/**+ * A partial batch's maximum wait. The linger is added to the request's own+ * response time, so ADR-023 takes 5 ms (82% of the available gain) over the+ * 25 ms that would roughly double a quiet install's ingest latency.+ */const DEFAULT_KAFKA_PRODUCER_BATCH_LINGER_MS = 5;const DEFAULT_KAFKA_HANDLER_MAX_ATTEMPTS = 3;const DEFAULT_KAFKA_HANDLER_RETRY_INITIAL_MS = 100;@@ -604,17 +606,8 @@ function deriveDashboardUrl(raw: RawEnv): string {return raw.DASHBOARD_URL ?? '';}-/**- * The Kafka transport's knobs, plus the three producer tuning knobs the ingest- * module reads off the same object (`KafkaProducerTuning`). They are its- * fields rather than `KafkaConfig`'s only because- * `packages/core/src/config.ts` is outside M16-002's scope; the follow-up- * moves them onto `KafkaConfig` and this return type becomes plain- * `CoreConfig['kafka']`.- */-function deriveKafkaConfig(- raw: RawEnv-): CoreConfig['kafka'] & KafkaProducerTuning {+/** The Kafka transport's knobs, including the producer's three throughput ones. */+function deriveKafkaConfig(raw: RawEnv): CoreConfig['kafka'] {return {clientId: raw.KAFKA_CLIENT_ID,brokers: raw.KAFKA_BROKERS,diff --git a/docs/KAFKA_PRODUCER_OPTIONS.md b/docs/KAFKA_PRODUCER_OPTIONS.mdindex 7f29bcf6a..2c23fac24 100644--- a/docs/KAFKA_PRODUCER_OPTIONS.md+++ b/docs/KAFKA_PRODUCER_OPTIONS.md@@ -1,12 +1,16 @@-# The Kafka producer's throughput knobs — what to measure, and how+# The Kafka producer's throughput knobs-M16-002. This document exists so the operator's four-configuration measurement-run, and the ADR Carl writes from it (ADR-004's producer-redesign question),-are built on precise statements about which knob produces which behaviour.+M16-002 built these knobs and measured them; **ADR-023 decided them**, and+batching now ships on — `KAFKA_PRODUCER_BATCH_SIZE=25`,+`KAFKA_PRODUCER_BATCH_LINGER_MS=5`. `KAFKA_PRODUCER_MAX_IN_FLIGHT` stays `1`:+raising it measured +2–3%, i.e. noise, for a documented reordering hazard.-**Nothing here is switched on.** Every default reproduces the behaviour-measured in `verification/benchmarks/05-v2-saturation.md`: one produce-round-trip at a time, one message per `send()`, awaited in the request path.+This document is the knob reference — what each one does, and the kafkajs+retry coupling you must understand before touching the in-flight one. The+measurement that fed the decision is in+`verification/benchmarks/05-v2-saturation.md` and `06-producer-options.md`; the+throughput a self-hoster should plan with is in+[`SCALING.md`](./SCALING.md).## The bottleneck being measured@@ -18,14 +22,15 @@ concurrency 25 → 400). The cause is in`maxInFlightRequests: 1`, and one awaited `send()` carrying a **single**message per request. ~800/s is ~1.25 ms per local ack, serialised.-Four options were named. This task makes **2** and **4** measurable:+Four options were named. M16-002 made **2** and **4** measurable; ADR-023 then+adopted 2 and rejected 4:| # | Option | Status ||---|---|---|| 1 | More api processes | Already the cloud deployment's scaling story (Carl). A live fallback, not a candidate — nothing below has to reach 5,000 ev/s in one process. |-| 2 | **Batch: one `send()`, many messages** | Built, default **off** — `KAFKA_PRODUCER_BATCH_SIZE` / `KAFKA_PRODUCER_BATCH_LINGER_MS`. |-| 3 | Don't await the broker; buffer and flush async | **Deliberately not built.** It changes what a 200 response promises — a durability decision for Carl, not a performance tweak. |-| 4 | **Raise `maxInFlightRequests`** | Built, default **1** (unchanged) — `KAFKA_PRODUCER_MAX_IN_FLIGHT`. |+| 2 | **Batch: one `send()`, many messages** | **Adopted (ADR-023), on by default** at `KAFKA_PRODUCER_BATCH_SIZE=25` / `KAFKA_PRODUCER_BATCH_LINGER_MS=5`. Measured 744 → ~1,515 ev/s. |+| 3 | Don't await the broker; buffer and flush async | **Deliberately not built.** It changes what a 200 response promises — a durability decision for Carl, not a performance tweak. ADR-023 defers it, it does not reject it. |+| 4 | **Raise `maxInFlightRequests`** | **Rejected (ADR-023).** Still settable via `KAFKA_PRODUCER_MAX_IN_FLIGHT`, default **1**. |## The knobs@@ -35,9 +40,9 @@ All three are read by the config loader, `apps/api/src/config/env.ts` (the sole| Env var | Default | What it does ||---|---|---|-| `KAFKA_PRODUCER_MAX_IN_FLIGHT` | `1` | kafkajs `maxInFlightRequests`: how many produce requests may be outstanding on the producer's connection at once. `1` is today's value and serialises every produce. Raising it lets requests overlap — read the retry coupling below before you do. |-| `KAFKA_PRODUCER_BATCH_SIZE` | `1` | Messages accumulated into ONE `send()`. `1` means batching is off and the code path is byte-for-byte today's: one message, one awaited send. `>= 2` turns the accumulator on. |-| `KAFKA_PRODUCER_BATCH_LINGER_MS` | `5` | How long a partially filled batch may wait for company before it is sent anyway. **Inert while the batch size is 1.** |+| `KAFKA_PRODUCER_MAX_IN_FLIGHT` | `1` | kafkajs `maxInFlightRequests`: how many produce requests may be outstanding on the producer's connection at once. `1` serialises every produce. Raising it lets requests overlap — ADR-023 measured that at +2–3% and rejected it; read the retry coupling below before you do it anyway. |+| `KAFKA_PRODUCER_BATCH_SIZE` | `25` | Messages accumulated into ONE `send()`. `>= 2` turns the accumulator on; ADR-023 ships `25`, above which the measured win is flat. `1` turns batching off, restoring the pre-ADR-023 path byte-for-byte: one message, one awaited send. |+| `KAFKA_PRODUCER_BATCH_LINGER_MS` | `5` | How long a partially filled batch may wait for company before it is sent anyway, **added to the response time of the request that is waiting**. Under load a batch fills by size first, so this is only really paid on a quiet install — which is why ADR-023 took 5 ms over the 21%-faster 25 ms. **Inert while the batch size is 1.** |Positive integers only; a blank or malformed value fails boot, like every other`KAFKA_*` knob. The resolved values are logged once, on the@@ -102,7 +107,7 @@ making it measurable.Note that option 2 sidesteps this entirely: batching amortises the round-tripwhile leaving `maxInFlightRequests`, idempotency and ordering exactly as they-are. That is why it is the likelier candidate for adoption.+are. That is why it is the one ADR-023 adopted.## Reproducing the four configurations@@ -112,11 +117,14 @@ documented in `docs/BENCHMARK_HARNESS.md`. The producer knobs belong to theand the stack is restarted between configurations.```bash-# 1. BASELINE — today's behaviour. No producer env at all.-cd apps/api && API_PORT=3333 bun run dev+# 1. BASELINE — the pre-ADR-023 path. Batching must be turned OFF explicitly+# now that it ships on.+cd apps/api && API_PORT=3333 KAFKA_PRODUCER_BATCH_SIZE=1 bun run dev# 2. HIGHER IN-FLIGHT (option 4)-cd apps/api && API_PORT=3333 KAFKA_PRODUCER_MAX_IN_FLIGHT=5 bun run dev+cd apps/api && API_PORT=3333 \+ KAFKA_PRODUCER_BATCH_SIZE=1 KAFKA_PRODUCER_MAX_IN_FLIGHT=5 \+ bun run dev# 3. BATCHING (option 2)cd apps/api && API_PORT=3333 \@@ -156,7 +164,7 @@ configuration 3 (e.g. size 10/25/100 at linger 5/10/25): the batch size isbounded in practice by how many requests are actually in flight at the offeredconcurrency, and a batch size far above that only ever fills by linger timeout.-## Proof runs performed by this task (not the measurement)+## Proof runs performed by M16-002 (not the measurement)Run by the agent on 2026-09-12 on the same 4-core box, against apurpose-booted `ROLE=api` api on `API_PORT=3399` producing into the running@@ -167,8 +175,8 @@ differ from each other by 50 %.| configuration | `session-stress` emit window | P50 | P95 | checks ||---|---|---|---|---|-| default (`maxInFlight=1`, batching off), run 1 | 0.9 s | 91.7 ms | 142.9 ms | 13/13 |-| default, run 2 | 0.6 s | 67.0 ms | 79.9 ms | 13/13 |+| M16-002's then-default (`maxInFlight=1`, batching off), run 1 | 0.9 s | 91.7 ms | 142.9 ms | 13/13 |+| same, run 2 | 0.6 s | 67.0 ms | 79.9 ms | 13/13 || `MAX_IN_FLIGHT=5 BATCH_SIZE=25 LINGER_MS=10`, run 1 | 0.5 s | 48.4 ms | 77.3 ms | 13/13 || same, run 2 | 0.5 s | 45.7 ms | 95.2 ms | 13/13 |@@ -182,36 +190,36 @@ the linger flush works: every one of those events left as a partial batch.## What is still unknown-- **Whether either option moves the plateau at all.** Nothing here has been- measured at saturation. The ~800 ev/s ceiling is attributed to the- serialised produce path by elimination (idle CPU, clean event loop, consumer- lag ≈ 0), not by an experiment that removed the serialisation.-- **Whether the produce path is the only serialising dependency.**- `05-v2-saturation.md` names two other candidates it did not exonerate: the- Redis session-state operations and the event buffer. If the plateau holds- with batching on, the next suspect is one of those, not a bigger batch.-- **The right batch size and linger for production**, and whether a linger of- a few ms is acceptable in the request path at all — it is latency added to- every `/track` that is not in a full batch.+*(The first three entries here were answered by the ADR-023 measurement runs;+what remains is what those runs did not settle.)*++- **What binds the box above ~1,900 ev/s.** Batching moved the ceiling and+ lowered api CPU peak from 102% to ~79% of a core, so the produce path was+ the constraint. The next one is CPU on this box, not a known backend limit:+ two api processes aggregated 1,936 ev/s (+28%, not 2x) with each peaking+ lower than one did alone. Redpanda, ClickHouse and Redis were never shown to+ bind — establishing that needs the load generated off-box or more cores.+- **Whether the produce path was the only serialising dependency.**+ `05-v2-saturation.md` named two other candidates it did not exonerate: the+ Redis session-state operations and the event buffer. Batching did not make+ them go away; it made them the next thing to look at.+- **Whether a `linger.ms=0` semantic would recover the rest.** ADR-023 names+ it as the follow-up worth taking: send immediately but batch whatever+ accumulates while the previous send is in flight. It is not expressible+ today — the schema uses `positive()` so `0` is rejected, and the accumulator+ flushes on a timer rather than on in-flight state.- **The durability posture.** Both the retry coupling above and option 3 are- Carl's to decide; this task deliberately leaves both alone.+ Carl's to decide; ADR-023 deliberately left both alone.- **Ordering under `maxInFlightRequests > 1` without batching** is exactly therisk the existing in-code comment describes, and it is unmeasured: aconfiguration-2 run that never hits a broker hiccup proves nothing aboutwhat happens when one occurs.-## One structural note for whoever lands the follow-up--The three fields ride on `config.kafka`, but they are declared as the optional-members of `KafkaProducerTuning` in-`packages/core/src/modules/ingest/ingest.constants.ts` rather than on-`KafkaConfig` in `packages/core/src/config.ts` — that file is outside-M16-002's declared scope. The loader always sets all three; the fallbacks in-`src/producer-tuning.ts` only apply to a `CoreConfig` built by a test fixture,-and they reproduce today's behaviour. The follow-up is one edit: move the-three fields onto `KafkaConfig` as required numbers, add them to-`packages/core/test/config-fixture.ts`, delete `KafkaProducerTuning` and the-fallbacks, and drop the intersection from `deriveKafkaConfig`'s return type.--The three env vars are also missing from `.env.example`, for the same reason —-it is not in this task's scope.+## Where the three knobs live++All three are required fields of `KafkaConfig` in+`packages/core/src/config.ts`, set by the loader+(`apps/api/src/config/env.ts`) and resolved into `batchingEnabled` by+`packages/core/src/modules/ingest/src/producer-tuning.ts`. There are no+fallbacks and no second default anywhere; `packages/core/test/config-fixture.ts`+carries the same shipped values for tests. They are listed in `.env.example`.diff --git a/docs/SCALING.md b/docs/SCALING.mdindex fce1d32a2..3e487f2fd 100644--- a/docs/SCALING.md+++ b/docs/SCALING.md@@ -16,9 +16,10 @@ at a different machine.| | events/second | events/month, flat out ||---|---|---|-| **What ships today** (no producer batching) | **744** | **1.96 bn** |-| With batching enabled (batch 25 / linger 25 ms) | 1,904 | 5.01 bn |-| Two api processes, batch 25 / linger 5 ms | 1,936 aggregate | 5.09 bn |+| **What ships today** (producer batching on: batch 25 / linger 5 ms) | **1,515** | **3.98 bn** |+| Batching turned off (`KAFKA_PRODUCER_BATCH_SIZE=1`) | 744 | 1.96 bn |+| A 25 ms linger instead of 5 ms | 1,904 | 5.01 bn |+| Two api processes at the shipped defaults | 1,936 aggregate | 5.09 bn |"Flat out" means every second of the month at that rate. **That is not yourcapacity.** Real traffic has a daily peak several times its average, and the box@@ -26,8 +27,9 @@ has to survive the peak. Skip to[Peak vs average](#peak-vs-average-the-only-number-worth-planning-with) —that section is why this page exists.-Batching is **off by default** and turning it on is pending an accepted ADR, so-the 744 ev/s row is what an unmodified install does today.+Producer batching is **on by default** (ADR-023: batch 25, linger 5 ms), so the+1,515 ev/s row is what an unmodified install does today. The 744 ev/s row is+what you get if you turn batching off.---@@ -55,12 +57,12 @@ after the load stops — not a burst rate.| Configuration | events/second | events/month, flat out ||---|---|---|-| No batching — **today's default** | **744** | 1,956,571,200 (**1.96 bn**) |+| Batch 25 / linger 5 ms — **today's default** | **1,515** | 3,984,147,000 (**3.98 bn**) |+| No batching (`KAFKA_PRODUCER_BATCH_SIZE=1`) | 744 | 1,956,571,200 (1.96 bn) || `maxInFlightRequests: 5`, no batching | 769 | 2,022,316,200 (2.02 bn) |-| Batch 25 / linger 5 ms | 1,515 | 3,984,147,000 (3.98 bn) || Batch 25 / linger 10 ms | 1,716 | 4,512,736,800 (4.51 bn) || Batch 25 / linger 25 ms | 1,904 | 5,007,139,200 (5.01 bn) |-| **Two api processes**, batch 25 / linger 5 ms | 1,936 aggregate (980 + 955) | 5,091,292,800 (5.09 bn) |+| **Two api processes**, at the shipped defaults | 1,936 aggregate (980 + 955) | 5,091,292,800 (5.09 bn) |Two results worth stating plainly, because they close off tuning directionsthat look promising:@@ -94,8 +96,8 @@ Worked, flat out — 100% duty cycle, every second of the month at the peak rate| events/second | x 2,629,800 | = events/month ||---|---|---|-| 744 (today's default) | 744 x 2,629,800 | 1,956,571,200 — **1.96 bn** |-| 1,515 | 1,515 x 2,629,800 | 3,984,147,000 — **3.98 bn** |+| 744 (batching off) | 744 x 2,629,800 | 1,956,571,200 — **1.96 bn** |+| 1,515 (today's default) | 1,515 x 2,629,800 | 3,984,147,000 — **3.98 bn** || 1,904 | 1,904 x 2,629,800 | 5,007,139,200 — **5.01 bn** || 1,936 | 1,936 x 2,629,800 | 5,091,292,800 — **5.09 bn** || 5,000 (the project's stated target) | 5,000 x 2,629,800 | 13,149,000,000 — **13.15 bn** |@@ -124,8 +126,8 @@ Worked at a ratio of 3:1:| peak capacity | average sustained | monthly events at 3:1 ||---|---|---|-| 744 ev/s (today's default) | 248 ev/s | **~0.65 bn/month** |-| 1,515 ev/s (batch 25 / linger 5 ms) | 505 ev/s | **~1.33 bn/month** |+| 1,515 ev/s (today's default) | 505 ev/s | **~1.33 bn/month** |+| 744 ev/s (batching off) | 248 ev/s | **~0.65 bn/month** || 1,904 ev/s (batch 25 / linger 25 ms) | ~635 ev/s | **~1.67 bn/month** || 1,936 ev/s (two api processes) | ~645 ev/s | **~1.70 bn/month** |@@ -143,20 +145,29 @@ still falls behind every afternoon.----## Turning batching on+## Batching, and the latency it costsBatching amortises one Kafka produce round-trip across many events, which is-where the 744 → ~1,900 ev/s difference comes from. It is configured with+where the 744 → ~1,515 ev/s difference comes from. It is configured with`KAFKA_PRODUCER_BATCH_SIZE` and `KAFKA_PRODUCER_BATCH_LINGER_MS`, bothdocumented in [`KAFKA_PRODUCER_OPTIONS.md`](./KAFKA_PRODUCER_OPTIONS.md).-**It is off by default and it stays off until an ADR accepts it.** The measured-rows above exist to inform that decision, not to describe what your install is-doing right now. The trade it makes is latency: a linger of *N* ms is up to *N*-ms added to every `/track` request that is not in an already-full batch, and the-figures above show larger lingers buying more throughput (5 ms → 1,515,-10 ms → 1,716, 25 ms → 1,904). Whether that latency belongs in the request path-is exactly the open question.+**It is on by default at batch 25 / linger 5 ms** (ADR-023). The trade it makes+is latency: a linger of *N* ms is up to *N* ms added to every `/track` request+that is not in an already-full batch, and larger lingers buy more throughput+(5 ms → 1,515, 10 ms → 1,716, 25 ms → 1,904).++5 ms is the default because that latency is only actually paid on a **quiet**+install — under load a batch fills by size before the timer ever fires, which+is why measured P50 is 168–194 ms across a 12x range of linger values. On a+busy box a 25 ms linger is free and worth 21% more throughput; on a quiet one+it would roughly double an ingest request's latency, and that is the install+the default has to be safe for.++To turn batching off entirely and get the 744 ev/s behaviour back, set+`KAFKA_PRODUCER_BATCH_SIZE=1`. Nothing about durability changes either way: the+request still waits for the broker to acknowledge its own event before it+answers 200, so a crash mid-linger means no 200 and the client retries.---diff --git a/packages/core/src/config.ts b/packages/core/src/config.tsindex 0843b4f80..a61822b40 100644--- a/packages/core/src/config.ts+++ b/packages/core/src/config.ts@@ -106,6 +106,12 @@ export interface KafkaConfig {producerRetries: number;producerInitialRetryMs: number;producerMaxRetryMs: number;+ /** kafkajs `maxInFlightRequests`. 1: one produce round-trip at a time. */+ producerMaxInFlight: number;+ /** Messages accumulated into one `send()`. Below 2 batching is off. */+ producerBatchSize: number;+ /** A partial batch's maximum wait, in ms, before it is sent anyway. */+ producerBatchLingerMs: number;handlerMaxAttempts: number;handlerRetryInitialMs: number;handlerRetryMaxMs: number;diff --git a/packages/core/src/modules/ingest/ingest.constants.ts b/packages/core/src/modules/ingest/ingest.constants.tsindex 8f4970c43..2a50fd571 100644--- a/packages/core/src/modules/ingest/ingest.constants.ts+++ b/packages/core/src/modules/ingest/ingest.constants.ts@@ -396,24 +396,3 @@ export interface DeprecatedDecrementProfilePayload {property: string;value: number;}--// ----- The Kafka producer's throughput knobs (M16-002) ------//-// `apps/api/src/config/env.ts` is the only reader of `process.env` (ADR-022-// R7), and `<name>.constants.ts` is the one file of this module it may import-// (R8) — so the shape of the three knobs it sets lives here, and both sides-// break at compile time if a field is renamed. `src/producer-tuning.ts`-// resolves them; `src/kafka.ts` applies them.-//-// They are optional because they ride on `config.kafka`, whose `KafkaConfig`-// interface (`packages/core/src/config.ts`) is outside M16-002's scope and so-// cannot declare them. See the header of `src/producer-tuning.ts`.--export interface KafkaProducerTuning {- /** kafkajs `maxInFlightRequests`. Default 1: one round-trip at a time. */- producerMaxInFlight?: number;- /** Messages per `send()`. Default 1: batching off. */- producerBatchSize?: number;- /** A partial batch's maximum wait, in ms. Inert while batching is off. */- producerBatchLingerMs?: number;-}diff --git a/packages/core/src/modules/ingest/src/kafka.ts b/packages/core/src/modules/ingest/src/kafka.tsindex cdb9745a3..02c75f6f1 100644--- a/packages/core/src/modules/ingest/src/kafka.ts+++ b/packages/core/src/modules/ingest/src/kafka.ts@@ -88,12 +88,13 @@ const getProducer = async (config: CoreConfig): Promise<Producer> => {const tuning = resolveProducerTuning(config.kafka);const p = client.producer({idempotent: true,- // Default 1 (not 5) to avoid in-flight reordering after a transient+ // Stays 1 (not 5) to avoid in-flight reordering after a transient// broker hiccup: with idempotency on and low retries, reordered batches// trip OUT_OF_ORDER_SEQUENCE_NUMBER and stick the producer per-partition.- // KAFKA_PRODUCER_MAX_IN_FLIGHT raises it for a measurement run — read- // docs/KAFKA_PRODUCER_OPTIONS.md first: it is coupled to the retry- // policy, which is why the two are measured together and not alone.+ // ADR-023 measured raising it at +2-3% and rejected it; batching is what+ // amortises the round-trip. KAFKA_PRODUCER_MAX_IN_FLIGHT still raises it+ // for a measurement run — read docs/KAFKA_PRODUCER_OPTIONS.md first: it+ // is coupled to the retry policy.maxInFlightRequests: tuning.maxInFlight,allowAutoTopicCreation: true,retry: {diff --git a/packages/core/src/modules/ingest/src/producer-tuning.test.ts b/packages/core/src/modules/ingest/src/producer-tuning.test.tsindex 2a4a027dd..c1f305a0b 100644--- a/packages/core/src/modules/ingest/src/producer-tuning.test.ts+++ b/packages/core/src/modules/ingest/src/producer-tuning.test.ts@@ -1,18 +1,22 @@/**- * M16-002's non-negotiable: every default reproduces today's behaviour — one- * produce round-trip at a time, one message per `send()`. A config that says- * nothing about the knobs must therefore resolve to exactly that.+ * ADR-023's decision, expressed as a test: batching is ON at the shipped+ * defaults (size 25, linger 5 ms) and `maxInFlightRequests` stays 1. There are+ * no fallbacks left to test — the three knobs are required `KafkaConfig`+ * fields, so a config that omits one does not compile.*/import { describe, expect, test } from 'bun:test';-import {- resolveProducerTuning,- type TunedKafkaConfig,-} from './producer-tuning';+import type { KafkaConfig } from '../../../config';+import { resolveProducerTuning } from './producer-tuning';-const kafkaConfig = (- tuning: Partial<TunedKafkaConfig> = {}-): TunedKafkaConfig =>+/** The values `apps/api/src/config/env.ts` resolves for an empty environment. */+const SHIPPED_DEFAULTS = {+ producerMaxInFlight: 1,+ producerBatchSize: 25,+ producerBatchLingerMs: 5,+} as const;++const kafkaConfig = (tuning: Partial<KafkaConfig> = {}): KafkaConfig =>({clientId: 'openpanel',brokers: [],@@ -30,44 +34,41 @@ const kafkaConfig = (producerRetries: 2,producerInitialRetryMs: 100,producerMaxRetryMs: 1000,+ ...SHIPPED_DEFAULTS,handlerMaxAttempts: 3,handlerRetryInitialMs: 100,handlerRetryMaxMs: 1000,...tuning,- }) satisfies TunedKafkaConfig;+ }) satisfies KafkaConfig;describe('resolveProducerTuning', () => {- test('an unconfigured config reproduces the pre-M16-002 behaviour', () => {+ test('the shipped defaults batch at 25 with a 5 ms linger', () => {const tuning = resolveProducerTuning(kafkaConfig());- expect(tuning.maxInFlight).toBe(1);+ expect(tuning.batchSize).toBe(25);+ expect(tuning.lingerMs).toBe(5);+ expect(tuning.batchingEnabled).toBe(true);+ });++ test('ADR-023 leaves maxInFlightRequests at one', () => {+ expect(resolveProducerTuning(kafkaConfig()).maxInFlight).toBe(1);+ });++ test('a batch size of one turns batching off again', () => {+ const tuning = resolveProducerTuning(kafkaConfig({ producerBatchSize: 1 }));expect(tuning.batchSize).toBe(1);expect(tuning.batchingEnabled).toBe(false);});- test('the loader-supplied defaults also leave batching off', () => {+ test('the knobs pass through to kafkajs untouched', () => {const tuning = resolveProducerTuning(kafkaConfig({- producerMaxInFlight: 1,- producerBatchSize: 1,- producerBatchLingerMs: 5,+ producerMaxInFlight: 5,+ producerBatchSize: 50,+ producerBatchLingerMs: 10,}));- expect(tuning.maxInFlight).toBe(1);- expect(tuning.batchingEnabled).toBe(false);- });-- test('a batch size of two or more turns batching on', () => {- const tuning = resolveProducerTuning(- kafkaConfig({ producerBatchSize: 50, producerBatchLingerMs: 10 })- );+ expect(tuning.maxInFlight).toBe(5);expect(tuning.batchSize).toBe(50);expect(tuning.lingerMs).toBe(10);- expect(tuning.batchingEnabled).toBe(true);- });-- test('the in-flight knob passes through to kafkajs untouched', () => {- expect(- resolveProducerTuning(kafkaConfig({ producerMaxInFlight: 5 })).maxInFlight- ).toBe(5);});});diff --git a/packages/core/src/modules/ingest/src/producer-tuning.ts b/packages/core/src/modules/ingest/src/producer-tuning.tsindex eba0d4069..ea1da1247 100644--- a/packages/core/src/modules/ingest/src/producer-tuning.ts+++ b/packages/core/src/modules/ingest/src/producer-tuning.ts@@ -1,50 +1,29 @@-// The Kafka producer's throughput knobs (M16-002), resolved in one place.+// The Kafka producer's throughput knobs (M16-002, turned on by ADR-023),+// resolved in one place.//-// WHERE THESE FIELDS ARE DECLARED, AND WHY IT IS NOT `KafkaConfig`:-// the config loader (`apps/api/src/config/env.ts`) sets them and they ride on-// `config.kafka`, but they are declared as the optional members of-// `KafkaProducerTuning` in this module's `ingest.constants.ts` — the one file-// of this module an app may import (ADR-022 R8) — rather than on `KafkaConfig`-// in `packages/core/src/config.ts`, which is outside M16-002's declared scope.-// Follow-up, one edit: move the three fields onto `KafkaConfig` as required-// numbers, drop `KafkaProducerTuning`, and delete the fallbacks below.-//-// The fallbacks are NOT a second production default: the loader always sets-// all three, so they only apply to a `CoreConfig` built by a test fixture.-// They reproduce today's behaviour exactly — one produce round-trip at a-// time, batching off.+// The three fields are required members of `KafkaConfig`+// (`packages/core/src/config.ts`): the config loader always sets them, so+// there is no fallback here and no second default anywhere. This file only+// decides whether the configured size batches at all.import type { KafkaConfig } from '../../../config';-import type { KafkaProducerTuning } from '../ingest.constants';--/** kafkajs `maxInFlightRequests`: one produce round-trip at a time. */-const FALLBACK_MAX_IN_FLIGHT = 1;-/** Messages per `send()`. 1 = batching off — one message per send, as today. */-const FALLBACK_BATCH_SIZE = 1;-/** A partial batch's maximum wait. Inert while batching is off. */-const FALLBACK_BATCH_LINGER_MS = 5;/** The smallest batch size that batches anything. */const BATCHING_MIN_SIZE = 2;-export type TunedKafkaConfig = KafkaConfig & KafkaProducerTuning;-export interface ResolvedProducerTuning {maxInFlight: number;batchSize: number;lingerMs: number;- /** False keeps the pre-M16-002 path: one message per awaited `send()`. */+ /** False keeps the pre-ADR-023 path: one message per awaited `send()`. */batchingEnabled: boolean;}export const resolveProducerTuning = (- kafka: TunedKafkaConfig-): ResolvedProducerTuning => {- const batchSize = kafka.producerBatchSize ?? FALLBACK_BATCH_SIZE;- return {- maxInFlight: kafka.producerMaxInFlight ?? FALLBACK_MAX_IN_FLIGHT,- batchSize,- lingerMs: kafka.producerBatchLingerMs ?? FALLBACK_BATCH_LINGER_MS,- batchingEnabled: batchSize >= BATCHING_MIN_SIZE,- };-};+ kafka: KafkaConfig+): ResolvedProducerTuning => ({+ maxInFlight: kafka.producerMaxInFlight,+ batchSize: kafka.producerBatchSize,+ lingerMs: kafka.producerBatchLingerMs,+ batchingEnabled: kafka.producerBatchSize >= BATCHING_MIN_SIZE,+});diff --git a/packages/core/test/config-fixture.ts b/packages/core/test/config-fixture.tsindex 1c929225f..5a7fa0307 100644--- a/packages/core/test/config-fixture.ts+++ b/packages/core/test/config-fixture.ts@@ -75,6 +75,9 @@ export function testCoreConfig(producerRetries: 2,producerInitialRetryMs: 100,producerMaxRetryMs: 1000,+ producerMaxInFlight: 1,+ producerBatchSize: 25,+ producerBatchLingerMs: 5,handlerMaxAttempts: 3,handlerRetryInitialMs: 100,handlerRetryMaxMs: 1000,