Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): run the rust logs consumer on kafka with a shadow mode (#127753)

Frank Hamand05ec28dd4944
diff --git a/.github/rust-images.yml b/.github/rust-images.yml
index 118018702a3b..0bf899dc9c07 100644
--- a/.github/rust-images.yml
+++ b/.github/rust-images.yml
@@ -101,6 +101,13 @@
dockerfile: ./rust/Dockerfile
project: mtvzx98zjn
+# rust/logs-ingestion. Named apart from the Node logs-ingestion release, and
+# shares ingestion-consumer's Depot project because both build on ingestion-framework.
+- image: logs-ingestion-rust
+ bin: logs-ingestion
+ dockerfile: ./rust/Dockerfile
+ project: mtvzx98zjn
+
# Internal web tool for the ingestion team (lag analysis, consumer debug
# visualizer).
- image: ingestion-control-plane
diff --git a/.github/workflows/rust-docker-build.yml b/.github/workflows/rust-docker-build.yml
index f95041fa4773..81cad2e91cc5 100644
--- a/.github/workflows/rust-docker-build.yml
+++ b/.github/workflows/rust-docker-build.yml
@@ -166,6 +166,9 @@ jobs:
- release: ingestion-consumer
digest_key: ingestion-consumer
values_key: image
+ - release: logs-ingestion-rust
+ digest_key: logs-ingestion-rust
+ values_key: image
- release: ingestion-control-plane
digest_key: ingestion-control-plane
values_key: image
diff --git a/nodejs/src/logs/benchmarks/logs-consumer-bench.ts b/nodejs/src/logs/benchmarks/logs-consumer-bench.ts
index 799859afd89c..6343d12f50cc 100644
--- a/nodejs/src/logs/benchmarks/logs-consumer-bench.ts
+++ b/nodejs/src/logs/benchmarks/logs-consumer-bench.ts
@@ -89,7 +89,6 @@ async function run(implementation: string, workloadPath: string, passes: number,
outputs.producedCount = 0
const batchMs: number[] = []
- let peakRss = process.memoryUsage().rss
const cpuStart = process.cpuUsage()
const wallStart = performance.now()
for (let pass = 0; pass < passes; pass++) {
@@ -97,11 +96,12 @@ async function run(implementation: string, workloadPath: string, passes: number,
const start = performance.now()
await runBatch(i)
batchMs.push(performance.now() - start)
- peakRss = Math.max(peakRss, process.memoryUsage().rss)
}
}
const wallMs = performance.now() - wallStart
const cpu = process.cpuUsage(cpuStart)
+ // The process high-water mark in KiB, as the Rust benchmark reads VmHWM.
+ const peakRssKb = process.resourceUsage().maxRSS
const messagesPerPass = scenario.batches.reduce((n, batch) => n + batch.length, 0)
const recordsPerPass = scenario.batches.reduce(
@@ -123,7 +123,7 @@ async function run(implementation: string, workloadPath: string, passes: number,
produced: outputs.producedCount,
wall_ms: wallMs,
cpu_ms: (cpu.user + cpu.system) / 1000,
- peak_rss_mb: peakRss / 1024 / 1024,
+ peak_rss_mb: peakRssKb / 1024,
messages_per_s: (messagesPerPass * passes) / seconds,
records_per_s: (recordsPerPass * passes) / seconds,
input_mb_per_s: (bytesPerPass * passes) / 1024 / 1024 / seconds,
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 83e0d88c3aa7..5d62f893d10e 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -7743,26 +7743,37 @@ dependencies = [
"ahash 0.8.12",
"anyhow",
"async-trait",
+ "axum 0.7.9",
"base64 0.22.1",
+ "bytes",
"chrono",
"common-alloc",
+ "common-database",
"futures",
"hashbrown 0.15.5",
"hex",
"hmac 0.12.1",
"ingestion-framework",
"itoa",
+ "lifecycle",
"metrics",
"rand 0.8.5",
+ "rdkafka",
"redis",
"regex",
"regex-automata",
+ "reqwest 0.12.28",
"serde",
"serde_json",
+ "serve-metrics",
"sha2 0.10.9",
+ "sqlx",
"thiserror 2.0.18",
"tokio",
+ "tonic 0.12.3",
"tracing",
+ "tracing-subscriber",
+ "usage-ingestion-proto",
"uuid",
"zstd 0.13.3",
]
diff --git a/rust/logs-ingestion/Cargo.toml b/rust/logs-ingestion/Cargo.toml
index 71b353b953f6..b5c1ee606c96 100644
--- a/rust/logs-ingestion/Cargo.toml
+++ b/rust/logs-ingestion/Cargo.toml
@@ -11,26 +11,37 @@ path = "src/lib.rs"
ahash = { workspace = true }
anyhow = { workspace = true }
async-trait = { workspace = true }
+axum = { workspace = true }
base64 = { workspace = true }
+bytes = { workspace = true }
chrono = { workspace = true }
common-alloc = { path = "../common/alloc" }
+common-database = { path = "../common/database" }
futures = { workspace = true }
hashbrown = { version = "0.15", default-features = false }
hex = "0.4"
hmac = { workspace = true }
itoa = { workspace = true }
ingestion-framework = { path = "../ingestion-framework" }
+lifecycle = { path = "../common/lifecycle" }
metrics = { workspace = true }
rand = { workspace = true }
+rdkafka = { workspace = true }
redis = { workspace = true }
regex = { workspace = true }
regex-automata = "0.4"
+reqwest = { workspace = true }
serde = { workspace = true }
+serve-metrics = { path = "../common/serve_metrics" }
serde_json = { workspace = true }
sha2 = { workspace = true }
+sqlx = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
+tonic = { workspace = true }
tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+usage-ingestion-proto = { path = "../usage-ingestion-proto" }
uuid = { workspace = true }
zstd = { workspace = true }
diff --git a/rust/logs-ingestion/PORTING.md b/rust/logs-ingestion/PORTING.md
index db4793a075fc..56f1bd01e21f 100644
--- a/rust/logs-ingestion/PORTING.md
+++ b/rust/logs-ingestion/PORTING.md
@@ -29,13 +29,15 @@ Node writes lone surrogates as `\udXXX`, and serde_json rejects them.
| `log-record-avro.ts` (normalize, flatten, enrich), `log-record-size.ts` | `normalize` | Done |
| `sampling/filter-group-match.ts` | `filter_group` | Done |
| `sampling/property-filter-match.ts`, node-re2 `new RE2(p, flags).test` | `property_filter` | Done |
-| `sampling/compile-rules.ts`, `sampling/evaluate.ts`, `sampling/logs-sampling.service.ts` | `sampling` | Done (rules cache not ported) |
+| `sampling/compile-rules.ts`, `sampling/evaluate.ts`, `sampling/logs-sampling.service.ts` | `sampling` | Done |
| `retention/*.ts` except the rules and tracing-config caches | `retention` | Done |
-| `metrics-rules/*.ts` | `metric_rules` | Done (HTTP POST and Postgres cache: traits) |
+| `metrics-rules/*.ts` | `metric_rules` | Done (POST and cache in `service`) |
| `internal-logs.ts` | `internal_logs` | Done |
| `pipeline/log-processing-pipeline.ts` | `content_weights` | Done |
| `services/logs-rate-limiter.service.ts`, `common/services/keyed-rate-limiter.service.ts` | `rate_limiter` | Done (Redis behind `TokenBucketRedis`) |
| `pipeline/*.ts`, `logs-ingestion-consumer.ts`, `traces-ingestion-consumer.ts`, `processLogMessageBuffer`, `app-metrics-aggregator.ts` | `pipeline` | Done (hog transformations: trait slot only) |
+| `servers/ingestion-logs-server.ts`, `servers/ingestion-traces-server.ts`, `logs/config.ts`, `logs/outputs/*`, `common/kafka/consumer/consumer-v2.ts` (batch loop) | `service`, `bin/logs-ingestion.rs` | Done, with shadow mode |
+| `common/utils/team-manager.ts`, `common/utils/lazy-loader.ts`, `quota-limiting.service.ts`, `sampling-rules-cache.ts`, `retention-rules-cache.ts`, `tracing-config-cache.ts`, `metric-rules-cache.ts`, `pattern-message-keys-cache.ts`, `metrics-emitter.ts` (POST), `usage-ingestion/*` | `service` | Done |
## API
@@ -233,17 +235,31 @@ A parsed `FilterGroupNode` implements `metric_rules::FilterGroupMatcher` with `f
- Dependencies are traits in `pipeline::deps`. The caller passes the token-bucket transport, the Redis key root and the clock, so a test can pin all three.
- `pipeline::scenario` builds a consumer from a parity scenario with the in-memory dependencies of `parity-harness.ts`. `tests/parity_corpus.rs` replays all scenarios with it; `PARITY_SCENARIO=<name>` runs one.
- `src/bin/logs-ingestion-bench.rs` runs a workload in the same format and prints the Node benchmark's result JSON.
+- `scenario_parts` returns the consumer inputs of a scenario before they become a consumer, so a test can swap the outputs.
+- `LogsIngestionConsumerDeps::source_offset_header` stamps `source_offset` on `logs` outputs. Only shadow mode sets it.
+- `KeyRoot::Shadow` (`@posthog-logs-rust-shadow`) is the shadow's Redis key root.
+
+### `service`
+
+[README.md](README.md) covers configuration, the modes and the shadow guards.
+
+- `config::ServiceConfig::from_env(&Env)` reads the Node keys with `overrideConfigWithEnv` semantics (`node_env`), and in shadow mode (`LOGS_RUST_SHADOW_MODE`, default true) appends the shadow suffixes to the group and the output topics.
+- `KAFKA_CONSUMER_*` parses as `parseEnvToRdkafkaConfig` with `skipKeysInDefaultConfig` (`config::NODE_DEFAULT_CONFIG_CONSUMER_KEYS`). `kafka_source::consumer_client_config` adds Node's `auto.offset.reset=earliest` default and applies `stripClassicProtocolConfig` last.
+- `consumer_loop::run_consumer_loop(source, processor, config, shutdown, on_tick)` is the batch loop. `BatchSource` (Kafka: `kafka_source::KafkaBatchSource`) and `BatchProcessor` (implemented by `LogsIngestionConsumer`) are its seams.
+- `kafka_outputs::KafkaOutputs` implements `IngestionOutputs` and awaits each delivery report.
+- `postgres` holds `PgTeamManager` and the rule caches, `redis_deps` holds `RedisQuotaLimiting` and a reconnecting Redis connection, `usage::GrpcUsageRecordBatch` the usage client, `lazy_loader::LazyLoader` the `LazyLoader` port.
+- `build_deps` builds what the Node server builds. `shadow_deps` applies the shadow rules to any deps.
+- Postgres errors from a lost connection, a pool timeout or one of Node's `POSTGRES_UNAVAILABLE_ERROR_MESSAGES` become `DependencyUnavailableError`.
+- `kafka_outputs::classify_produce_error` keeps the rdkafka code as a `KafkaProduceError`, named `MessageSizeTooLarge` for code 10 and `Error` otherwise, so the `logs` output rule applies to real delivery errors. An error with no code is a plain `Error`.
## Not reproduced
- **Writer schemas that fail to parse.** avsc's error text comes from `JSON.parse` or `Type.forSchema`. Capture always writes a valid schema.
- **Writer-schema validation.** avsc's `Type.forSchema` checks are not ported, so accept and reject decisions differ, not only error text. A corrupted record `name` key makes an anonymous record, which avsc decodes and this port rejects. A name with invalid characters (`LogRec\u007frd`) is rejected by avsc and decoded here. Ambiguous unions, field defaults and enum symbols are not validated either.
-- **Counts that run far past the data.** For an array or map count in the millions whose items run past the end, avsc keeps reading empty items and Node can throw "Invalid array length" or run out of memory; this port stops at the end with "truncated block". Values that read no bytes, records and array items alike, share a cap of 2^20 per container ("too many empty records"); Node holds them all or runs out of memory.
+- **Counts that run far past the data.** For an array or map count in the millions whose items run past the end, avsc keeps reading empty items and Node can throw "Invalid array length" or run out of memory; this port stops at the end with "truncated block". Values that read no bytes, at any depth, share a cap of 2^20 per container ("too many empty records"); Node holds them all or runs out of memory.
- **An `int` that decodes past i64.** avsc keeps the float; this port stores an `i64`.
- **Decode error precedence across blocks.** When an early block has a bad record and a later block fails to decompress, avsc's async order decides which error comes first. This port reports the first block in order.
- **Encode error text.** It follows avsc's `invalid <schema>: <value>` shape, but the value is printed with Rust `Debug`. Only a hog transformation that writes a wrongly typed value can reach it. Port it with the transformations.
-- **The HTTP POST of `LogsMetricsEmitter`.** The crate has no HTTP client. `LogsMetricsEmitter` takes an `OtlpTransport` and keeps the body, the two attempts, the 2xx check and the `logs_metrics_rules_export_failed_total{team_id}` counter.
-- **`MetricRulesCache`.** Only its interface is ported, as `MetricRulesSource`.
- **Flattened JSON keys that differ only by a lone surrogate.** A body such as `{"\ud800":1,"\ud801":2}` gives two attribute keys in Node, and avsc writes both as U+FFFD, so the map holds a duplicate key. `AvroMap` keys are Rust strings, so this port keeps one entry. The same holds for such a key next to a sender key `\ufffd`.
- **The OpenTelemetry skip counter** (`recordJsonEnrichmentSkipped`). `normalize` emits only the Prometheus `logs_ingestion_json_enrichment_skipped_total`.
- **`logs_ingestion_pattern_stage_error_total`.** The Rust stamping cannot fail, so it has no catch path.
@@ -254,7 +270,19 @@ A parsed `FilterGroupNode` implements `metric_rules::FilterGroupMatcher` with `f
- **A Kafka header with a null value.** `parseKafkaHeaders` would throw on it in Node; this port reads it as an empty string.
- **The OpenTelemetry usage counters** (`recordLogsReceived` and the others in `ingestion-otel-metrics.ts`). The Prometheus counters are ported.
- **`re2_objects_created_total`.** The tracked-RE2 allocation counter and its LRU cache are not ported; they do not change a result.
-- **The sampling and retention rules caches** (`sampling-rules-cache.ts`, `retention-rules-cache.ts`, `tracing-config-cache.ts`). The compilers take rows.
+
+### Service gaps
+
+- **`first team event ingested`.** `setTeamIngestedEvent` captures a PostHog analytics event to every organization member after the first `ingested_event` write. `PgTeamManager` writes the flag and refreshes the cache, but captures nothing.
+- **Hog transformations in primary mode.** No `LogsTransformer` exists, so primary mode refuses to start when `LOGS_TRANSFORMATIONS_ENABLED_TEAMS` is not empty.
+- **Offsets after a failed background task.** Node's default consumer (`consumer-v1.ts`, `CONSUMER_USE_V2=false`) stores a batch's offsets in a `finally`, so a failed task's offsets can still commit before the process exits. The port follows `consumer-v2.ts`: a failed task stores nothing from it or any later batch.
+- **Drain on revoke.** `consumer-v2.ts` waits for in-flight tasks inside the revoke callback. The port fences their offset stores by the same per-partition epoch, but does not wait, so the new owner can process a revoked partition while an old task still runs. That costs duplicates, not data.
+- **Consumer telemetry.** The node-rdkafka statistics, loop-stall health check and batch histograms are not ported. Liveness is the lifecycle heartbeat on every loop turn.
+- **A full producer queue.** node-rdkafka fails the produce at once; the port waits up to 10 s for queue space first. Both then retry in place and fail the batch, so the wait runs once per retry attempt.
+- **`USAGE_INGESTION_TLS=true`.** The workspace `tonic` has no TLS feature, so the service refuses to start with it.
+- **`LazyLoader` instrumentation.** The OpenTelemetry spans and the hit and miss counters are not ported; `lazy_loader_cache_size` is. Loader retries are not ported, because the logs servers build their loaders without `loaderRetry`.
+- **Rule-cache spans.** The `logs.sampling.*`, `logs.retention.*`, `traces.retention.*` and `logs.patterns.*` span attributes are not ported.
+- **The logs Redis password.** Node never passes `LOGS_REDIS_PASSWORD` to the logs Redis client, and neither does the port.
## Regenerating references
diff --git a/rust/logs-ingestion/README.md b/rust/logs-ingestion/README.md
new file mode 100644
index 000000000000..5a39f1ac10b9
--- /dev/null
+++ b/rust/logs-ingestion/README.md
@@ -0,0 +1,98 @@
+# logs-ingestion
+
+Rust port of the Node logs and traces ingestion consumers (`nodejs/src/logs`).
+The library holds the pipeline, kept 1:1 with Node by the parity corpus (see [PORTING.md](PORTING.md)).
+The `logs-ingestion` binary runs it against Kafka, as the primary consumer or as a shadow beside the Node consumer.
+
+## The consumer service
+
+`PLUGIN_SERVER_MODE` picks the consumer, as it picks the Node server:
+
+| `PLUGIN_SERVER_MODE` | Consumer | Reads | Group (Node default) |
+| --- | --- | --- | --- |
+| `ingestion-logs` | `LogsIngestionConsumer` | `LOGS_INGESTION_CONSUMER_CONSUME_TOPIC` | `ingestion-logs` |
+| `ingestion-traces` | `TracesIngestionConsumer` | `TRACES_INGESTION_CONSUMER_CONSUME_TOPIC` | `ingestion-traces` |
+
+`LOGS_RUST_SHADOW_MODE` is the only setting the Node servers do not have. It defaults to `true` (shadow mode), and only `false` (or `0`, `no`, `off`) selects primary mode. Any other value stops startup.
+The image is `logs-ingestion-rust`, so a Node logs or traces deployment can switch to it with the same env.
+
+Every setting the Node servers read keeps its Node name, default and parse: the `LOGS_*` and `TRACES_*` keys of `logs/config.ts`, the `LOGS_INGESTION_OUTPUT_*` keys, `KAFKA_WARPSTREAM_LOGS_PRODUCER_*` and `KAFKA_WARPSTREAM_INGESTION_PRODUCER_*`, `KAFKA_CONSUMER_*` (rdkafka keys, so `KAFKA_CONSUMER_METADATA_BROKER_LIST` is `metadata.broker.list`), `KAFKA_CLIENT_RACK`, `KAFKA_PREFIX`, `NODE_ENV`, `DATABASE_URL`, `DATABASE_READONLY_URL`, `POSTGRES_CONNECTION_POOL_SIZE`, `REDIS_URL`, `POSTHOG_REDIS_*`, `USAGE_INGESTION_*`, `CONSUMER_BATCH_SIZE`, `CONSUMER_MAX_BACKGROUND_TASKS`, `CONSUMER_BACKGROUND_TASK_TIMEOUT_MS` and `HTTP_SERVER_PORT` (default 6738).
+The defaults that depend on `NODE_ENV` (limiter, metric rules, retention and backfill allowlists, Redis TLS, `_test` topic suffix) follow Node's `isProdEnv`.
+The Kafka consumer follows `consumer-v1.ts` and `consumer-v2.ts`: `auto.offset.reset` defaults to `earliest` (`KAFKA_CONSUMER_AUTO_OFFSET_RESET` overrides it), and with `KAFKA_CONSUMER_GROUP_PROTOCOL=consumer` the classic-protocol keys (`session.timeout.ms`, `heartbeat.interval.ms`, `partition.assignment.strategy`, `group.protocol.type`) are removed, as `stripClassicProtocolConfig` does.
+
+HTTP: `/_health` (liveness), `/_ready` (503 once shutdown starts) and `/metrics` on `HTTP_SERVER_PORT`.
+
+### Batches and offsets
+
+The loop follows the Node consumer (`consumer-v2.ts`):
+
+- It reads up to `CONSUMER_BATCH_SIZE` messages within 500 ms and runs `process_kafka_batch`.
+- Up to `CONSUMER_MAX_BACKGROUND_TASKS` background tasks run at once (default 1).
+- A batch's offsets are stored only after its background task and every earlier one succeeded. librdkafka's auto commit commits them, and shutdown commits once more.
+- A batch that fails, or a background task that fails or runs past `CONSUMER_BACKGROUND_TASK_TIMEOUT_MS` after its dispatch, stops the loop and exits the process with an error. Nothing from that batch or later is stored, so the restart replays it (at least once).
+- A produce to the `logs` topic that fails with a Kafka outage, such as a delivery timeout, a full queue or a topic authorization failure, is retried in place and then fails the batch. Only an error that a replay cannot fix, such as an oversized or invalid record, sends the message to the DLQ.
+- A consume error is tried 3 times, 100 ms apart doubling, as `retryIfRetriable` does, and then exits the process.
+- A revoke bumps the epoch of each revoked partition. A task dispatched before the revoke does not store that partition's offset, as `consumer-v2.ts` fences it.
+- SIGTERM stops fetching, lets the in-flight batch finish, stores and commits its offsets, then exits. A task still running 60 s after SIGTERM is aborted.
+
+### Primary mode
+
+Primary mode behaves as the Node consumer: Node's topics and producers, `ingested_event` writes, usage records to usage-ingestion over gRPC, and metric-rule OTLP pushes to `LOGS_METRICS_RULES_EXPORT_URL` (or the traces one). [PORTING.md](PORTING.md) lists what it does not reproduce.
+Hog transformations are not ported, so primary mode refuses to start when `LOGS_TRANSFORMATIONS_ENABLED_TEAMS` is not empty. The traces consumer reads the same key, as in Node.
+
+## Shadow mode
+
+Shadow mode reads the same input topic as Node and cannot affect production.
+It derives every name it writes from the Node settings:
+
+- The consumer group is the Node group with `-rust-shadow` appended, for example `ingestion-logs-rust-shadow`. No Node or ClickHouse group uses that suffix, so the shadow never takes Node's partitions or moves its offsets.
+- Each output topic is the Node topic with `_rust_shadow` appended, for example `clickhouse_logs_rust_shadow`, `logs_ingestion_dlq_rust_shadow` and `clickhouse_app_metrics2_rust_shadow`.
+- Each shadow topic must exist. The shadow producers set `allow.auto.create.topics=false`, and at startup they read the metadata of all topics and stop if a shadow topic is missing, because brokers below Metadata v4 ignore that setting and create a topic that a metadata request names.
+- Postgres is read through `DATABASE_READONLY_URL` when it is set, and through `DATABASE_URL` otherwise. Shadow mode opens no write pool.
+
+What shadow mode suppresses, and why:
+
+| Effect | In shadow mode | Why |
+| --- | --- | --- |
+| `logs`, `logs_dlq`, `app_metrics` outputs | Written to the shadow topics | ClickHouse reads the production topics. `app_metrics` rows are customer-visible usage. |
+| Usage records (usage-ingestion gRPC) | Dropped, no client is built | They bill. The shadow mints its own record ids, so they would bill twice. |
+| `ingested_event` write | Skipped, no write pool is opened | Postgres is read only for the shadow. |
+| Metric-rule OTLP push | Reported as sent, nothing is sent | It would create customer metrics twice. The emitter stays present, because its presence decides whether a buffer is decoded, and so the DLQ routing. |
+| Hog transformations and their function logs | Not run (not ported) | They write to `log_entries`. |
+| Rate-limit and sampling buckets | Kept under `@posthog-logs-rust-shadow/` in Node's logs or traces Redis | Spending Node's tokens would drop Node's traffic. The shadow sees the same traffic, so its buckets drain at the same rate. |
+| Quota limits | Read from the production PostHog Redis (`POSTHOG_REDIS_*`), as Node does | This is a deliberate read of production state. It is `ZRANGEBYSCORE` only, so it writes nothing, and the shadow needs the same quota decisions as Node for parity. |
+
+The shadow `logs` output carries `source_offset` next to Node's `source_topic` and `source_partition`, so each output names its input message exactly. Primary mode never writes it. The DLQ already carries all three in both implementations.
+
+Run it locally against the dev stack:
+
+```bash
+cd rust
+PLUGIN_SERVER_MODE=ingestion-logs NODE_ENV=dev \
+KAFKA_CONSUMER_METADATA_BROKER_LIST=localhost:9092 \
+KAFKA_WARPSTREAM_LOGS_PRODUCER_METADATA_BROKER_LIST=localhost:9092 \
+KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_BROKER_LIST=localhost:9092 \
+HTTP_SERVER_PORT=6740 \
+cargo run --release -p logs-ingestion --bin logs-ingestion
+```
+
+## Tests
+
+```bash
+cd rust
+cargo test --release -p logs-ingestion
+```
+
+- `tests/parity_corpus.rs` replays the corpus through the pipeline. It needs Redis on 127.0.0.1:6379.
+- `tests/service_postgres.rs` runs the Postgres dependencies against `test_posthog` on localhost:5432. It copies a team, seeds rule rows and deletes them. It skips without Postgres.
+- `tests/service_kafka.rs` runs on librdkafka's mock cluster: the Kafka outputs, commit-after-success with a real consumer group, and shadow mode end to end. For the `passthrough`, `parse-failures` and `codecs` scenarios it produces the inputs, runs one batch, and checks that the shadow topics hold the outputs Node recorded and that no production topic received anything.
+
+## Image
+
+`rust/Dockerfile` builds one binary named by `BIN`:
+
+```bash
+docker build -f rust/Dockerfile --build-arg BIN=logs-ingestion -t logs-ingestion-rust rust
+```
+
+CI builds it as the `logs-ingestion-rust` image (`.github/rust-images.yml`). The build needs `protoc`, which the Dockerfile installs, for `usage-ingestion-proto`.
diff --git a/rust/logs-ingestion/src/avro/decode.rs b/rust/logs-ingestion/src/avro/decode.rs
index ba94781373f2..ab7e5d6edfee 100644
--- a/rust/logs-ingestion/src/avro/decode.rs
+++ b/rust/logs-ingestion/src/avro/decode.rs
@@ -150,6 +150,16 @@ impl<'a> Tap<'a> {
/// Running out of data is not an error here, because the caller sees it through `is_valid`.
pub(crate) fn read_value(tap: &mut Tap<'_>, schema: &Schema) -> Result<AvroValue, AvroError> {
+ let start = tap.pos;
+ let value = read_schema_value(tap, schema)?;
+ // Any value can expand into many empty children, so every value that reads no bytes is charged, at every depth.
+ if tap.pos == start {
+ tap.count_empty_value()?;
+ }
+ Ok(value)
+}
+
+fn read_schema_value(tap: &mut Tap<'_>, schema: &Schema) -> Result<AvroValue, AvroError> {
Ok(match schema {
Schema::Null => AvroValue::Null,
Schema::Boolean => AvroValue::Boolean(tap.byte() != 0),
@@ -227,11 +237,7 @@ pub(crate) fn read_value(tap: &mut Tap<'_>, schema: &Schema) -> Result<AvroValue
// A capacity hint only: a count can claim more items than the data holds.
out.reserve((n as usize).min(tap.remaining()).min(4096));
for _ in 0..n as u64 {
- let start = tap.pos;
out.push(read_value(tap, items)?);
- if tap.pos == start {
- tap.count_empty_value()?;
- }
if !tap.is_valid() {
return Ok(AvroValue::Array(out));
}
@@ -433,11 +439,8 @@ pub fn decode_container(buf: &[u8]) -> Result<Container, AvroError> {
if !block.is_valid() {
return Err(AvroError::Decode("truncated block".to_string()));
}
- if block.pos == start {
- block.count_empty_value()?;
- if remaining < 0 {
- return Err(AvroError::Decode("too many empty records".to_string()));
- }
+ if block.pos == start && remaining < 0 {
+ return Err(AvroError::Decode("too many empty records".to_string()));
}
records.push(value);
}
diff --git a/rust/logs-ingestion/src/avro/schema.rs b/rust/logs-ingestion/src/avro/schema.rs
index 3b8336256c42..a3fb877cc498 100644
--- a/rust/logs-ingestion/src/avro/schema.rs
+++ b/rust/logs-ingestion/src/avro/schema.rs
@@ -128,6 +128,14 @@ fn qualify(name: &str, namespace: Option<&str>) -> String {
}
impl Parser {
+ /// avsc rejects a second definition of a name. Without this check, a redefinition that refers to the previous one can grow the schema exponentially.
+ fn check_unique(&self, name: &str) -> Result<(), AvroError> {
+ if self.named.contains_key(name) {
+ return Err(schema_error(format!("duplicate type name: {name}")));
+ }
+ Ok(())
+ }
+
fn parse(&mut self, value: &JsValue, namespace: Option<&str>) -> Result<Schema, AvroError> {
match value {
JsValue::String(name) => self.parse_name(&name.to_string(), namespace),
@@ -173,6 +181,7 @@ impl Parser {
get_str(object, "name").ok_or_else(|| schema_error("missing record name"))?,
get_str(object, "namespace").or(namespace),
);
+ self.check_unique(&name)?;
let ns = name.rsplit_once('.').map(|(ns, _)| ns.to_string());
let mut fields = Vec::new();
let Some(JsValue::Array(raw_fields)) = object.get("fields") else {
@@ -228,6 +237,7 @@ impl Parser {
get_str(object, "name").ok_or_else(|| schema_error("missing enum name"))?,
get_str(object, "namespace").or(namespace),
);
+ self.check_unique(&name)?;
let schema = Schema::Enum(Arc::new(EnumSchema {
name: name.clone(),
symbols: get_strings(object, "symbols"),
@@ -243,6 +253,7 @@ impl Parser {
get_str(object, "name").ok_or_else(|| schema_error("missing fixed name"))?,
get_str(object, "namespace").or(namespace),
);
+ self.check_unique(&name)?;
let size = match object.get("size") {
Some(JsValue::Number(n)) if *n >= 0.0 => *n as usize,
_ => return Err(schema_error("invalid fixed size")),
diff --git a/rust/logs-ingestion/src/bin/logs-ingestion-bench.rs b/rust/logs-ingestion/src/bin/logs-ingestion-bench.rs
index 8a539d4ca9e3..f6baefa4a417 100644
--- a/rust/logs-ingestion/src/bin/logs-ingestion-bench.rs
+++ b/rust/logs-ingestion/src/bin/logs-ingestion-bench.rs
@@ -49,6 +49,14 @@ fn main() -> anyhow::Result<()> {
let passes: usize = args.get(2).map_or(10, |p| p.parse().unwrap());
let warmup: usize = args.get(3).map_or(2, |p| p.parse().unwrap());
+ // The production recorder and a warn filter match Node's per-call metric and log work. Logs go to stderr, which keeps stdout for the result.
+ let _recorder = serve_metrics::setup_metrics_recorder();
+ tracing_subscriber::fmt()
+ .json()
+ .with_writer(std::io::stderr)
+ .with_env_filter(tracing_subscriber::EnvFilter::new("warn"))
+ .init();
+
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(std::thread::available_parallelism().map_or(4, |n| n.get()))
.enable_all()
diff --git a/rust/logs-ingestion/src/bin/logs-ingestion.rs b/rust/logs-ingestion/src/bin/logs-ingestion.rs
new file mode 100644
index 000000000000..2613ac7f0801
--- /dev/null
+++ b/rust/logs-ingestion/src/bin/logs-ingestion.rs
@@ -0,0 +1,16 @@
+use logs_ingestion::service::{self, config::ServiceConfig, node_env::process_env};
+use tracing_subscriber::EnvFilter;
+
+common_alloc::used!();
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+ tracing_subscriber::fmt()
+ .json()
+ .with_env_filter(
+ EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
+ )
+ .init();
+ let config = ServiceConfig::from_env(&process_env())?;
+ service::run(config).await
+}
diff --git a/rust/logs-ingestion/src/lib.rs b/rust/logs-ingestion/src/lib.rs
index 726e939766c3..1b341a34007d 100644
--- a/rust/logs-ingestion/src/lib.rs
+++ b/rust/logs-ingestion/src/lib.rs
@@ -17,3 +17,4 @@ pub mod rate_limiter;
pub mod record;
pub mod retention;
pub mod sampling;
+pub mod service;
diff --git a/rust/logs-ingestion/src/metric_rules/emitter.rs b/rust/logs-ingestion/src/metric_rules/emitter.rs
index 4c5fab431684..467b4b6d870b 100644
--- a/rust/logs-ingestion/src/metric_rules/emitter.rs
+++ b/rust/logs-ingestion/src/metric_rules/emitter.rs
@@ -1,4 +1,4 @@
-//! The crate has no HTTP client, so the POST is behind `OtlpTransport`. `LogsMetricsEmitter` keeps Node's two attempts and 2xx check.
+//! `LogsMetricsEmitter` keeps Node's two attempts and 2xx check around the `OtlpTransport` POST.
use std::sync::Arc;
diff --git a/rust/logs-ingestion/src/metric_rules/source.rs b/rust/logs-ingestion/src/metric_rules/source.rs
index 1dec20bf91cc..774525fc0c9f 100644
--- a/rust/logs-ingestion/src/metric_rules/source.rs
+++ b/rust/logs-ingestion/src/metric_rules/source.rs
@@ -1,5 +1,3 @@
-//! Only the interface of `metric-rules-cache.ts` is ported, not its Postgres-backed cache.
-
use std::sync::Arc;
use async_trait::async_trait;
diff --git a/rust/logs-ingestion/src/pipeline/deps.rs b/rust/logs-ingestion/src/pipeline/deps.rs
index c8f1b0bce9ce..008b9648dbb0 100644
--- a/rust/logs-ingestion/src/pipeline/deps.rs
+++ b/rust/logs-ingestion/src/pipeline/deps.rs
@@ -100,4 +100,6 @@ pub struct LogsIngestionConsumerDeps {
pub key_root: KeyRoot,
/// Epoch milliseconds. Everything that reads `Date.now()` in Node reads this.
pub clock: Clock,
+ /// Shadow mode only, so a comparator can find the input message. Node never writes `source_offset` there.
+ pub source_offset_header: bool,
}
diff --git a/rust/logs-ingestion/src/pipeline/process.rs b/rust/logs-ingestion/src/pipeline/process.rs
index 6dc4316664ca..66d6fdb9c98c 100644
--- a/rust/logs-ingestion/src/pipeline/process.rs
+++ b/rust/logs-ingestion/src/pipeline/process.rs
@@ -295,6 +295,9 @@ async fn process_log_message(
headers.set("retention-days", number_to_string(retention_days));
headers.set("source_topic", message.message.topic.clone());
headers.set("source_partition", message.message.partition.to_string());
+ if context.deps.source_offset_header {
+ headers.set("source_offset", message.message.offset.to_string());
+ }
if let Some(n) = override_uncompressed {
headers.set("bytes_uncompressed", number_to_string(n));
}
diff --git a/rust/logs-ingestion/src/pipeline/scenario.rs b/rust/logs-ingestion/src/pipeline/scenario.rs
index 110991f16f7a..90bdd57afe32 100644
--- a/rust/logs-ingestion/src/pipeline/scenario.rs
+++ b/rust/logs-ingestion/src/pipeline/scenario.rs
@@ -515,6 +515,15 @@ pub struct ScenarioConsumer {
pub recorded: Arc<Recorded>,
}
+/// The service tests swap the outputs for Kafka ones.
+pub struct ScenarioParts {
+ pub identity: LogsConsumerIdentity,
+ pub config: LogsIngestionConsumerConfig,
+ pub deps: LogsIngestionConsumerDeps,
+ pub outputs: Arc<CapturingOutputs>,
+ pub recorded: Arc<Recorded>,
+}
+
/// `discard` counts produced messages instead of keeping them.
pub fn build_scenario_consumer(
scenario: &JsValue,
@@ -522,6 +531,20 @@ pub fn build_scenario_consumer(
clock: Clock,
discard: bool,
) -> ScenarioConsumer {
+ let parts = scenario_parts(scenario, redis, clock, discard);
+ ScenarioConsumer {
+ consumer: LogsIngestionConsumer::new(parts.identity, parts.config, parts.deps),
+ outputs: parts.outputs,
+ recorded: parts.recorded,
+ }
+}
+
+pub fn scenario_parts(
+ scenario: &JsValue,
+ redis: Arc<dyn TokenBucketRedis>,
+ clock: Clock,
+ discard: bool,
+) -> ScenarioParts {
let recorded = Arc::new(Recorded::default());
let outputs = Arc::new(CapturingOutputs {
faults: arr(get(scenario, "produce_faults"))
@@ -649,27 +672,28 @@ pub fn build_scenario_consumer(
token_bucket_redis: redis,
key_root: KeyRoot::Test,
clock,
+ source_offset_header: false,
};
let config = match get(scenario, "config") {
Some(JsValue::Object(o)) => o.clone(),
_ => JsObject::new(),
};
let traces = s(get(scenario, "consumer")).as_deref() == Some("traces");
- let consumer = if traces {
- LogsIngestionConsumer::new(
+ let (identity, config) = if traces {
+ (
LogsConsumerIdentity::traces(),
LogsIngestionConsumerConfig::from_traces_config(&config),
- deps,
)
} else {
- LogsIngestionConsumer::new(
+ (
LogsConsumerIdentity::logs(),
LogsIngestionConsumerConfig::from_logs_config(&config),
- deps,
)
};
- ScenarioConsumer {
- consumer,
+ ScenarioParts {
+ identity,
+ config,
+ deps,
outputs,
recorded,
}
diff --git a/rust/logs-ingestion/src/rate_limiter/keyed.rs b/rust/logs-ingestion/src/rate_limiter/keyed.rs
index 80fc154aef84..e60018267ca6 100644
--- a/rust/logs-ingestion/src/rate_limiter/keyed.rs
+++ b/rust/logs-ingestion/src/rate_limiter/keyed.rs
@@ -10,11 +10,12 @@ use super::redis::{
ioredis_integer, token_bucket_v2, token_bucket_v3, LuaScript, ScriptCall, TokenBucketRedis,
};
-/// The root of every key: `@posthog-test` under `NODE_ENV=test`, else `@posthog`.
+/// `@posthog-test` under `NODE_ENV=test`, else `@posthog`. `Shadow` keeps the Rust shadow off the Node consumer's tokens.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum KeyRoot {
Production,
Test,
+ Shadow,
}
impl KeyRoot {
@@ -30,6 +31,7 @@ impl KeyRoot {
match self {
KeyRoot::Production => "@posthog",
KeyRoot::Test => "@posthog-test",
+ KeyRoot::Shadow => "@posthog-logs-rust-shadow",
}
}
}
diff --git a/rust/logs-ingestion/src/service/config.rs b/rust/logs-ingestion/src/service/config.rs
new file mode 100644
index 000000000000..6eacbcd02bd7
--- /dev/null
+++ b/rust/logs-ingestion/src/service/config.rs
@@ -0,0 +1,660 @@
+//! Node settings keep their Node name, default and parse. `LOGS_RUST_SHADOW_MODE` is the only setting Node does not have.
+
+use std::time::Duration;
+
+use crate::js::{JsObject, JsValue};
+use crate::pipeline::LogsIngestionConsumerConfig;
+
+use super::node_env::{
+ determine_node_env, get_bool, get_number, get_string, kafka_topic, override_with_env,
+ rdkafka_from_env, Env, NodeEnv,
+};
+
+pub const WARPSTREAM_LOGS_PRODUCER: &str = "WARPSTREAM_LOGS_PRODUCER";
+pub const WARPSTREAM_INGESTION_PRODUCER: &str = "WARPSTREAM_INGESTION_PRODUCER";
+pub const NODE_LOGS_GROUP_ID: &str = "ingestion-logs";
+pub const NODE_TRACES_GROUP_ID: &str = "ingestion-traces";
+pub const SHADOW_TOPIC_SUFFIX: &str = "_rust_shadow";
+pub const SHADOW_GROUP_SUFFIX: &str = "-rust-shadow";
+/// Node's `defaultConfig` has no `KAFKA_CONSUMER_*` key, so `skipKeysInDefaultConfig` skips none.
+pub const NODE_DEFAULT_CONFIG_CONSUMER_KEYS: &[&str] = &[];
+pub const SHADOW_MODE_KEY: &str = "LOGS_RUST_SHADOW_MODE";
+const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(60);
+
+const PRODUCER_KEYS: &[(&str, &str)] = &[
+ ("client.id", "CLIENT_ID"),
+ ("metadata.broker.list", "METADATA_BROKER_LIST"),
+ ("security.protocol", "SECURITY_PROTOCOL"),
+ ("sasl.mechanisms", "SASL_MECHANISMS"),
+ ("sasl.username", "SASL_USERNAME"),
+ ("sasl.password", "SASL_PASSWORD"),
+ ("compression.codec", "COMPRESSION_CODEC"),
+ ("linger.ms", "LINGER_MS"),
+ ("batch.size", "BATCH_SIZE"),
+ (
+ "queue.buffering.max.messages",
+ "QUEUE_BUFFERING_MAX_MESSAGES",
+ ),
+ ("queue.buffering.max.kbytes", "QUEUE_BUFFERING_MAX_KBYTES"),
+ (
+ "enable.ssl.certificate.verification",
+ "ENABLE_SSL_CERTIFICATE_VERIFICATION",
+ ),
+ ("enable.idempotence", "ENABLE_IDEMPOTENCE"),
+ ("message.max.bytes", "MESSAGE_MAX_BYTES"),
+ ("batch.num.messages", "BATCH_NUM_MESSAGES"),
+ (
+ "sticky.partitioning.linger.ms",
+ "STICKY_PARTITIONING_LINGER_MS",
+ ),
+ (
+ "topic.metadata.refresh.interval.ms",
+ "TOPIC_METADATA_REFRESH_INTERVAL_MS",
+ ),
+ ("metadata.max.age.ms", "METADATA_MAX_AGE_MS"),
+ ("message.send.max.retries", "RETRIES"),
+ (
+ "max.in.flight.requests.per.connection",
+ "MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION",
+ ),
+];
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ConsumerKind {
+ Logs,
+ Traces,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RunMode {
+ Primary,
+ Shadow,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct OutputRoute {
+ pub output: &'static str,
+ pub topic: String,
+ pub producer: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct RedisTarget {
+ pub url_or_host: String,
+ pub port: Option<u16>,
+ pub password: Option<String>,
+ pub tls: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct UsageConfig {
+ pub addr: String,
+ pub tls: bool,
+ pub timeout: Duration,
+ pub max_batch_size: usize,
+ pub report_teams: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct ConsumerLoopConfig {
+ pub batch_size: usize,
+ pub batch_timeout: Duration,
+ pub max_background_tasks: usize,
+ pub background_task_timeout: Duration,
+}
+
+#[derive(Debug, Clone)]
+pub struct ServiceConfig {
+ pub kind: ConsumerKind,
+ pub mode: RunMode,
+ pub node_env: NodeEnv,
+ pub node: JsObject,
+ pub consumer: LogsIngestionConsumerConfig,
+ pub group_id: String,
+ pub input_topic: String,
+ pub routes: Vec<OutputRoute>,
+ pub metrics_export_url: String,
+ pub kafka_consumer: Vec<(String, String)>,
+ pub producers: Vec<(String, Vec<(String, String)>)>,
+ pub kafka_client_rack: Option<String>,
+ pub consumer_loop: ConsumerLoopConfig,
+ pub database_reader_url: String,
+ /// `None` in shadow mode, which opens no write pool.
+ pub database_writer_url: Option<String>,
+ pub pool_size: u32,
+ pub posthog_redis: RedisTarget,
+ pub logs_redis: RedisTarget,
+ pub usage: UsageConfig,
+ pub http_port: u16,
+ pub shutdown_timeout: Duration,
+}
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum ConfigError {
+ #[error("PLUGIN_SERVER_MODE must be ingestion-logs or ingestion-traces, got {0:?}")]
+ ConsumerKind(String),
+ #[error("{SHADOW_MODE_KEY} must be true or false, got {0:?}")]
+ ShadowMode(String),
+ #[error(
+ "{0} is not empty, but hog transformations are not ported, so primary mode refuses it"
+ )]
+ TransformationsNotPorted(&'static str),
+ #[error("{0}")]
+ Invalid(String),
+}
+
+fn s(v: &str) -> JsValue {
+ JsValue::string(v)
+}
+
+fn n(v: f64) -> JsValue {
+ JsValue::Number(v)
+}
+
+fn b(v: bool) -> JsValue {
+ JsValue::Bool(v)
+}
+
+fn node_defaults(env: &Env) -> Vec<(String, JsValue)> {
+ let node_env = determine_node_env(env);
+ let prod = node_env == NodeEnv::Production;
+ let off_in_prod = |_: ()| s(if prod { "" } else { "*" });
+ let topic = |name: &str| s(&kafka_topic(env, name));
+ let d: Vec<(&str, JsValue)> = vec![
+ ("LOGS_INGESTION_CONSUMER_GROUP_ID", s(NODE_LOGS_GROUP_ID)),
+ (
+ "LOGS_INGESTION_CONSUMER_CONSUME_TOPIC",
+ topic("logs_ingestion"),
+ ),
+ (
+ "LOGS_INGESTION_CONSUMER_OVERFLOW_TOPIC",
+ topic("logs_ingestion_overflow"),
+ ),
+ (
+ "LOGS_INGESTION_CONSUMER_DLQ_TOPIC",
+ topic("logs_ingestion_dlq"),
+ ),
+ (
+ "LOGS_INGESTION_CONSUMER_CLICKHOUSE_TOPIC",
+ topic("clickhouse_logs"),
+ ),
+ ("LOGS_REDIS_HOST", s("127.0.0.1")),
+ ("LOGS_REDIS_PORT", n(6379.0)),
+ ("LOGS_REDIS_PASSWORD", s("")),
+ ("LOGS_REDIS_TLS", b(prod)),
+ ("LOGS_LIMITER_ENABLED_TEAMS", off_in_prod(())),
+ ("LOGS_LIMITER_DISABLED_FOR_TEAMS", s("")),
+ ("LOGS_LIMITER_BUCKET_SIZE_KB", n(10000.0)),
+ ("LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND", n(1000.0)),
+ ("LOGS_LIMITER_TTL_SECONDS", n(86400.0)),
+ ("LOGS_LIMITER_TEAM_BUCKET_SIZE_KB", s("")),
+ ("LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND", s("")),
+ ("LOGS_SAMPLING_ENABLED_TEAMS", s("*")),
+ ("LOGS_SAMPLING_KILLSWITCH", b(false)),
+ ("LOGS_METRICS_RULES_ENABLED_TEAMS", off_in_prod(())),
+ ("LOGS_METRICS_RULES_KILLSWITCH", b(false)),
+ ("LOGS_METRICS_RULES_EXPORT_URL", s("")),
+ ("LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS", s("")),
+ ("LOGS_RETENTION_ENABLED_TEAMS", off_in_prod(())),
+ ("LOGS_BACKFILL_ENABLED_TEAMS", off_in_prod(())),
+ ("LOGS_RETENTION_KILLSWITCH", b(false)),
+ ("LOGS_INTERNAL_INGESTION_ENABLED_TEAMS", s("")),
+ ("LOGS_INTERNAL_INGESTION_SIGNING_KEYS", s("")),
+ ("LOGS_PATTERN_MASKING_ENABLED_TEAMS", s("")),
+ ("LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS", s("")),
+ ("LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS", s("")),
+ ("LOGS_BILLING_PRORATE_ENABLED", b(false)),
+ ("LOGS_TRANSFORMATIONS_ENABLED_TEAMS", s("")),
+ ("LOGS_TRANSFORMATIONS_KILLSWITCH", b(false)),
+ ("REDIS_URL", s("redis://127.0.0.1")),
+ (
+ "TRACES_INGESTION_CONSUMER_GROUP_ID",
+ s(NODE_TRACES_GROUP_ID),
+ ),
+ (
+ "TRACES_INGESTION_CONSUMER_CONSUME_TOPIC",
+ topic("ingestion-traces"),
+ ),
+ (
+ "TRACES_INGESTION_CONSUMER_OVERFLOW_TOPIC",
+ topic("ingestion-traces-overflow"),
+ ),
+ (
+ "TRACES_INGESTION_CONSUMER_DLQ_TOPIC",
+ topic("ingestion-traces-dlq"),
+ ),
+ (
+ "TRACES_INGESTION_CONSUMER_CLICKHOUSE_TOPIC",
+ topic("clickhouse_traces"),
+ ),
+ ("TRACES_REDIS_HOST", s("127.0.0.1")),
+ ("TRACES_REDIS_PORT", n(6379.0)),
+ ("TRACES_REDIS_PASSWORD", s("")),
+ ("TRACES_REDIS_TLS", b(prod)),
+ ("TRACES_LIMITER_ENABLED_TEAMS", off_in_prod(())),
+ ("TRACES_LIMITER_DISABLED_FOR_TEAMS", s("")),
+ ("TRACES_LIMITER_BUCKET_SIZE_KB", n(10000.0)),
+ ("TRACES_LIMITER_REFILL_RATE_KB_PER_SECOND", n(1000.0)),
+ ("TRACES_LIMITER_TTL_SECONDS", n(86400.0)),
+ ("TRACES_LIMITER_TEAM_BUCKET_SIZE_KB", s("")),
+ ("TRACES_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND", s("")),
+ ("TRACES_METRICS_RULES_ENABLED_TEAMS", off_in_prod(())),
+ ("TRACES_METRICS_RULES_KILLSWITCH", b(false)),
+ ("TRACES_METRICS_RULES_EXPORT_URL", s("")),
+ ("TRACES_DEFAULT_METRIC_RULES_ENABLED_TEAMS", s("")),
+ ("TRACES_RETENTION_ENABLED_TEAMS", off_in_prod(())),
+ ("TRACES_RETENTION_KILLSWITCH", b(false)),
+ (
+ "LOGS_INGESTION_OUTPUT_APP_METRICS_TOPIC",
+ topic("clickhouse_app_metrics2"),
+ ),
+ (
+ "LOGS_INGESTION_OUTPUT_APP_METRICS_PRODUCER",
+ s(WARPSTREAM_INGESTION_PRODUCER),
+ ),
+ (
+ "LOGS_INGESTION_OUTPUT_LOG_ENTRIES_TOPIC",
+ topic("log_entries"),
+ ),
+ (
+ "LOGS_INGESTION_OUTPUT_LOG_ENTRIES_PRODUCER",
+ s(WARPSTREAM_INGESTION_PRODUCER),
+ ),
+ (
+ "LOGS_INGESTION_OUTPUT_LOGS_PRODUCER",
+ s(WARPSTREAM_LOGS_PRODUCER),
+ ),
+ (
+ "LOGS_INGESTION_OUTPUT_DLQ_PRODUCER",
+ s(WARPSTREAM_LOGS_PRODUCER),
+ ),
+ (
+ "DATABASE_URL",
+ s(match node_env {
+ NodeEnv::Test => "postgres://posthog:posthog@localhost:5432/test_posthog",
+ NodeEnv::Development => "postgres://posthog:posthog@localhost:5432/posthog",
+ NodeEnv::Production => "",
+ }),
+ ),
+ ("DATABASE_READONLY_URL", s("")),
+ ("POSTGRES_CONNECTION_POOL_SIZE", n(10.0)),
+ ("POSTHOG_REDIS_HOST", s("")),
+ ("POSTHOG_REDIS_PORT", n(6379.0)),
+ ("POSTHOG_REDIS_PASSWORD", s("")),
+ (
+ "USAGE_INGESTION_ADDR",
+ s(if node_env == NodeEnv::Development {
+ "localhost:7143"
+ } else {
+ ""
+ }),
+ ),
+ ("USAGE_INGESTION_TLS", b(false)),
+ ("USAGE_INGESTION_TIMEOUT_MS", n(5000.0)),
+ ("USAGE_INGESTION_MAX_BATCH_SIZE", n(500.0)),
+ ("USAGE_INGESTION_REPORT_TEAMS", s("")),
+ ("CONSUMER_BATCH_SIZE", n(500.0)),
+ ("CONSUMER_MAX_BACKGROUND_TASKS", n(1.0)),
+ ("CONSUMER_BACKGROUND_TASK_TIMEOUT_MS", n(60000.0)),
+ ("HTTP_SERVER_PORT", n(6738.0)),
+ ("KAFKA_CLIENT_RACK", s("")),
+ ];
+ let mut d: Vec<(String, JsValue)> = d.into_iter().map(|(k, v)| (k.to_string(), v)).collect();
+ for prefix in [
+ "KAFKA_WARPSTREAM_LOGS_PRODUCER_",
+ "KAFKA_WARPSTREAM_INGESTION_PRODUCER_",
+ ] {
+ for (_, suffix) in PRODUCER_KEYS {
+ d.push((format!("{prefix}{suffix}"), s("")));
+ }
+ }
+ d
+}
+
+fn positive(config: &JsObject, key: &str) -> Result<f64, ConfigError> {
+ let v = get_number(config, key);
+ if v.is_finite() && v > 0.0 {
+ Ok(v)
+ } else {
+ Err(ConfigError::Invalid(format!(
+ "{key} must be a positive number"
+ )))
+ }
+}
+
+/// Unset or empty means shadow, so a copied Node deployment cannot start as primary by accident.
+pub fn run_mode(env: &Env) -> Result<RunMode, ConfigError> {
+ let raw = env.get(SHADOW_MODE_KEY).map(|v| v.trim()).unwrap_or("");
+ match raw.to_lowercase().as_str() {
+ "" | "y" | "yes" | "t" | "true" | "on" | "1" => Ok(RunMode::Shadow),
+ "n" | "no" | "f" | "false" | "off" | "0" => Ok(RunMode::Primary),
+ _ => Err(ConfigError::ShadowMode(raw.to_string())),
+ }
+}
+
+fn shadow_route(route: OutputRoute) -> OutputRoute {
+ OutputRoute {
+ topic: format!("{}{SHADOW_TOPIC_SUFFIX}", route.topic),
+ ..route
+ }
+}
+
+impl ServiceConfig {
+ pub fn from_env(env: &Env) -> Result<Self, ConfigError> {
+ let kind = match env.get("PLUGIN_SERVER_MODE").map(String::as_str) {
+ Some("ingestion-logs") => ConsumerKind::Logs,
+ Some("ingestion-traces") => ConsumerKind::Traces,
+ other => return Err(ConfigError::ConsumerKind(other.unwrap_or("").to_string())),
+ };
+ let mode = run_mode(env)?;
+ let node_env = determine_node_env(env);
+ let defaults = node_defaults(env);
+ let defaults: Vec<(&str, JsValue)> = defaults
+ .iter()
+ .map(|(k, v)| (k.as_str(), v.clone()))
+ .collect();
+ let node = override_with_env(&defaults, env);
+ let p = match kind {
+ ConsumerKind::Logs => "LOGS",
+ ConsumerKind::Traces => "TRACES",
+ };
+ let consumer = match kind {
+ ConsumerKind::Logs => LogsIngestionConsumerConfig::from_logs_config(&node),
+ ConsumerKind::Traces => LogsIngestionConsumerConfig::from_traces_config(&node),
+ };
+ // The traces consumer extends the logs one in Node and reads this same key.
+ if mode == RunMode::Primary && !consumer.transformations_enabled_teams.trim().is_empty() {
+ return Err(ConfigError::TransformationsNotPorted(
+ "LOGS_TRANSFORMATIONS_ENABLED_TEAMS",
+ ));
+ }
+ let input_topic = get_string(&node, &format!("{p}_INGESTION_CONSUMER_CONSUME_TOPIC"));
+ let production_routes = vec![
+ OutputRoute {
+ output: crate::pipeline::LOGS_OUTPUT,
+ topic: get_string(&node, &format!("{p}_INGESTION_CONSUMER_CLICKHOUSE_TOPIC")),
+ producer: get_string(&node, "LOGS_INGESTION_OUTPUT_LOGS_PRODUCER"),
+ },
+ OutputRoute {
+ output: crate::pipeline::LOGS_DLQ_OUTPUT,
+ topic: get_string(&node, &format!("{p}_INGESTION_CONSUMER_DLQ_TOPIC")),
+ producer: get_string(&node, "LOGS_INGESTION_OUTPUT_DLQ_PRODUCER"),
+ },
+ OutputRoute {
+ output: crate::pipeline::app_metrics::APP_METRICS_OUTPUT,
+ topic: get_string(&node, "LOGS_INGESTION_OUTPUT_APP_METRICS_TOPIC"),
+ producer: get_string(&node, "LOGS_INGESTION_OUTPUT_APP_METRICS_PRODUCER"),
+ },
+ ];
+ let node_group = get_string(&node, &format!("{p}_INGESTION_CONSUMER_GROUP_ID"));
+ let (group_id, routes) = match mode {
+ RunMode::Primary => (node_group, production_routes),
+ RunMode::Shadow => (
+ format!("{node_group}{SHADOW_GROUP_SUFFIX}"),
+ production_routes.into_iter().map(shadow_route).collect(),
+ ),
+ };
+ for route in &routes {
+ if route.producer != WARPSTREAM_LOGS_PRODUCER
+ && route.producer != WARPSTREAM_INGESTION_PRODUCER
+ {
+ return Err(ConfigError::Invalid(format!(
+ "unknown producer {:?} for output {}",
+ route.producer, route.output
+ )));
+ }
+ }
+ let mut producers = Vec::new();
+ for (name, prefix) in [
+ (WARPSTREAM_LOGS_PRODUCER, "KAFKA_WARPSTREAM_LOGS_PRODUCER_"),
+ (
+ WARPSTREAM_INGESTION_PRODUCER,
+ "KAFKA_WARPSTREAM_INGESTION_PRODUCER_",
+ ),
+ ] {
+ let values = PRODUCER_KEYS
+ .iter()
+ .filter_map(|(rdkafka, suffix)| {
+ let v = get_string(&node, &format!("{prefix}{suffix}"));
+ (!v.is_empty()).then(|| (rdkafka.to_string(), v))
+ })
+ .collect();
+ producers.push((name.to_string(), values));
+ }
+ let redis_target =
+ |host_key: &str, port_key: &str, tls_key: Option<&str>, password_key: Option<&str>| {
+ let host = get_string(&node, host_key);
+ if host.is_empty() {
+ RedisTarget {
+ url_or_host: get_string(&node, "REDIS_URL"),
+ port: None,
+ password: None,
+ tls: false,
+ }
+ } else {
+ RedisTarget {
+ url_or_host: host,
+ port: Some(get_number(&node, port_key) as u16),
+ password: password_key
+ .map(|k| get_string(&node, k))
+ .filter(|p| !p.is_empty()),
+ tls: tls_key.is_some_and(|k| get_bool(&node, k)),
+ }
+ }
+ };
+ // Node passes no password for the logs Redis, only host, port and TLS. The shadow keeps its buckets under its own key root there.
+ let logs_redis = redis_target(
+ &format!("{p}_REDIS_HOST"),
+ &format!("{p}_REDIS_PORT"),
+ Some(&format!("{p}_REDIS_TLS")),
+ None,
+ );
+ let posthog_redis = redis_target(
+ "POSTHOG_REDIS_HOST",
+ "POSTHOG_REDIS_PORT",
+ None,
+ Some("POSTHOG_REDIS_PASSWORD"),
+ );
+ let database_url = get_string(&node, "DATABASE_URL").trim().to_string();
+ let readonly_url = get_string(&node, "DATABASE_READONLY_URL")
+ .trim()
+ .to_string();
+ if database_url.is_empty() && readonly_url.is_empty() {
+ return Err(ConfigError::Invalid("DATABASE_URL is required".to_string()));
+ }
+ let database_reader_url = if readonly_url.is_empty() {
+ database_url.clone()
+ } else {
+ readonly_url
+ };
+ let database_writer_url = match mode {
+ RunMode::Primary if database_url.is_empty() => {
+ return Err(ConfigError::Invalid("DATABASE_URL is required".to_string()))
+ }
+ RunMode::Primary => Some(database_url),
+ RunMode::Shadow => None,
+ };
+ let usage = UsageConfig {
+ addr: get_string(&node, "USAGE_INGESTION_ADDR"),
+ tls: get_bool(&node, "USAGE_INGESTION_TLS"),
+ timeout: Duration::from_millis(positive(&node, "USAGE_INGESTION_TIMEOUT_MS")? as u64),
+ max_batch_size: positive(&node, "USAGE_INGESTION_MAX_BATCH_SIZE")? as usize,
+ report_teams: get_string(&node, "USAGE_INGESTION_REPORT_TEAMS"),
+ };
+ let rack = get_string(&node, "KAFKA_CLIENT_RACK");
+ Ok(Self {
+ kind,
+ mode,
+ node_env,
+ metrics_export_url: get_string(&node, &format!("{p}_METRICS_RULES_EXPORT_URL")),
+ kafka_consumer: rdkafka_from_env(
+ env,
+ "KAFKA_CONSUMER_",
+ NODE_DEFAULT_CONFIG_CONSUMER_KEYS,
+ ),
+ consumer_loop: ConsumerLoopConfig {
+ batch_size: positive(&node, "CONSUMER_BATCH_SIZE")? as usize,
+ batch_timeout: Duration::from_millis(500),
+ max_background_tasks: positive(&node, "CONSUMER_MAX_BACKGROUND_TASKS")? as usize,
+ background_task_timeout: Duration::from_millis(positive(
+ &node,
+ "CONSUMER_BACKGROUND_TASK_TIMEOUT_MS",
+ )? as u64),
+ },
+ database_reader_url,
+ database_writer_url,
+ pool_size: positive(&node, "POSTGRES_CONNECTION_POOL_SIZE")? as u32,
+ usage,
+ http_port: positive(&node, "HTTP_SERVER_PORT")? as u16,
+ kafka_client_rack: (!rack.is_empty()).then_some(rack),
+ shutdown_timeout: SHUTDOWN_TIMEOUT,
+ group_id,
+ input_topic,
+ routes,
+ producers,
+ logs_redis,
+ posthog_redis,
+ consumer,
+ node,
+ })
+ }
+
+ pub fn route(&self, output: &str) -> Option<&OutputRoute> {
+ self.routes.iter().find(|r| r.output == output)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn env(pairs: &[(&str, &str)]) -> Env {
+ pairs
+ .iter()
+ .map(|(k, v)| (k.to_string(), v.to_string()))
+ .collect()
+ }
+
+ fn node_env(extra: &[(&str, &str)]) -> Env {
+ let mut e = env(&[
+ ("PLUGIN_SERVER_MODE", "ingestion-logs"),
+ ("DATABASE_URL", "postgres://x"),
+ ("LOGS_REDIS_HOST", "logs-redis"),
+ ]);
+ e.extend(extra.iter().map(|(k, v)| (k.to_string(), v.to_string())));
+ e
+ }
+
+ fn topics(c: &ServiceConfig) -> Vec<(&str, &str, &str)> {
+ c.routes
+ .iter()
+ .map(|r| (r.output, r.topic.as_str(), r.producer.as_str()))
+ .collect()
+ }
+
+ #[test]
+ fn a_node_env_runs_as_a_shadow_with_suffixed_group_and_outputs() {
+ let c = ServiceConfig::from_env(&node_env(&[
+ ("LOGS_INGESTION_CONSUMER_GROUP_ID", "custom-group"),
+ ("LOGS_INGESTION_OUTPUT_APP_METRICS_TOPIC", "custom_metrics"),
+ ]))
+ .unwrap();
+ assert_eq!(c.mode, RunMode::Shadow);
+ assert_eq!(c.group_id, "custom-group-rust-shadow");
+ assert_eq!(c.input_topic, "logs_ingestion");
+ assert_eq!(
+ topics(&c),
+ vec![
+ (
+ "logs",
+ "clickhouse_logs_rust_shadow",
+ WARPSTREAM_LOGS_PRODUCER
+ ),
+ (
+ "logs_dlq",
+ "logs_ingestion_dlq_rust_shadow",
+ WARPSTREAM_LOGS_PRODUCER
+ ),
+ (
+ "app_metrics",
+ "custom_metrics_rust_shadow",
+ WARPSTREAM_INGESTION_PRODUCER
+ ),
+ ]
+ );
+ assert_eq!(c.logs_redis.url_or_host, "logs-redis");
+ assert_eq!(c.database_reader_url, "postgres://x");
+ assert_eq!(c.database_writer_url, None);
+
+ let replica = ServiceConfig::from_env(&node_env(&[(
+ "DATABASE_READONLY_URL",
+ "postgres://replica",
+ )]))
+ .unwrap();
+ assert_eq!(replica.database_reader_url, "postgres://replica");
+ }
+
+ #[test]
+ fn only_an_explicit_false_selects_primary() {
+ for (value, expected) in [
+ ("", Ok(RunMode::Shadow)),
+ ("true", Ok(RunMode::Shadow)),
+ (" TRUE ", Ok(RunMode::Shadow)),
+ ("false", Ok(RunMode::Primary)),
+ ("0", Ok(RunMode::Primary)),
+ ("fals", Err(ConfigError::ShadowMode("fals".into()))),
+ ] {
+ assert_eq!(
+ run_mode(&env(&[(SHADOW_MODE_KEY, value)])),
+ expected,
+ "{value:?}"
+ );
+ }
+ assert_eq!(run_mode(&env(&[])), Ok(RunMode::Shadow));
+ }
+
+ #[test]
+ fn primary_config_matches_node_defaults() {
+ let c = ServiceConfig::from_env(&env(&[
+ ("PLUGIN_SERVER_MODE", "ingestion-traces"),
+ (SHADOW_MODE_KEY, "false"),
+ ("DATABASE_URL", "postgres://x"),
+ ("TRACES_LIMITER_BUCKET_SIZE_KB", "2.5"),
+ ]))
+ .unwrap();
+ assert_eq!(c.group_id, NODE_TRACES_GROUP_ID);
+ assert_eq!(c.input_topic, "ingestion-traces");
+ assert_eq!(c.route("logs").unwrap().topic, "clickhouse_traces");
+ assert_eq!(c.route("logs_dlq").unwrap().topic, "ingestion-traces-dlq");
+ assert_eq!(
+ c.route("app_metrics").unwrap().topic,
+ "clickhouse_app_metrics2"
+ );
+ assert_eq!(c.consumer.limiter_bucket_size_kb, 2.5);
+ assert_eq!(c.consumer.limiter_enabled_teams, "");
+ assert!(c.logs_redis.tls);
+ assert_eq!(c.consumer_loop.max_background_tasks, 1);
+ assert_eq!(c.database_writer_url.as_deref(), Some("postgres://x"));
+
+ let transformations = ServiceConfig::from_env(&env(&[
+ ("PLUGIN_SERVER_MODE", "ingestion-logs"),
+ (SHADOW_MODE_KEY, "false"),
+ ("DATABASE_URL", "postgres://x"),
+ ("LOGS_TRANSFORMATIONS_ENABLED_TEAMS", "2"),
+ ]))
+ .unwrap_err();
+ assert!(transformations
+ .to_string()
+ .contains("LOGS_TRANSFORMATIONS_ENABLED_TEAMS"));
+ let no_writer = ServiceConfig::from_env(&env(&[
+ ("PLUGIN_SERVER_MODE", "ingestion-logs"),
+ (SHADOW_MODE_KEY, "false"),
+ ("DATABASE_READONLY_URL", "postgres://replica"),
+ ]))
+ .unwrap_err();
+ assert_eq!(
+ no_writer,
+ ConfigError::Invalid("DATABASE_URL is required".into())
+ );
+ }
+}
diff --git a/rust/logs-ingestion/src/service/consumer_loop.rs b/rust/logs-ingestion/src/service/consumer_loop.rs
new file mode 100644
index 000000000000..239c82dea505
--- /dev/null
+++ b/rust/logs-ingestion/src/service/consumer_loop.rs
@@ -0,0 +1,383 @@
+//! Offsets are stored once a batch's task and every earlier one succeeded. A task's timeout runs from its dispatch, as `trackTask` does.
+
+use std::collections::{BTreeMap, VecDeque};
+use std::time::Duration;
+
+use async_trait::async_trait;
+use futures::future::BoxFuture;
+use ingestion_framework::KafkaMessage;
+use tokio::task::JoinHandle;
+
+use super::config::ConsumerLoopConfig;
+
+pub type BackgroundTask = BoxFuture<'static, anyhow::Result<()>>;
+
+pub type OffsetsToStore = Vec<(String, i32, i64)>;
+
+#[async_trait]
+pub trait BatchSource: Send {
+ async fn next_batch(
+ &mut self,
+ max: usize,
+ timeout: Duration,
+ ) -> anyhow::Result<Vec<KafkaMessage>>;
+ fn epochs(&self, offsets: &OffsetsToStore) -> Vec<u64> {
+ vec![0; offsets.len()]
+ }
+ fn store_offsets(&mut self, offsets: &OffsetsToStore, epochs: &[u64]) -> anyhow::Result<()>;
+ fn commit(&mut self) -> anyhow::Result<()>;
+}
+
+#[async_trait]
+pub trait BatchProcessor: Send + Sync {
+ async fn process(&self, messages: Vec<KafkaMessage>) -> anyhow::Result<Option<BackgroundTask>>;
+}
+
+#[async_trait]
+impl BatchProcessor for crate::pipeline::LogsIngestionConsumer {
+ async fn process(&self, messages: Vec<KafkaMessage>) -> anyhow::Result<Option<BackgroundTask>> {
+ Ok(self.process_kafka_batch(messages).await?.background_task)
+ }
+}
+
+pub fn offsets_to_store(messages: &[KafkaMessage]) -> OffsetsToStore {
+ let mut highest: BTreeMap<(String, i32), i64> = BTreeMap::new();
+ for m in messages {
+ let entry = highest
+ .entry((m.topic.clone(), m.partition))
+ .or_insert(m.offset);
+ *entry = (*entry).max(m.offset);
+ }
+ highest
+ .into_iter()
+ .map(|((topic, partition), offset)| (topic, partition, offset + 1))
+ .collect()
+}
+
+struct InFlight {
+ task: JoinHandle<anyhow::Result<()>>,
+ offsets: OffsetsToStore,
+ epochs: Vec<u64>,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct LoopStats {
+ pub batches: u64,
+ pub messages: u64,
+}
+
+async fn settle(entry: InFlight, source: &mut dyn BatchSource) -> anyhow::Result<()> {
+ let result = match entry.task.await {
+ Err(join_error) => Err(anyhow::anyhow!("background task panicked: {join_error}")),
+ Ok(result) => result,
+ };
+ if let Err(error) = result {
+ tracing::error!(error = %error, "kafka_consumer_background_task_failed");
+ metrics::counter!("logs_ingestion_consumer_background_task_failed_total").increment(1);
+ return Err(error);
+ }
+ source.store_offsets(&entry.offsets, &entry.epochs)?;
+ Ok(())
+}
+
+/// Either way it commits what was stored.
+pub async fn run_consumer_loop(
+ source: &mut dyn BatchSource,
+ processor: &dyn BatchProcessor,
+ config: &ConsumerLoopConfig,
+ shutdown: impl std::future::Future<Output = ()> + Send,
+ on_tick: &(dyn Fn() + Send + Sync),
+) -> anyhow::Result<LoopStats> {
+ tokio::pin!(shutdown);
+ let mut in_flight: VecDeque<InFlight> = VecDeque::new();
+ let mut stats = LoopStats::default();
+ let mut outcome: anyhow::Result<()> = Ok(());
+ loop {
+ on_tick();
+ while in_flight.front().is_some_and(|f| f.task.is_finished()) {
+ let entry = in_flight.pop_front().unwrap();
+ if let Err(e) = settle(entry, source).await {
+ outcome = Err(e);
+ break;
+ }
+ }
+ if outcome.is_err() {
+ break;
+ }
+ let batch = tokio::select! {
+ biased;
+ _ = &mut shutdown => break,
+ batch = source.next_batch(config.batch_size, config.batch_timeout) => batch,
+ };
+ let messages = match batch {
+ Ok(m) => m,
+ Err(e) => {
+ outcome = Err(e);
+ break;
+ }
+ };
+ if messages.is_empty() {
+ continue;
+ }
+ stats.batches += 1;
+ stats.messages += messages.len() as u64;
+ let offsets = offsets_to_store(&messages);
+ let epochs = source.epochs(&offsets);
+ let task = match processor.process(messages).await {
+ Ok(task) => task,
+ Err(e) => {
+ outcome = Err(e);
+ break;
+ }
+ };
+ let timeout = config.background_task_timeout;
+ let task = tokio::spawn(async move {
+ let Some(task) = task else { return Ok(()) };
+ match tokio::time::timeout(timeout, task).await {
+ Ok(result) => result,
+ Err(_) => Err(anyhow::anyhow!(
+ "background_task_timeout_after_{}ms",
+ timeout.as_millis()
+ )),
+ }
+ });
+ in_flight.push_back(InFlight {
+ task,
+ offsets,
+ epochs,
+ });
+ if in_flight.len() >= config.max_background_tasks {
+ let entry = in_flight.pop_front().unwrap();
+ if let Err(e) = settle(entry, source).await {
+ outcome = Err(e);
+ break;
+ }
+ }
+ }
+ if outcome.is_ok() {
+ while let Some(entry) = in_flight.pop_front() {
+ if let Err(e) = settle(entry, source).await {
+ outcome = Err(e);
+ break;
+ }
+ }
+ } else {
+ for entry in in_flight {
+ entry.task.abort();
+ }
+ }
+ if let Err(error) = source.commit() {
+ tracing::error!(error = %error, "final offset commit failed");
+ }
+ outcome.map(|()| stats)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use futures::FutureExt;
+ use std::sync::{Arc, Mutex};
+
+ #[derive(Default)]
+ struct FakeSource {
+ batches: VecDeque<Vec<KafkaMessage>>,
+ stored: Arc<Mutex<Vec<OffsetsToStore>>>,
+ commits: Arc<Mutex<u32>>,
+ }
+
+ #[async_trait]
+ impl BatchSource for FakeSource {
+ async fn next_batch(
+ &mut self,
+ _max: usize,
+ timeout: Duration,
+ ) -> anyhow::Result<Vec<KafkaMessage>> {
+ match self.batches.pop_front() {
+ Some(b) => Ok(b),
+ None => {
+ tokio::time::sleep(timeout).await;
+ Ok(vec![])
+ }
+ }
+ }
+ fn store_offsets(
+ &mut self,
+ offsets: &OffsetsToStore,
+ _epochs: &[u64],
+ ) -> anyhow::Result<()> {
+ self.stored.lock().unwrap().push(offsets.clone());
+ Ok(())
+ }
+ fn commit(&mut self) -> anyhow::Result<()> {
+ *self.commits.lock().unwrap() += 1;
+ Ok(())
+ }
+ }
+
+ #[derive(Default)]
+ struct FakeProcessor {
+ fail_task_at: Vec<i64>,
+ fail_batch_at: Vec<i64>,
+ hang_task_at: Vec<i64>,
+ }
+
+ #[async_trait]
+ impl BatchProcessor for FakeProcessor {
+ async fn process(
+ &self,
+ messages: Vec<KafkaMessage>,
+ ) -> anyhow::Result<Option<BackgroundTask>> {
+ let first = messages[0].offset;
+ if self.fail_batch_at.contains(&first) {
+ anyhow::bail!("batch failed");
+ }
+ let fail = self.fail_task_at.contains(&first);
+ let hang = self.hang_task_at.contains(&first);
+ Ok(Some(
+ async move {
+ if hang {
+ futures::future::pending::<()>().await;
+ }
+ tokio::time::sleep(Duration::from_millis(5)).await;
+ if fail {
+ anyhow::bail!("produce failed")
+ }
+ Ok(())
+ }
+ .boxed(),
+ ))
+ }
+ }
+
+ fn msg(partition: i32, offset: i64) -> KafkaMessage {
+ KafkaMessage {
+ topic: "in".into(),
+ partition,
+ offset,
+ ..Default::default()
+ }
+ }
+
+ fn config(max_background_tasks: usize) -> ConsumerLoopConfig {
+ ConsumerLoopConfig {
+ batch_size: 10,
+ batch_timeout: Duration::from_millis(1),
+ max_background_tasks,
+ background_task_timeout: Duration::from_secs(5),
+ }
+ }
+
+ async fn run(
+ batches: Vec<Vec<KafkaMessage>>,
+ processor: FakeProcessor,
+ max_background_tasks: usize,
+ ) -> (anyhow::Result<LoopStats>, Vec<OffsetsToStore>, u32) {
+ let mut source = FakeSource {
+ batches: batches.into(),
+ ..Default::default()
+ };
+ let stored = source.stored.clone();
+ let commits = source.commits.clone();
+ let result = run_consumer_loop(
+ &mut source,
+ &processor,
+ &config(max_background_tasks),
+ tokio::time::sleep(Duration::from_millis(200)),
+ &|| {},
+ )
+ .await;
+ let stored = stored.lock().unwrap().clone();
+ let commits = *commits.lock().unwrap();
+ (result, stored, commits)
+ }
+
+ fn batches() -> Vec<Vec<KafkaMessage>> {
+ vec![
+ vec![msg(0, 0), msg(1, 7), msg(0, 1)],
+ vec![msg(0, 2)],
+ vec![msg(1, 8)],
+ ]
+ }
+
+ #[tokio::test]
+ async fn offsets_are_stored_in_order_after_each_task_succeeds() {
+ for max in [1, 3] {
+ let processor = FakeProcessor {
+ fail_task_at: vec![],
+ fail_batch_at: vec![],
+ ..Default::default()
+ };
+ let (result, stored, commits) = run(batches(), processor, max).await;
+ assert_eq!(result.unwrap().batches, 3);
+ assert_eq!(
+ stored,
+ vec![
+ vec![("in".into(), 0, 2), ("in".into(), 1, 8)],
+ vec![("in".into(), 0, 3)],
+ vec![("in".into(), 1, 9)],
+ ]
+ );
+ assert_eq!(commits, 1);
+ }
+ }
+
+ #[tokio::test]
+ async fn a_failed_task_stores_nothing_from_it_or_later_batches() {
+ for max in [1, 3] {
+ let processor = FakeProcessor {
+ fail_task_at: vec![2],
+ fail_batch_at: vec![],
+ ..Default::default()
+ };
+ let (result, stored, commits) = run(batches(), processor, max).await;
+ assert_eq!(result.unwrap_err().to_string(), "produce failed");
+ assert_eq!(stored, vec![vec![("in".into(), 0, 2), ("in".into(), 1, 8)]]);
+ assert_eq!(commits, 1);
+ }
+ }
+
+ #[tokio::test]
+ async fn a_hung_task_times_out_from_dispatch_when_traffic_stops() {
+ let processor = FakeProcessor {
+ hang_task_at: vec![2],
+ ..Default::default()
+ };
+ let mut source = FakeSource {
+ batches: batches().into(),
+ ..Default::default()
+ };
+ let config = ConsumerLoopConfig {
+ background_task_timeout: Duration::from_millis(50),
+ ..config(3)
+ };
+ let result = tokio::time::timeout(
+ Duration::from_secs(5),
+ run_consumer_loop(
+ &mut source,
+ &processor,
+ &config,
+ futures::future::pending(),
+ &|| {},
+ ),
+ )
+ .await
+ .expect("the loop must end without a shutdown or a full queue");
+ assert_eq!(
+ result.unwrap_err().to_string(),
+ "background_task_timeout_after_50ms"
+ );
+ assert_eq!(source.stored.lock().unwrap().len(), 1);
+ }
+
+ #[tokio::test]
+ async fn a_failed_batch_stops_the_loop_before_its_offsets() {
+ let processor = FakeProcessor {
+ fail_task_at: vec![],
+ fail_batch_at: vec![8],
+ ..Default::default()
+ };
+ let (result, stored, _) = run(batches(), processor, 1).await;
+ assert_eq!(result.unwrap_err().to_string(), "batch failed");
+ assert_eq!(stored.len(), 2);
+ }
+}
diff --git a/rust/logs-ingestion/src/service/kafka_outputs.rs b/rust/logs-ingestion/src/service/kafka_outputs.rs
new file mode 100644
index 000000000000..ebbe09506fc4
--- /dev/null
+++ b/rust/logs-ingestion/src/service/kafka_outputs.rs
@@ -0,0 +1,212 @@
+//! Every send waits for its delivery report, as `KafkaProducerWrapper.produce` awaits the callback.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use futures::future::join_all;
+use ingestion_framework::{IngestionOutputs, OutputMessage};
+use rdkafka::error::{KafkaError, RDKafkaErrorCode};
+use rdkafka::message::{Header, OwnedHeaders};
+use rdkafka::producer::{FutureProducer, FutureRecord, Producer};
+use rdkafka::util::Timeout;
+use rdkafka::ClientConfig;
+
+use crate::pipeline::{KafkaProduceError, NamedError};
+
+use super::config::OutputRoute;
+
+/// Env values from the producer's config map win.
+pub fn producer_client_config(
+ overrides: &[(String, String)],
+ client_rack: Option<&str>,
+) -> ClientConfig {
+ let mut config = ClientConfig::new();
+ let host = hostname();
+ for (k, v) in [
+ ("client.id", host.as_str()),
+ ("metadata.broker.list", "kafka:9092"),
+ ("linger.ms", "20"),
+ ("log_level", "4"),
+ ("batch.size", "8388608"),
+ ("queue.buffering.max.messages", "100000"),
+ ("compression.codec", "snappy"),
+ ("enable.idempotence", "true"),
+ ("metadata.max.age.ms", "30000"),
+ ("retry.backoff.ms", "500"),
+ ("socket.timeout.ms", "30000"),
+ ("max.in.flight.requests.per.connection", "5"),
+ ("statistics.interval.ms", "30000"),
+ ] {
+ config.set(k, v);
+ }
+ if let Some(rack) = client_rack {
+ config.set("client.rack", rack);
+ }
+ for (k, v) in overrides {
+ config.set(k, v);
+ }
+ config
+}
+
+pub fn hostname() -> String {
+ std::env::var("HOSTNAME").unwrap_or_else(|_| "logs-ingestion".to_string())
+}
+
+/// The error keeps its librdkafka code, so the pipeline replays an outage and quarantines only a code a replay cannot fix.
+pub fn classify_produce_error(error: KafkaError) -> anyhow::Error {
+ let Some(code) = error.rdkafka_error_code() else {
+ return anyhow::Error::new(NamedError::new("Error", error.to_string()));
+ };
+ let name = match code {
+ RDKafkaErrorCode::MessageSizeTooLarge => "MessageSizeTooLarge",
+ _ => "Error",
+ };
+ anyhow::Error::new(KafkaProduceError {
+ code: code as i32,
+ name: name.to_string(),
+ message: error.to_string(),
+ })
+}
+
+struct Route {
+ topic: String,
+ producer: FutureProducer,
+}
+
+pub struct KafkaOutputs {
+ routes: HashMap<&'static str, Route>,
+ producers: Vec<FutureProducer>,
+ queue_timeout: Duration,
+}
+
+impl KafkaOutputs {
+ /// A route that names a missing producer is a startup error.
+ pub fn new(
+ routes: &[OutputRoute],
+ producers: &HashMap<String, FutureProducer>,
+ ) -> anyhow::Result<Self> {
+ let mut out = HashMap::new();
+ for r in routes {
+ let producer = producers
+ .get(&r.producer)
+ .ok_or_else(|| anyhow::anyhow!("no producer {} for {}", r.producer, r.output))?
+ .clone();
+ out.insert(
+ r.output,
+ Route {
+ topic: r.topic.clone(),
+ producer,
+ },
+ );
+ }
+ Ok(Self {
+ routes: out,
+ producers: producers.values().cloned().collect(),
+ queue_timeout: Duration::from_secs(10),
+ })
+ }
+
+ pub fn topic(&self, output: &str) -> Option<&str> {
+ self.routes.get(output).map(|r| r.topic.as_str())
+ }
+
+ pub fn flush(&self, timeout: Duration) {
+ for p in &self.producers {
+ if let Err(error) = p.flush(Timeout::After(timeout)) {
+ tracing::error!(error = %error, "producer flush failed");
+ }
+ }
+ }
+
+ async fn send(&self, route: &Route, message: &OutputMessage) -> anyhow::Result<()> {
+ let mut headers = OwnedHeaders::new_with_capacity(message.headers.len());
+ for (k, v) in message.headers.iter() {
+ headers = headers.insert(Header {
+ key: k,
+ value: Some(v.as_bytes()),
+ });
+ }
+ let mut record: FutureRecord<'_, [u8], [u8]> =
+ FutureRecord::to(&route.topic).headers(headers);
+ if let Some(key) = &message.key {
+ record = record.key(key.as_slice());
+ }
+ if let Some(value) = &message.value {
+ record = record.payload(value.as_ref());
+ }
+ let topic: Arc<str> = Arc::from(route.topic.as_str());
+ match route
+ .producer
+ .send(record, Timeout::After(self.queue_timeout))
+ .await
+ {
+ Ok(_) => {
+ metrics::counter!("logs_ingestion_kafka_produced_total", "topic" => topic)
+ .increment(1);
+ Ok(())
+ }
+ Err((error, _)) => {
+ tracing::error!(error = %error, topic = %route.topic, "kafka_produce_error");
+ metrics::counter!("logs_ingestion_kafka_produce_failed_total", "topic" => topic)
+ .increment(1);
+ Err(classify_produce_error(error))
+ }
+ }
+ }
+}
+
+#[async_trait]
+impl IngestionOutputs for KafkaOutputs {
+ async fn produce(&self, output: &str, message: OutputMessage) -> anyhow::Result<()> {
+ self.queue_messages(output, vec![message]).await
+ }
+
+ async fn queue_messages(
+ &self,
+ output: &str,
+ messages: Vec<OutputMessage>,
+ ) -> anyhow::Result<()> {
+ let route = self
+ .routes
+ .get(output)
+ .ok_or_else(|| anyhow::anyhow!("Unknown output: {output}"))?;
+ let results = join_all(messages.iter().map(|m| self.send(route, m))).await;
+ results.into_iter().collect()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::pipeline::errors::{
+ error_name, is_dependency_unavailable, kafka_outage_as_dependency,
+ };
+
+ #[test]
+ fn produce_errors_classify_like_the_node_consumer() {
+ let classify = |error| kafka_outage_as_dependency(classify_produce_error(error));
+ let too_large = classify(KafkaError::MessageProduction(
+ RDKafkaErrorCode::MessageSizeTooLarge,
+ ));
+ assert_eq!(error_name(&too_large), "MessageSizeTooLarge");
+ assert!(!is_dependency_unavailable(&too_large));
+ for error in [
+ KafkaError::MessageProduction(RDKafkaErrorCode::InvalidRecord),
+ KafkaError::Canceled,
+ ] {
+ let classified = classify(error.clone());
+ assert_eq!(error_name(&classified), "Error", "{error:?}");
+ assert!(!is_dependency_unavailable(&classified), "{error:?}");
+ }
+ for code in [
+ RDKafkaErrorCode::QueueFull,
+ RDKafkaErrorCode::MessageTimedOut,
+ RDKafkaErrorCode::TopicAuthorizationFailed,
+ ] {
+ let error = classify(KafkaError::MessageProduction(code));
+ assert!(is_dependency_unavailable(&error), "{code:?}");
+ }
+ }
+}
diff --git a/rust/logs-ingestion/src/service/kafka_source.rs b/rust/logs-ingestion/src/service/kafka_source.rs
new file mode 100644
index 000000000000..49c7c1b5c8bc
--- /dev/null
+++ b/rust/logs-ingestion/src/service/kafka_source.rs
@@ -0,0 +1,373 @@
+//! Configured as `consumer-v1.ts` configures node-rdkafka, with offsets stored by the loop and committed by auto commit.
+
+use std::collections::HashMap;
+use std::sync::Mutex;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use ingestion_framework::KafkaMessage;
+use rand::Rng;
+use rdkafka::consumer::{
+ BaseConsumer, CommitMode, Consumer, ConsumerContext, Rebalance, StreamConsumer,
+};
+use rdkafka::error::KafkaError;
+use rdkafka::message::{BorrowedMessage, Headers, Message};
+use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};
+
+use super::consumer_loop::{BatchSource, OffsetsToStore};
+
+pub fn consumer_client_config(
+ group_id: &str,
+ overrides: &[(String, String)],
+ client_rack: Option<&str>,
+ test_env: bool,
+) -> ClientConfig {
+ let mut config = ClientConfig::new();
+ let host = super::kafka_outputs::hostname();
+ for (k, v) in [
+ ("client.id", host.as_str()),
+ ("security.protocol", "plaintext"),
+ ("metadata.broker.list", "kafka:9092"),
+ ("log_level", "4"),
+ ("session.timeout.ms", "30000"),
+ ("max.poll.interval.ms", "300000"),
+ ("max.partition.fetch.bytes", "1048576"),
+ ("fetch.error.backoff.ms", "100"),
+ ("fetch.message.max.bytes", "10485760"),
+ ("fetch.wait.max.ms", "50"),
+ ("queued.min.messages", "100000"),
+ ("queued.max.messages.kbytes", "102400"),
+ ("metadata.max.age.ms", "30000"),
+ ("socket.timeout.ms", "30000"),
+ ("enable.partition.eof", "true"),
+ // Node sets this in both consumer versions, so a new group starts at the oldest offset.
+ ("auto.offset.reset", "earliest"),
+ ] {
+ config.set(k, v);
+ }
+ if let Some(rack) = client_rack {
+ config.set("client.rack", rack);
+ }
+ for (k, v) in overrides {
+ config.set(k, v);
+ }
+ config
+ .set("group.id", group_id)
+ .set(
+ "partition.assignment.strategy",
+ if test_env {
+ "roundrobin"
+ } else {
+ "cooperative-sticky"
+ },
+ )
+ .set("enable.auto.offset.store", "false")
+ .set("enable.auto.commit", "true");
+ strip_classic_protocol_config(&mut config);
+ config
+}
+
+/// librdkafka rejects these classic-protocol keys under `group.protocol=consumer`, so they go after every other key is set.
+pub fn strip_classic_protocol_config(config: &mut ClientConfig) {
+ if config.get("group.protocol") != Some("consumer") {
+ return;
+ }
+ for key in [
+ "session.timeout.ms",
+ "heartbeat.interval.ms",
+ "partition.assignment.strategy",
+ "group.protocol.type",
+ ] {
+ config.remove(key);
+ }
+}
+
+pub fn to_kafka_message(m: &BorrowedMessage<'_>) -> KafkaMessage {
+ let headers = m
+ .headers()
+ .map(|hs| {
+ hs.iter()
+ .map(|h| (h.key.to_string(), h.value.map(<[u8]>::to_vec)))
+ .collect()
+ })
+ .unwrap_or_default();
+ KafkaMessage {
+ topic: m.topic().to_string(),
+ partition: m.partition(),
+ offset: m.offset(),
+ key: m.key().map(<[u8]>::to_vec),
+ value: m.payload().map(bytes::Bytes::copy_from_slice),
+ headers,
+ timestamp: m.timestamp().to_millis(),
+ }
+}
+
+/// A revoke bumps each revoked partition's epoch, so a task dispatched before it cannot store its offset after a reassignment.
+#[derive(Default)]
+pub struct PartitionEpochs(Mutex<HashMap<(String, i32), u64>>);
+
+impl PartitionEpochs {
+ pub fn revoke(&self, partitions: impl IntoIterator<Item = (String, i32)>) {
+ let mut epochs = self.0.lock().unwrap();
+ for p in partitions {
+ *epochs.entry(p).or_default() += 1;
+ }
+ }
+
+ pub fn snapshot(&self) -> HashMap<(String, i32), u64> {
+ self.0.lock().unwrap().clone()
+ }
+
+ pub fn storable(&self, offsets: &OffsetsToStore, dispatch: &[u64]) -> OffsetsToStore {
+ let epochs = self.0.lock().unwrap();
+ offsets
+ .iter()
+ .zip(dispatch)
+ .filter(|((topic, partition, _), at)| {
+ epochs
+ .get(&(topic.clone(), *partition))
+ .copied()
+ .unwrap_or_default()
+ == **at
+ })
+ .map(|(o, _)| o.clone())
+ .collect()
+ }
+}
+
+#[derive(Default)]
+pub struct FencingContext {
+ pub epochs: PartitionEpochs,
+}
+
+impl ClientContext for FencingContext {}
+
+impl ConsumerContext for FencingContext {
+ fn pre_rebalance(&self, _: &BaseConsumer<Self>, rebalance: &Rebalance<'_>) {
+ if let Rebalance::Revoke(tpl) = rebalance {
+ self.epochs.revoke(
+ tpl.elements()
+ .iter()
+ .map(|e| (e.topic().to_string(), e.partition())),
+ );
+ }
+ }
+}
+
+pub type LogsConsumer = StreamConsumer<FencingContext>;
+
+pub struct KafkaBatchSource {
+ consumer: std::mem::ManuallyDrop<LogsConsumer>,
+ dispatch_epochs: HashMap<(String, i32), u64>,
+}
+
+impl KafkaBatchSource {
+ pub fn new(consumer: LogsConsumer) -> Self {
+ Self {
+ consumer: std::mem::ManuallyDrop::new(consumer),
+ dispatch_epochs: HashMap::new(),
+ }
+ }
+
+ pub fn consumer(&self) -> &LogsConsumer {
+ &self.consumer
+ }
+}
+
+impl Drop for KafkaBatchSource {
+ fn drop(&mut self) {
+ // librdkafka deadlocks when it destroys a client after a fatal error, so that client leaks and the process exits.
+ if self.consumer.client().fatal_error().is_some() {
+ return;
+ }
+ // SAFETY: `consumer` is not used after this point, and `drop` runs once.
+ // nosemgrep: rust.lang.security.unsafe-usage.unsafe-usage
+ unsafe { std::mem::ManuallyDrop::drop(&mut self.consumer) }
+ }
+}
+
+/// node-rdkafka sets no `isRetriable` on a consume error, so every error is tried 3 times before the loop fails.
+pub async fn retry_consume<T, F, Fut>(mut attempt: F) -> anyhow::Result<T>
+where
+ F: FnMut() -> Fut,
+ Fut: std::future::Future<Output = anyhow::Result<T>>,
+{
+ const TRIES: u32 = 3;
+ const JITTER: f64 = 0.05;
+ let mut sleep_ms = 100.0_f64;
+ let mut tries = 1;
+ loop {
+ match attempt().await {
+ Ok(v) => return Ok(v),
+ Err(error) if tries >= TRIES => return Err(error),
+ Err(error) => {
+ tracing::warn!(error = %error, tries, "kafka consume failed, retrying");
+ let jittered = sleep_ms * (1.0 - JITTER + rand::thread_rng().gen::<f64>() * JITTER);
+ tokio::time::sleep(Duration::from_secs_f64(jittered / 1000.0)).await;
+ sleep_ms = (sleep_ms * 2.0).min(10_000.0);
+ tries += 1;
+ }
+ }
+ }
+}
+
+/// An error is returned only when no message was read, as node-rdkafka's `ConsumeNum` does.
+pub async fn recv_batch<C: ConsumerContext + 'static>(
+ consumer: &StreamConsumer<C>,
+ max: usize,
+ timeout: Duration,
+) -> anyhow::Result<Vec<KafkaMessage>> {
+ let deadline = tokio::time::Instant::now() + timeout;
+ let mut out = Vec::new();
+ while out.len() < max {
+ match tokio::time::timeout_at(deadline, consumer.recv()).await {
+ Err(_) => break,
+ Ok(Ok(m)) => out.push(to_kafka_message(&m)),
+ Ok(Err(KafkaError::PartitionEOF(_))) => continue,
+ Ok(Err(e)) => {
+ tracing::warn!(error = %e, "kafka consume error");
+ if out.is_empty() {
+ return Err(e.into());
+ }
+ break;
+ }
+ }
+ }
+ Ok(out)
+}
+
+#[async_trait]
+impl BatchSource for KafkaBatchSource {
+ async fn next_batch(
+ &mut self,
+ max: usize,
+ timeout: Duration,
+ ) -> anyhow::Result<Vec<KafkaMessage>> {
+ // Rebalance callbacks run inside recv, so every message here was read under this epoch or an older one.
+ self.dispatch_epochs = self.consumer.context().epochs.snapshot();
+ let consumer = &self.consumer;
+ let batch = retry_consume(|| recv_batch(consumer, max, timeout)).await;
+ // A fatal error is reported once, and later polls return empty batches, so the client state decides.
+ if let Some((code, reason)) = consumer.client().fatal_error() {
+ anyhow::bail!("fatal kafka consumer error {code:?}: {reason}");
+ }
+ batch
+ }
+
+ fn epochs(&self, offsets: &OffsetsToStore) -> Vec<u64> {
+ offsets
+ .iter()
+ .map(|(topic, partition, _)| {
+ self.dispatch_epochs
+ .get(&(topic.clone(), *partition))
+ .copied()
+ .unwrap_or_default()
+ })
+ .collect()
+ }
+
+ fn store_offsets(&mut self, offsets: &OffsetsToStore, epochs: &[u64]) -> anyhow::Result<()> {
+ let storable = self.consumer.context().epochs.storable(offsets, epochs);
+ let fenced = offsets.len() - storable.len();
+ if fenced > 0 {
+ metrics::counter!("logs_ingestion_consumer_stale_store_offsets_skipped_total")
+ .increment(fenced as u64);
+ }
+ if storable.is_empty() {
+ return Ok(());
+ }
+ let mut tpl = TopicPartitionList::new();
+ for (topic, partition, offset) in &storable {
+ tpl.add_partition_offset(topic, *partition, Offset::Offset(*offset))?;
+ }
+ // librdkafka also rejects a store for a partition this member no longer holds.
+ if let Err(error) = self.consumer.store_offsets(&tpl) {
+ tracing::warn!(error = %error, "offset store rejected");
+ }
+ Ok(())
+ }
+
+ fn commit(&mut self) -> anyhow::Result<()> {
+ match self.consumer.commit_consumer_state(CommitMode::Sync) {
+ Ok(())
+ | Err(KafkaError::ConsumerCommit(rdkafka::types::RDKafkaErrorCode::NoOffset)) => Ok(()),
+ Err(e) => Err(e.into()),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::atomic::{AtomicU32, Ordering};
+
+ #[tokio::test(start_paused = true)]
+ async fn consume_errors_retry_like_retry_if_retriable_then_fail() {
+ let calls = AtomicU32::new(0);
+ let recovered = retry_consume(|| async {
+ match calls.fetch_add(1, Ordering::SeqCst) {
+ 0 | 1 => Err(anyhow::anyhow!("transient")),
+ _ => Ok(7),
+ }
+ })
+ .await;
+ assert_eq!(recovered.unwrap(), 7);
+ calls.store(0, Ordering::SeqCst);
+ let failed: anyhow::Result<()> = retry_consume(|| async {
+ calls.fetch_add(1, Ordering::SeqCst);
+ Err(anyhow::anyhow!("down"))
+ })
+ .await;
+ assert_eq!(failed.unwrap_err().to_string(), "down");
+ assert_eq!(calls.load(Ordering::SeqCst), 3);
+ }
+
+ #[test]
+ fn a_revoke_fences_only_the_revoked_partition() {
+ let epochs = PartitionEpochs::default();
+ let offsets: OffsetsToStore = vec![("in".into(), 0, 5), ("in".into(), 1, 9)];
+ let at_dispatch = vec![0, 0];
+ epochs.revoke([("in".to_string(), 1)]);
+ assert_eq!(
+ epochs.storable(&offsets, &at_dispatch),
+ vec![("in".to_string(), 0, 5)]
+ );
+ let after = epochs.snapshot();
+ assert_eq!(after.get(&("in".to_string(), 1)), Some(&1));
+ assert_eq!(epochs.storable(&offsets, &[0, 1]), offsets);
+ }
+
+ fn config(overrides: &[(&str, &str)]) -> ClientConfig {
+ let overrides: Vec<(String, String)> = overrides
+ .iter()
+ .map(|(k, v)| (k.to_string(), v.to_string()))
+ .collect();
+ consumer_client_config("g", &overrides, None, false)
+ }
+
+ #[test]
+ fn consumer_config_matches_node_offset_reset_and_kip848_strip() {
+ let classic = config(&[]);
+ assert_eq!(classic.get("auto.offset.reset"), Some("earliest"));
+ assert_eq!(
+ classic.get("partition.assignment.strategy"),
+ Some("cooperative-sticky")
+ );
+ assert_eq!(
+ config(&[("auto.offset.reset", "latest")]).get("auto.offset.reset"),
+ Some("latest")
+ );
+ let kip848 = config(&[
+ ("group.protocol", "consumer"),
+ ("heartbeat.interval.ms", "3000"),
+ ]);
+ for key in [
+ "session.timeout.ms",
+ "heartbeat.interval.ms",
+ "partition.assignment.strategy",
+ ] {
+ assert_eq!(kip848.get(key), None, "{key}");
+ }
+ assert_eq!(kip848.get("group.id"), Some("g"));
+ }
+}
diff --git a/rust/logs-ingestion/src/service/lazy_loader.rs b/rust/logs-ingestion/src/service/lazy_loader.rs
new file mode 100644
index 000000000000..7abbfee509c6
--- /dev/null
+++ b/rust/logs-ingestion/src/service/lazy_loader.rs
@@ -0,0 +1,324 @@
+//! Loader retries are not ported, because the logs servers build their loaders without `loaderRetry`.
+
+use std::collections::{HashMap, HashSet};
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use futures::future::{BoxFuture, FutureExt, Shared};
+use rand::Rng;
+
+use crate::pipeline::DependencyUnavailableError;
+
+pub type LoadResult<V> = anyhow::Result<HashMap<String, Option<V>>>;
+pub type Loader<V> = Arc<dyn Fn(Vec<String>) -> BoxFuture<'static, LoadResult<V>> + Send + Sync>;
+type SharedLoad = Shared<BoxFuture<'static, Result<(), Arc<anyhow::Error>>>>;
+
+#[derive(Clone)]
+pub struct LazyLoaderOptions {
+ pub name: &'static str,
+ pub refresh_age: Duration,
+ pub refresh_null_age: Option<Duration>,
+ pub background_age: Option<Duration>,
+ pub jitter: Option<Duration>,
+ pub buffer: Duration,
+ pub max_size: usize,
+}
+
+impl LazyLoaderOptions {
+ pub fn new(name: &'static str, refresh_age: Duration) -> Self {
+ Self {
+ name,
+ refresh_age,
+ refresh_null_age: None,
+ background_age: None,
+ jitter: None,
+ buffer: Duration::from_millis(10),
+ max_size: 100_000,
+ }
+ }
+}
+
+struct Entry<V> {
+ value: Option<V>,
+ last_used: Instant,
+ /// `None` after `mark_for_refresh`.
+ cache_until: Option<Instant>,
+ background_after: Option<Instant>,
+}
+
+struct State<V> {
+ cache: HashMap<String, Entry<V>>,
+ pending: HashMap<String, SharedLoad>,
+ buffer: Option<(HashSet<String>, SharedLoad)>,
+}
+
+struct Inner<V> {
+ options: LazyLoaderOptions,
+ loader: Loader<V>,
+ state: Mutex<State<V>>,
+}
+
+pub struct LazyLoader<V> {
+ inner: Arc<Inner<V>>,
+}
+
+impl<V> Clone for LazyLoader<V> {
+ fn clone(&self) -> Self {
+ Self {
+ inner: self.inner.clone(),
+ }
+ }
+}
+
+/// Keeps `DependencyUnavailableError` recognisable after the error went through a shared load.
+fn rebuild_error(error: &anyhow::Error) -> anyhow::Error {
+ match error.downcast_ref::<DependencyUnavailableError>() {
+ Some(e) => anyhow::Error::new(e.clone()),
+ None => anyhow::anyhow!("{error:#}"),
+ }
+}
+
+impl<V: Clone + Send + Sync + 'static> LazyLoader<V> {
+ pub fn new(options: LazyLoaderOptions, loader: Loader<V>) -> Self {
+ Self {
+ inner: Arc::new(Inner {
+ options,
+ loader,
+ state: Mutex::new(State {
+ cache: HashMap::new(),
+ pending: HashMap::new(),
+ buffer: None,
+ }),
+ }),
+ }
+ }
+
+ pub async fn get(&self, key: &str) -> anyhow::Result<Option<V>> {
+ let wait = {
+ let mut state = self.inner.state.lock().unwrap();
+ let now = Instant::now();
+ if let Some(entry) = state.cache.get_mut(key) {
+ entry.last_used = now;
+ if entry.cache_until.is_some_and(|until| now <= until) {
+ let value = entry.value.clone();
+ if entry.background_after.is_none_or(|b| now <= b) {
+ return Ok(value);
+ }
+ let load = self.load_locked(&mut state, key);
+ let name = self.inner.options.name;
+ let key = key.to_string();
+ tokio::spawn(async move {
+ if let Err(error) = load.await {
+ tracing::warn!(loader = name, key, error = %error, "LazyLoader background refresh failed");
+ }
+ });
+ return Ok(value);
+ }
+ }
+ self.load_locked(&mut state, key)
+ };
+ if let Err(error) = wait.await {
+ return Err(rebuild_error(&error));
+ }
+ let state = self.inner.state.lock().unwrap();
+ Ok(state.cache.get(key).and_then(|e| e.value.clone()))
+ }
+
+ /// The next lookup reloads before it returns.
+ pub fn mark_for_refresh(&self, keys: &[String]) {
+ let mut state = self.inner.state.lock().unwrap();
+ for key in keys {
+ if let Some(entry) = state.cache.get_mut(key) {
+ entry.cache_until = None;
+ entry.background_after = None;
+ }
+ }
+ }
+
+ fn load_locked(&self, state: &mut State<V>, key: &str) -> SharedLoad {
+ if let Some(pending) = state.pending.get(key) {
+ return pending.clone();
+ }
+ let load = match &mut state.buffer {
+ Some((keys, load)) => {
+ keys.insert(key.to_string());
+ load.clone()
+ }
+ None => {
+ let load = self.spawn_buffer();
+ let mut keys = HashSet::new();
+ keys.insert(key.to_string());
+ state.buffer = Some((keys, load.clone()));
+ load
+ }
+ };
+ state.pending.insert(key.to_string(), load.clone());
+ load
+ }
+
+ fn spawn_buffer(&self) -> SharedLoad {
+ let inner = self.inner.clone();
+ let task = tokio::spawn(async move {
+ tokio::time::sleep(inner.options.buffer).await;
+ let keys: Vec<String> = {
+ let mut state = inner.state.lock().unwrap();
+ state
+ .buffer
+ .take()
+ .map(|(k, _)| k.into_iter().collect())
+ .unwrap_or_default()
+ };
+ let result = (inner.loader)(keys.clone()).await;
+ let mut state = inner.state.lock().unwrap();
+ for key in &keys {
+ state.pending.remove(key);
+ }
+ match result {
+ Ok(mut loaded) => {
+ for key in keys {
+ loaded.entry(key).or_insert(None);
+ }
+ set_values(&inner.options, &mut state, loaded);
+ Ok(())
+ }
+ Err(e) => Err(Arc::new(e)),
+ }
+ });
+ async move {
+ match task.await {
+ Ok(r) => r,
+ Err(e) => Err(Arc::new(anyhow::anyhow!("lazy loader task failed: {e}"))),
+ }
+ }
+ .boxed()
+ .shared()
+ }
+}
+
+fn set_values<V>(
+ options: &LazyLoaderOptions,
+ state: &mut State<V>,
+ loaded: HashMap<String, Option<V>>,
+) {
+ let now = Instant::now();
+ let jitter_max = options.jitter.unwrap_or(options.refresh_age / 5);
+ let null_age = options.refresh_null_age.unwrap_or(options.refresh_age);
+ let mut rng = rand::thread_rng();
+ for (key, value) in loaded {
+ let jitter = if jitter_max.is_zero() {
+ Duration::ZERO
+ } else {
+ Duration::from_millis(rng.gen_range(0..jitter_max.as_millis().max(1) as u64))
+ };
+ let age = if value.is_none() {
+ null_age
+ } else {
+ options.refresh_age
+ };
+ let background_after = options
+ .background_age
+ .map(|b| now + if value.is_none() { null_age } else { b } + jitter);
+ state.cache.insert(
+ key,
+ Entry {
+ value,
+ last_used: now,
+ cache_until: Some(now + age + jitter),
+ background_after,
+ },
+ );
+ }
+ if state.cache.len() > options.max_size {
+ let headroom = if options.max_size > 100 {
+ options.max_size.div_ceil(10)
+ } else {
+ 0
+ };
+ let evict = state.cache.len() - options.max_size + headroom;
+ let mut keys: Vec<(Instant, String)> = state
+ .cache
+ .iter()
+ .map(|(k, e)| (e.last_used, k.clone()))
+ .collect();
+ keys.sort();
+ for (_, key) in keys.into_iter().take(evict) {
+ state.cache.remove(&key);
+ }
+ }
+ metrics::gauge!("lazy_loader_cache_size", "name" => options.name).set(state.cache.len() as f64);
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ fn counting_loader(calls: Arc<Mutex<Vec<Vec<String>>>>) -> Loader<String> {
+ Arc::new(move |mut keys: Vec<String>| {
+ keys.sort();
+ calls.lock().unwrap().push(keys.clone());
+ async move {
+ Ok(keys
+ .into_iter()
+ .map(|k| {
+ let v = (k != "missing").then(|| format!("v-{k}"));
+ (k, v)
+ })
+ .collect())
+ }
+ .boxed()
+ })
+ }
+
+ #[tokio::test]
+ async fn concurrent_misses_share_one_buffered_load_and_hits_skip_it() {
+ let calls = Arc::new(Mutex::new(Vec::new()));
+ let loader = LazyLoader::new(
+ LazyLoaderOptions::new("t", Duration::from_secs(60)),
+ counting_loader(calls.clone()),
+ );
+ let (a, b, c) = tokio::join!(loader.get("a"), loader.get("b"), loader.get("a"));
+ assert_eq!(a.unwrap().as_deref(), Some("v-a"));
+ assert_eq!(b.unwrap().as_deref(), Some("v-b"));
+ assert_eq!(c.unwrap().as_deref(), Some("v-a"));
+ assert_eq!(loader.get("missing").await.unwrap(), None);
+ assert_eq!(loader.get("missing").await.unwrap(), None);
+ assert_eq!(loader.get("a").await.unwrap().as_deref(), Some("v-a"));
+ assert_eq!(
+ *calls.lock().unwrap(),
+ vec![
+ vec!["a".to_string(), "b".to_string()],
+ vec!["missing".to_string()]
+ ]
+ );
+ loader.mark_for_refresh(&["a".to_string()]);
+ loader.get("a").await.unwrap();
+ assert_eq!(calls.lock().unwrap().len(), 3);
+ }
+
+ #[tokio::test]
+ async fn a_failed_load_reaches_every_waiter_and_is_not_cached() {
+ let attempts = Arc::new(AtomicUsize::new(0));
+ let a = attempts.clone();
+ let loader: LazyLoader<String> = LazyLoader::new(
+ LazyLoaderOptions::new("t", Duration::from_secs(60)),
+ Arc::new(move |_keys| {
+ a.fetch_add(1, Ordering::SeqCst);
+ async move {
+ Err(anyhow::Error::new(DependencyUnavailableError {
+ message: "down".into(),
+ dependency_name: "Postgres".into(),
+ }))
+ }
+ .boxed()
+ }),
+ );
+ let (x, y) = tokio::join!(loader.get("a"), loader.get("a"));
+ assert!(crate::pipeline::errors::is_dependency_unavailable(
+ &x.unwrap_err()
+ ));
+ assert!(y.is_err());
+ assert!(loader.get("a").await.is_err());
+ assert_eq!(attempts.load(Ordering::SeqCst), 2);
+ }
+}
diff --git a/rust/logs-ingestion/src/service/mod.rs b/rust/logs-ingestion/src/service/mod.rs
new file mode 100644
index 000000000000..6b0883eaa7f4
--- /dev/null
+++ b/rust/logs-ingestion/src/service/mod.rs
@@ -0,0 +1,309 @@
+//! Shadow mode runs beside the Node consumer without affecting production.
+
+pub mod config;
+pub mod consumer_loop;
+pub mod kafka_outputs;
+pub mod kafka_source;
+pub mod lazy_loader;
+pub mod node_env;
+pub mod postgres;
+pub mod redis_deps;
+pub mod shadow;
+pub mod usage;
+
+use std::collections::HashMap;
+use std::future::ready;
+use std::sync::Arc;
+use std::time::Duration;
+
+use axum::response::IntoResponse;
+use axum::routing::get;
+use axum::Router;
+use lifecycle::{ComponentOptions, Manager};
+use rdkafka::consumer::Consumer;
+use rdkafka::metadata::Metadata;
+use rdkafka::producer::{FutureProducer, Producer};
+use sqlx::PgPool;
+
+use crate::metric_rules::LogsMetricsEmitter;
+use crate::pipeline::{
+ DependencyRetry, LogsConsumerIdentity, LogsIngestionConsumer, LogsIngestionConsumerDeps,
+};
+use crate::rate_limiter::{system_clock, KeyRoot, RedisTokenBucket};
+
+use config::{ConsumerKind, RunMode, ServiceConfig};
+use consumer_loop::run_consumer_loop;
+use kafka_outputs::{producer_client_config, KafkaOutputs};
+use kafka_source::{consumer_client_config, FencingContext, KafkaBatchSource, LogsConsumer};
+use node_env::NodeEnv;
+
+const SERVICE_NAME: &str = "logs-ingestion";
+/// How long a task may take to finish once the lifecycle monitor has returned.
+const TASK_JOIN_GRACE: Duration = Duration::from_secs(2);
+
+/// The shadow-mode rules, applied to any set of dependencies (README.md lists them).
+pub fn shadow_deps(mut deps: LogsIngestionConsumerDeps) -> LogsIngestionConsumerDeps {
+ deps.team_manager = Arc::new(shadow::ReadOnlyTeamManager(deps.team_manager));
+ deps.usage_batch = Arc::new(shadow::DiscardUsageRecords);
+ deps.metrics_emitter = deps
+ .metrics_emitter
+ .map(|_| Arc::new(shadow::SuppressedMetricsEmitter) as _);
+ deps.logs_transformer = None;
+ deps.key_root = KeyRoot::Shadow;
+ deps.source_offset_header = true;
+ deps
+}
+
+pub fn identity(kind: ConsumerKind) -> LogsConsumerIdentity {
+ match kind {
+ ConsumerKind::Logs => LogsConsumerIdentity::logs(),
+ ConsumerKind::Traces => LogsConsumerIdentity::traces(),
+ }
+}
+
+pub fn create_producers(config: &ServiceConfig) -> anyhow::Result<HashMap<String, FutureProducer>> {
+ let mut out = HashMap::new();
+ for (name, overrides) in &config.producers {
+ let topics: Vec<&str> = config
+ .routes
+ .iter()
+ .filter(|r| &r.producer == name)
+ .map(|r| r.topic.as_str())
+ .collect();
+ if topics.is_empty() {
+ continue;
+ }
+ let mut client = producer_client_config(overrides, config.kafka_client_rack.as_deref());
+ if config.mode == RunMode::Shadow {
+ // A mistyped shadow topic must fail here instead of creating a topic on the cluster.
+ client.set("allow.auto.create.topics", "false");
+ }
+ let producer: FutureProducer = client.create()?;
+ let metadata = producer
+ .client()
+ .fetch_metadata(None, Duration::from_secs(15))
+ .map_err(|e| anyhow::anyhow!("producer {name} cannot reach Kafka: {e}"))?;
+ if config.mode == RunMode::Shadow {
+ for topic in topics {
+ require_topic(&metadata, topic)?;
+ }
+ }
+ out.insert(name.clone(), producer);
+ }
+ Ok(out)
+}
+
+/// Brokers below Metadata v4 ignore `allow.auto.create.topics` and create a named topic, so this checks the all-topics metadata.
+fn require_topic(metadata: &Metadata, topic: &str) -> anyhow::Result<()> {
+ let exists = metadata
+ .topics()
+ .iter()
+ .any(|t| t.name() == topic && t.error().is_none() && !t.partitions().is_empty());
+ anyhow::ensure!(exists, "shadow topic {topic} does not exist");
+ Ok(())
+}
+
+pub fn create_consumer(config: &ServiceConfig) -> anyhow::Result<LogsConsumer> {
+ let consumer: LogsConsumer = consumer_client_config(
+ &config.group_id,
+ &config.kafka_consumer,
+ config.kafka_client_rack.as_deref(),
+ config.node_env == NodeEnv::Test,
+ )
+ .create_with_context(FencingContext::default())?;
+ consumer.subscribe(&[config.input_topic.as_str()])?;
+ Ok(consumer)
+}
+
+fn pool(url: &str, size: u32) -> anyhow::Result<PgPool> {
+ Ok(common_database::get_pool_with_config(
+ url,
+ common_database::PoolConfig {
+ max_connections: size,
+ pool_name: Some(SERVICE_NAME.to_string()),
+ ..Default::default()
+ },
+ )?)
+}
+
+pub fn build_deps(
+ config: &ServiceConfig,
+ outputs: Arc<KafkaOutputs>,
+) -> anyhow::Result<LogsIngestionConsumerDeps> {
+ let reader = pool(&config.database_reader_url, config.pool_size)?;
+ let writer = match (config.mode, &config.database_writer_url) {
+ (RunMode::Primary, Some(url)) => Some(pool(url, config.pool_size)?),
+ _ => None,
+ };
+ let clock = system_clock();
+ let posthog_redis =
+ redis_deps::ReconnectingRedis::new(redis_deps::connection_info(&config.posthog_redis)?)?;
+ let logs_redis =
+ redis_deps::ReconnectingRedis::new(redis_deps::connection_info(&config.logs_redis)?)?;
+ let logs = config.kind == ConsumerKind::Logs;
+ let metrics_enabled = !config.metrics_export_url.is_empty();
+ let usage_batch: Arc<dyn crate::pipeline::UsageRecordBatch> = match config.mode {
+ RunMode::Shadow => Arc::new(shadow::DiscardUsageRecords),
+ RunMode::Primary => Arc::new(usage::GrpcUsageRecordBatch::new(
+ &config.usage,
+ if logs { "logs" } else { "apm-traces" },
+ clock.clone(),
+ )?),
+ };
+ let deps = LogsIngestionConsumerDeps {
+ team_manager: Arc::new(postgres::PgTeamManager::new(reader.clone(), writer)),
+ quota_limiting: Arc::new(redis_deps::RedisQuotaLimiting::new(
+ posthog_redis,
+ clock.clone(),
+ )),
+ outputs,
+ usage_batch,
+ sampling_rules: logs.then(|| Arc::new(postgres::PgSamplingRules::new(reader.clone())) as _),
+ metric_rules: metrics_enabled
+ .then(|| Arc::new(postgres::PgMetricRules::new(reader.clone())) as _),
+ metrics_emitter: if metrics_enabled {
+ Some(Arc::new(LogsMetricsEmitter::new(
+ config.metrics_export_url.clone(),
+ shadow::ReqwestOtlpTransport::new()?,
+ )) as _)
+ } else {
+ None
+ },
+ logs_transformer: None,
+ retention_rules: Some(Arc::new(postgres::PgRetentionRules::new(reader.clone()))),
+ pattern_message_keys: logs
+ .then(|| Arc::new(postgres::PgPatternMessageKeys::new(reader.clone())) as _),
+ tracing_config: (!logs)
+ .then(|| Arc::new(postgres::PgTracingConfig::new(reader.clone())) as _),
+ dependency_retry: DependencyRetry::default(),
+ internal_source_policies: None,
+ token_bucket_redis: Arc::new(RedisTokenBucket::new(logs_redis)),
+ key_root: if config.node_env == NodeEnv::Test {
+ KeyRoot::Test
+ } else {
+ KeyRoot::Production
+ },
+ clock,
+ source_offset_header: false,
+ };
+ Ok(match config.mode {
+ RunMode::Primary => deps,
+ RunMode::Shadow => shadow_deps(deps),
+ })
+}
+
+pub fn health_router(
+ readiness: lifecycle::ReadinessHandler,
+ liveness: lifecycle::LivenessHandler,
+ render_metrics: Arc<dyn Fn() -> String + Send + Sync>,
+) -> Router {
+ Router::new()
+ .route("/", get(|| async { SERVICE_NAME }))
+ .route(
+ "/_health",
+ get(move || {
+ let liveness = liveness.clone();
+ async move { liveness.check().into_response() }
+ }),
+ )
+ .route(
+ "/_ready",
+ get(move || {
+ let readiness = readiness.clone();
+ async move { readiness.check().await }
+ }),
+ )
+ .route("/metrics", get(move || ready(render_metrics())))
+}
+
+/// A failed batch ends the process with an error, as the Node consumer crashes, so the restart replays from the last commit.
+pub async fn run(config: ServiceConfig) -> anyhow::Result<()> {
+ tracing::info!(
+ kind = ?config.kind,
+ mode = ?config.mode,
+ group_id = %config.group_id,
+ input_topic = %config.input_topic,
+ routes = ?config.routes,
+ "starting logs-ingestion"
+ );
+ let recorder = serve_metrics::setup_metrics_recorder();
+ let mut manager = Manager::builder(SERVICE_NAME)
+ .with_global_shutdown_timeout(config.shutdown_timeout)
+ .build();
+ let metrics_handle =
+ manager.register("metrics", ComponentOptions::new().is_observability(true));
+ let consumer_handle = manager.register(
+ "consumer",
+ ComponentOptions::new()
+ .with_graceful_shutdown(config.shutdown_timeout)
+ .with_liveness_deadline(Duration::from_secs(300))
+ .with_stall_threshold(2),
+ );
+ let readiness = manager.readiness_handler();
+ let liveness = manager.liveness_handler();
+
+ let producers = create_producers(&config)?;
+ let outputs = Arc::new(KafkaOutputs::new(&config.routes, &producers)?);
+ let deps = build_deps(&config, outputs.clone())?;
+ let consumer = LogsIngestionConsumer::new(identity(config.kind), config.consumer.clone(), deps);
+ let mut source = KafkaBatchSource::new(create_consumer(&config)?);
+ let guard = manager.monitor_background();
+
+ let loop_config = config.consumer_loop.clone();
+ let consumer_task = tokio::spawn(async move {
+ let handle = consumer_handle;
+ let _scope = handle.process_scope();
+ let tick_handle = handle.clone();
+ let result = run_consumer_loop(
+ &mut source,
+ &consumer,
+ &loop_config,
+ handle.shutdown_signal(),
+ &move || tick_handle.report_healthy(),
+ )
+ .await;
+ outputs.flush(Duration::from_secs(10));
+ match &result {
+ Ok(stats) => tracing::info!(
+ batches = stats.batches,
+ messages = stats.messages,
+ "consumer stopped"
+ ),
+ Err(error) => handle.signal_failure(format!("{error:#}")),
+ }
+ result
+ });
+
+ let app = health_router(readiness, liveness, Arc::new(move || recorder.render()));
+ let listener = tokio::net::TcpListener::bind(("0.0.0.0", config.http_port)).await?;
+ let server_task = tokio::spawn(async move {
+ let result = axum::serve(listener, app)
+ .with_graceful_shutdown(metrics_handle.shutdown_signal())
+ .await;
+ match &result {
+ Ok(()) => metrics_handle.work_completed(),
+ Err(error) => metrics_handle.signal_failure(format!("http server failed: {error}")),
+ }
+ result
+ });
+ // The monitor returns by the global shutdown deadline, so nothing after it waits unbounded.
+ let monitor = guard.wait().await;
+ let consumer_result = join_or_abort("consumer", consumer_task).await;
+ let server_result = join_or_abort("http server", server_task).await;
+ consumer_result??;
+ server_result??;
+ monitor?;
+ Ok(())
+}
+
+async fn join_or_abort<T>(name: &str, mut task: tokio::task::JoinHandle<T>) -> anyhow::Result<T> {
+ match tokio::time::timeout(TASK_JOIN_GRACE, &mut task).await {
+ Ok(joined) => Ok(joined?),
+ Err(_) => {
+ task.abort();
+ Err(anyhow::anyhow!(
+ "{name} did not stop by the shutdown deadline"
+ ))
+ }
+ }
+}
diff --git a/rust/logs-ingestion/src/service/node_env.rs b/rust/logs-ingestion/src/service/node_env.rs
new file mode 100644
index 000000000000..47cee9aca246
--- /dev/null
+++ b/rust/logs-ingestion/src/service/node_env.rs
@@ -0,0 +1,153 @@
+use std::collections::HashMap;
+
+use crate::js::{JsObject, JsString, JsValue};
+use crate::metric_rules::js_parse_float;
+use crate::property_filter::coerce::js_parse_int;
+
+pub type Env = HashMap<String, String>;
+
+pub fn process_env() -> Env {
+ std::env::vars().collect()
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum NodeEnv {
+ Development,
+ Production,
+ Test,
+}
+
+pub fn string_to_boolean(value: &str) -> bool {
+ matches!(
+ value.to_lowercase().as_str(),
+ "y" | "yes" | "t" | "true" | "on" | "1"
+ )
+}
+
+pub fn determine_node_env(env: &Env) -> NodeEnv {
+ if let Some(raw) = env.get("NODE_ENV").filter(|v| !v.is_empty()) {
+ let raw = raw.to_lowercase();
+ if raw.starts_with("test") {
+ return NodeEnv::Test;
+ }
+ if raw.starts_with("dev") {
+ return NodeEnv::Development;
+ }
+ }
+ if env.get("DEBUG").is_some_and(|v| string_to_boolean(v)) {
+ return NodeEnv::Development;
+ }
+ NodeEnv::Production
+}
+
+pub fn kafka_topic(env: &Env, name: &str) -> String {
+ let prefix = env.get("KAFKA_PREFIX").map(String::as_str).unwrap_or("");
+ let suffix = if determine_node_env(env) == NodeEnv::Test {
+ "_test"
+ } else {
+ ""
+ };
+ format!("{prefix}{name}{suffix}")
+}
+
+/// The default's type picks the parse.
+pub fn override_with_env(defaults: &[(&str, JsValue)], env: &Env) -> JsObject {
+ let mut out = JsObject::new();
+ for (key, default) in defaults {
+ let value = match env.get(*key) {
+ None => default.clone(),
+ Some(raw) => match default {
+ // `indexOf('.')` is 0 only for a leading dot, which takes `parseInt`.
+ JsValue::Number(_) if raw.starts_with('.') => {
+ JsValue::Number(js_parse_int(raw, None))
+ }
+ JsValue::Number(_) => JsValue::Number(js_parse_float(&JsString::from(raw.clone()))),
+ JsValue::Bool(_) => JsValue::Bool(string_to_boolean(raw)),
+ _ => JsValue::string(raw),
+ },
+ };
+ out.insert_str(key, value);
+ }
+ out
+}
+
+/// Empty values are skipped. `skip` lists Node `defaultConfig` keys that the prefix also matches.
+pub fn rdkafka_from_env(env: &Env, prefix: &str, skip: &[&str]) -> Vec<(String, String)> {
+ let mut out: Vec<(String, String)> = env
+ .iter()
+ .filter(|(k, v)| k.starts_with(prefix) && !v.is_empty() && !skip.contains(&k.as_str()))
+ .map(|(k, v)| {
+ let key = k[prefix.len()..].to_lowercase().replace('_', ".");
+ let value = match v.to_lowercase().as_str() {
+ "true" => "true".to_string(),
+ "false" => "false".to_string(),
+ _ => v.clone(),
+ };
+ (key, value)
+ })
+ .collect();
+ out.sort();
+ out
+}
+
+pub fn get_string(config: &JsObject, key: &str) -> String {
+ match config.get(key) {
+ Some(JsValue::String(s)) => s.to_utf8_lossy().into_owned(),
+ Some(JsValue::Number(n)) => crate::js::number_to_string(*n),
+ Some(JsValue::Bool(b)) => b.to_string(),
+ _ => String::new(),
+ }
+}
+
+pub fn get_number(config: &JsObject, key: &str) -> f64 {
+ match config.get(key) {
+ Some(JsValue::Number(n)) => *n,
+ _ => f64::NAN,
+ }
+}
+
+pub fn get_bool(config: &JsObject, key: &str) -> bool {
+ matches!(config.get(key), Some(JsValue::Bool(true)))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn env(pairs: &[(&str, &str)]) -> Env {
+ pairs
+ .iter()
+ .map(|(k, v)| (k.to_string(), v.to_string()))
+ .collect()
+ }
+
+ #[test]
+ fn env_overrides_parse_like_override_config_with_env() {
+ let defaults = [
+ ("N", JsValue::Number(1.0)),
+ ("B", JsValue::Bool(false)),
+ ("S", JsValue::string("x")),
+ ("D", JsValue::Number(2.0)),
+ ];
+ let o = override_with_env(&defaults, &env(&[("N", "12.5kb"), ("B", "on"), ("S", "")]));
+ assert_eq!(o.get("N"), Some(&JsValue::Number(12.5)));
+ assert_eq!(o.get("B"), Some(&JsValue::Bool(true)));
+ assert_eq!(o.get("S"), Some(&JsValue::string("")));
+ assert_eq!(o.get("D"), Some(&JsValue::Number(2.0)));
+ let o = override_with_env(&defaults, &env(&[("N", ".5")]));
+ assert!(matches!(o.get("N"), Some(JsValue::Number(n)) if n.is_nan()));
+ }
+
+ #[test]
+ fn topic_names_follow_prefix_and_test_suffix() {
+ assert_eq!(kafka_topic(&env(&[]), "clickhouse_logs"), "clickhouse_logs");
+ assert_eq!(
+ kafka_topic(&env(&[("NODE_ENV", "test"), ("KAFKA_PREFIX", "p_")]), "x"),
+ "p_x_test"
+ );
+ assert_eq!(
+ determine_node_env(&env(&[("DEBUG", "1")])),
+ NodeEnv::Development
+ );
+ }
+}
diff --git a/rust/logs-ingestion/src/service/postgres.rs b/rust/logs-ingestion/src/service/postgres.rs
new file mode 100644
index 000000000000..48065d235e65
--- /dev/null
+++ b/rust/logs-ingestion/src/service/postgres.rs
@@ -0,0 +1,668 @@
+//! The queries, TTLs and fallbacks match their TypeScript counterparts.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use futures::future::{BoxFuture, FutureExt, Shared};
+use sqlx::{PgPool, Row};
+
+use crate::js::{json_parse, JsObject, JsValue};
+use crate::metric_rules::{
+ compile_metric_rules, CompiledMetricRule, MetricRuleRow, MetricRulesSource,
+};
+use crate::pattern::MESSAGE_KEYS;
+use crate::pipeline::scenario::logs_settings;
+use crate::pipeline::{
+ DependencyUnavailableError, PatternMessageKeysSource, RetentionRulesSource,
+ SamplingRulesSource, Team, TeamManager, TracingConfigSource,
+};
+use crate::retention::{
+ compile_retention_rule_set, CompiledRetentionRuleSet, RetentionRuleRow,
+ MAX_ENABLED_RETENTION_RULES,
+};
+use crate::sampling::{compile_rule_set, CompiledRuleSet, SamplingRuleRow};
+
+use super::lazy_loader::{LazyLoader, LazyLoaderOptions};
+
+const UNAVAILABLE_MESSAGES: &[&str] = &[
+ "connection to server at",
+ "could not translate host",
+ "server conn crashed",
+ "no more connections allowed",
+ "server closed the connection unexpectedly",
+ "getaddrinfo EAI_AGAIN",
+ "getaddrinfo ENOTFOUND",
+ "Connection terminated unexpectedly",
+ "ECONNREFUSED",
+ "ECONNRESET",
+ "EHOSTUNREACH",
+ "ETIMEDOUT",
+ "query_wait_timeout",
+ "server login has been failing",
+ "pooler is shutting down",
+ "Cannot use a pool after calling end on the pool",
+ "conflict with recovery",
+];
+
+/// sqlx reports a lost connection as an I/O or pool error instead.
+pub fn classify_pg_error(error: sqlx::Error) -> anyhow::Error {
+ let transient = match &error {
+ sqlx::Error::Io(_)
+ | sqlx::Error::PoolTimedOut
+ | sqlx::Error::PoolClosed
+ | sqlx::Error::WorkerCrashed
+ | sqlx::Error::Tls(_) => true,
+ other => {
+ let text = other.to_string();
+ UNAVAILABLE_MESSAGES.iter().any(|m| text.contains(m))
+ }
+ };
+ if transient {
+ metrics::counter!("logs_ingestion_postgres_unavailable_total").increment(1);
+ anyhow::Error::new(DependencyUnavailableError {
+ message: error.to_string(),
+ dependency_name: "Postgres".to_string(),
+ })
+ } else {
+ anyhow::Error::new(error)
+ }
+}
+
+fn parse_json_column(text: Option<String>) -> JsValue {
+ text.and_then(|t| json_parse(&t).ok())
+ .unwrap_or(JsValue::Null)
+}
+
+/// Postgres team ids are int4 and start at 1 (`isValidTeamId`).
+fn valid_team_id(raw: &str) -> Option<i32> {
+ raw.parse::<i32>().ok().filter(|n| *n >= 1)
+}
+
+/// A token lookup must never reach a team by id, so the two kinds of key never share a name.
+fn id_key(team_id: impl std::fmt::Display) -> String {
+ format!("id:{team_id}")
+}
+
+fn token_key(token: &str) -> String {
+ format!("token:{token}")
+}
+
+#[derive(Debug, Clone)]
+pub struct CachedTeam {
+ pub team: Team,
+ pub ingested_event: bool,
+}
+
+/// Without a `writer` it never writes `ingested_event`.
+pub struct PgTeamManager {
+ loader: LazyLoader<CachedTeam>,
+ writer: Option<PgPool>,
+}
+
+impl PgTeamManager {
+ pub fn new(reader: PgPool, writer: Option<PgPool>) -> Self {
+ let mut options = LazyLoaderOptions::new("TeamManager", Duration::from_secs(120));
+ options.jitter = Some(Duration::from_secs(30));
+ let loader = LazyLoader::new(
+ options,
+ Arc::new(move |keys: Vec<String>| {
+ let pool = reader.clone();
+ async move { fetch_teams(&pool, keys).await }.boxed()
+ }),
+ );
+ Self { loader, writer }
+ }
+
+ async fn cached(&self, key: &str) -> anyhow::Result<Option<CachedTeam>> {
+ self.loader.get(key).await
+ }
+
+ async fn set_ingested_event(&self, writer: &PgPool, team: &CachedTeam) -> anyhow::Result<()> {
+ if team.ingested_event {
+ return Ok(());
+ }
+ sqlx::query(
+ "UPDATE posthog_team SET ingested_event = $1 WHERE id = $2 AND NOT ingested_event",
+ )
+ .bind(true)
+ .bind(team.team.id as i32)
+ .execute(writer)
+ .await
+ .map_err(classify_pg_error)?;
+ self.loader
+ .mark_for_refresh(&[id_key(team.team.id), token_key(&team.team.api_token)]);
+ Ok(())
+ }
+}
+
+async fn fetch_teams(
+ pool: &PgPool,
+ keys: Vec<String>,
+) -> anyhow::Result<HashMap<String, Option<CachedTeam>>> {
+ let mut ids: Vec<i32> = Vec::new();
+ let mut tokens: Vec<String> = Vec::new();
+ for key in &keys {
+ if let Some(raw) = key.strip_prefix("id:") {
+ ids.extend(valid_team_id(raw));
+ } else if let Some(token) = key.strip_prefix("token:") {
+ tokens.push(token.to_string());
+ }
+ }
+ let rows = sqlx::query(
+ "SELECT t.id, t.api_token, t.logs_settings::text AS logs_settings, t.ingested_event \
+ FROM posthog_team t JOIN posthog_organization o ON o.id = t.organization_id \
+ WHERE t.id = ANY($1) OR t.api_token = ANY($2)",
+ )
+ .bind(&ids)
+ .bind(&tokens)
+ .fetch_all(pool)
+ .await
+ .map_err(classify_pg_error)?;
+ let mut out: HashMap<String, Option<CachedTeam>> =
+ keys.into_iter().map(|k| (k, None)).collect();
+ for row in rows {
+ let id: i32 = row.try_get("id")?;
+ let api_token: String = row.try_get("api_token")?;
+ let settings = parse_json_column(row.try_get("logs_settings")?);
+ let team = CachedTeam {
+ team: Team {
+ id: id as i64,
+ api_token: api_token.clone(),
+ logs_settings: logs_settings(Some(&settings)),
+ },
+ ingested_event: row.try_get("ingested_event")?,
+ };
+ out.insert(id_key(id), Some(team.clone()));
+ out.insert(token_key(&api_token), Some(team));
+ }
+ Ok(out)
+}
+
+#[async_trait]
+impl TeamManager for PgTeamManager {
+ async fn get_team(&self, team_id: i64) -> anyhow::Result<Option<Team>> {
+ Ok(self.cached(&id_key(team_id)).await?.map(|t| t.team))
+ }
+
+ async fn get_team_by_token(&self, token: &str) -> anyhow::Result<Option<Team>> {
+ Ok(self.cached(&token_key(token)).await?.map(|t| t.team))
+ }
+
+ async fn set_teams_ingested_data(&self, team_ids: &[i64]) -> anyhow::Result<()> {
+ let Some(writer) = &self.writer else {
+ return Ok(());
+ };
+ let mut unique: Vec<i64> = team_ids.to_vec();
+ unique.sort_unstable();
+ unique.dedup();
+ futures::future::join_all(unique.into_iter().map(|team_id| async move {
+ let result = async {
+ if let Some(team) = self.cached(&id_key(team_id)).await? {
+ self.set_ingested_event(writer, &team).await?;
+ }
+ anyhow::Ok(())
+ }
+ .await;
+ if let Err(error) = result {
+ tracing::error!(team_id, error = %error, "Failed to set ingested_event");
+ }
+ }))
+ .await;
+ Ok(())
+ }
+}
+
+const RULES_REFRESH: Duration = Duration::from_secs(30);
+
+struct Timed<V> {
+ value: Arc<V>,
+ fetched_at: Instant,
+}
+
+/// Concurrent misses each query, as in Node.
+struct TtlCache<K, V>(Mutex<HashMap<K, Timed<V>>>);
+
+impl<K: std::hash::Hash + Eq + Clone, V> TtlCache<K, V> {
+ fn new() -> Self {
+ Self(Mutex::new(HashMap::new()))
+ }
+
+ fn fresh(&self, key: &K) -> Option<Arc<V>> {
+ let cache = self.0.lock().unwrap();
+ cache
+ .get(key)
+ .filter(|e| e.fetched_at.elapsed() < RULES_REFRESH)
+ .map(|e| e.value.clone())
+ }
+
+ fn stale(&self, key: &K) -> Option<Arc<V>> {
+ self.0.lock().unwrap().get(key).map(|e| e.value.clone())
+ }
+
+ fn put(&self, key: K, value: Arc<V>, fetched_at: Instant) {
+ self.0
+ .lock()
+ .unwrap()
+ .insert(key, Timed { value, fetched_at });
+ }
+}
+
+/// A fetch or compile error propagates.
+pub struct PgSamplingRules {
+ pool: PgPool,
+ cache: TtlCache<i64, CompiledRuleSet>,
+}
+
+impl PgSamplingRules {
+ pub fn new(pool: PgPool) -> Self {
+ Self {
+ pool,
+ cache: TtlCache::new(),
+ }
+ }
+}
+
+#[async_trait]
+impl SamplingRulesSource for PgSamplingRules {
+ async fn get_compiled_rule_set(&self, team_id: i64) -> anyhow::Result<Arc<CompiledRuleSet>> {
+ if let Some(v) = self.cache.fresh(&team_id) {
+ return Ok(v);
+ }
+ let started = Instant::now();
+ let rows = sqlx::query(
+ "SELECT id::text AS id, rule_type, scope_service, scope_path_pattern, \
+ scope_attribute_filters::text AS scope_attribute_filters, config::text AS config, version \
+ FROM logs_logsexclusionrule \
+ WHERE team_id = $1 AND enabled = true \
+ ORDER BY priority ASC, created_at ASC",
+ )
+ .bind(team_id)
+ .fetch_all(&self.pool)
+ .await
+ .map_err(classify_pg_error)?;
+ let rows: Vec<SamplingRuleRow> = rows
+ .into_iter()
+ .map(|r| -> anyhow::Result<SamplingRuleRow> {
+ let config = match parse_json_column(r.try_get("config")?) {
+ JsValue::Null => JsValue::Object(JsObject::new()),
+ v => v,
+ };
+ Ok(SamplingRuleRow {
+ id: r.try_get("id")?,
+ rule_type: r.try_get("rule_type")?,
+ scope_service: r.try_get("scope_service")?,
+ scope_path_pattern: r.try_get("scope_path_pattern")?,
+ scope_attribute_filters: parse_json_column(
+ r.try_get("scope_attribute_filters")?,
+ ),
+ config,
+ version: Some(r.try_get::<i32, _>("version")? as i64),
+ })
+ })
+ .collect::<anyhow::Result<_>>()?;
+ let compiled = Arc::new(compile_rule_set(&rows).map_err(anyhow::Error::new)?);
+ self.cache.put(team_id, compiled.clone(), started);
+ Ok(compiled)
+ }
+}
+
+/// A fetch error serves the last rules, or none.
+pub struct PgRetentionRules {
+ pool: PgPool,
+ cache: TtlCache<(i64, String), CompiledRetentionRuleSet>,
+}
+
+impl PgRetentionRules {
+ pub fn new(pool: PgPool) -> Self {
+ Self {
+ pool,
+ cache: TtlCache::new(),
+ }
+ }
+
+ async fn fetch(&self, team_id: i64, source: &str) -> anyhow::Result<Vec<RetentionRuleRow>> {
+ let table = match source {
+ "spans" => "tracing_tracesretentionrule",
+ _ => "logs_logsretentionrule",
+ };
+ let sql = format!(
+ "SELECT id::text AS id, config::text AS config, version FROM {table} \
+ WHERE team_id = $1 AND enabled = true \
+ ORDER BY priority ASC, created_at ASC LIMIT {MAX_ENABLED_RETENTION_RULES}"
+ );
+ let rows = sqlx::query(&sql)
+ .bind(team_id)
+ .fetch_all(&self.pool)
+ .await
+ .map_err(classify_pg_error)?;
+ rows.into_iter()
+ .map(|r| {
+ let config = match parse_json_column(r.try_get("config")?) {
+ JsValue::Null => JsValue::Object(JsObject::new()),
+ v => v,
+ };
+ Ok(RetentionRuleRow {
+ id: r.try_get("id")?,
+ config,
+ version: Some(r.try_get::<i32, _>("version")? as i64),
+ })
+ })
+ .collect()
+ }
+}
+
+#[async_trait]
+impl RetentionRulesSource for PgRetentionRules {
+ async fn get_compiled_rule_set(
+ &self,
+ team_id: i64,
+ source: &str,
+ ) -> anyhow::Result<Arc<CompiledRetentionRuleSet>> {
+ let key = (team_id, source.to_string());
+ if let Some(v) = self.cache.fresh(&key) {
+ return Ok(v);
+ }
+ let started = Instant::now();
+ let rows = match self.fetch(team_id, source).await {
+ Ok(rows) => rows,
+ Err(error) => {
+ tracing::warn!(team_id, source, error = %error, "[logs-retention] rules fetch failed — falling back");
+ return match self.cache.stale(&key) {
+ Some(v) => Ok(v),
+ None => Ok(Arc::new(
+ compile_retention_rule_set(&[]).map_err(anyhow::Error::new)?,
+ )),
+ };
+ }
+ };
+ let compiled = compile_retention_rule_set(&rows).map_err(anyhow::Error::new)?;
+ let dropped = rows.len().saturating_sub(compiled.rules.len());
+ if dropped > 0 {
+ let name = if source == "spans" {
+ "traces_ingestion_retention_rules_dropped_total"
+ } else {
+ "logs_ingestion_retention_rules_dropped_total"
+ };
+ let team: Arc<str> = Arc::from(team_id.to_string());
+ metrics::counter!(name, "team_id" => team).increment(dropped as u64);
+ tracing::warn!(
+ team_id,
+ source,
+ fetched = rows.len(),
+ dropped,
+ "[logs-retention] enabled rules discarded at compile — check retention_days tier"
+ );
+ }
+ let compiled = Arc::new(compiled);
+ self.cache.put(key, compiled.clone(), started);
+ Ok(compiled)
+ }
+}
+
+/// A fetch or compile error propagates, and the consumer fails open on it.
+pub struct PgMetricRules {
+ pool: PgPool,
+ cache: TtlCache<i64, Vec<CompiledMetricRule>>,
+}
+
+impl PgMetricRules {
+ pub fn new(pool: PgPool) -> Self {
+ Self {
+ pool,
+ cache: TtlCache::new(),
+ }
+ }
+}
+
+#[async_trait]
+impl MetricRulesSource for PgMetricRules {
+ async fn get_compiled_rules(
+ &self,
+ team_id: i64,
+ ) -> anyhow::Result<Arc<Vec<CompiledMetricRule>>> {
+ if let Some(v) = self.cache.fresh(&team_id) {
+ return Ok(v);
+ }
+ let started = Instant::now();
+ let rows = sqlx::query(
+ "SELECT id::text AS id, metric_name, filter_group::text AS filter_group, value_attribute, \
+ group_by::text[] AS group_by, source \
+ FROM logs_logsmetricrule \
+ WHERE team_id = $1 AND enabled = true \
+ ORDER BY created_at ASC",
+ )
+ .bind(team_id)
+ .fetch_all(&self.pool)
+ .await
+ .map_err(classify_pg_error)?;
+ let rows: Vec<MetricRuleRow> = rows
+ .into_iter()
+ .map(|r| -> anyhow::Result<MetricRuleRow> {
+ let opt_string =
+ |v: Option<String>| v.map(|s| JsValue::string(&s)).unwrap_or(JsValue::Null);
+ let mut row = JsObject::new();
+ row.insert_str("id", JsValue::string(&r.try_get::<String, _>("id")?));
+ row.insert_str("metric_name", opt_string(r.try_get("metric_name")?));
+ row.insert_str(
+ "filter_group",
+ parse_json_column(r.try_get("filter_group")?),
+ );
+ row.insert_str("value_attribute", opt_string(r.try_get("value_attribute")?));
+ let group_by: Vec<Option<String>> = r.try_get("group_by")?;
+ row.insert_str(
+ "group_by",
+ JsValue::Array(group_by.into_iter().map(opt_string).collect()),
+ );
+ row.insert_str("source", opt_string(r.try_get("source")?));
+ Ok(MetricRuleRow::from_js(&row))
+ })
+ .collect::<anyhow::Result<_>>()?;
+ let compiled = Arc::new(compile_metric_rules(&rows).map_err(anyhow::Error::new)?);
+ self.cache.put(team_id, compiled.clone(), started);
+ Ok(compiled)
+ }
+}
+
+type KeysLoad = Shared<BoxFuture<'static, Result<Arc<Vec<String>>, Arc<anyhow::Error>>>>;
+
+struct KeysEntry {
+ keys: Arc<Vec<String>>,
+ expires_at: Instant,
+}
+
+/// Only an outage with nothing cached propagates.
+pub struct PgPatternMessageKeys {
+ pool: PgPool,
+ cache: Arc<Mutex<HashMap<i64, KeysEntry>>>,
+ inflight: Arc<Mutex<HashMap<i64, KeysLoad>>>,
+}
+
+impl PgPatternMessageKeys {
+ pub fn new(pool: PgPool) -> Self {
+ Self {
+ pool,
+ cache: Default::default(),
+ inflight: Default::default(),
+ }
+ }
+}
+
+fn default_message_keys() -> Arc<Vec<String>> {
+ Arc::new(MESSAGE_KEYS.iter().map(|k| k.to_string()).collect())
+}
+
+async fn refresh_message_keys(
+ pool: PgPool,
+ cache: Arc<Mutex<HashMap<i64, KeysEntry>>>,
+ team_id: i64,
+) -> anyhow::Result<Arc<Vec<String>>> {
+ let existing = cache.lock().unwrap().get(&team_id).map(|e| e.keys.clone());
+ let fetched = sqlx::query("SELECT logs_pattern_message_keys::text[] AS keys FROM logs_teamlogsconfig WHERE team_id = $1")
+ .bind(team_id)
+ .fetch_optional(&pool)
+ .await
+ .map_err(classify_pg_error)
+ .and_then(|row| match row {
+ None => Ok(default_message_keys()),
+ Some(row) => {
+ let keys: Option<Vec<Option<String>>> = row.try_get("keys")?;
+ Ok(match keys.and_then(|k| k.into_iter().collect::<Option<Vec<String>>>()) {
+ Some(keys) => Arc::new(keys),
+ None => default_message_keys(),
+ })
+ }
+ });
+ match fetched {
+ Ok(keys) => {
+ cache.lock().unwrap().insert(
+ team_id,
+ KeysEntry {
+ keys: keys.clone(),
+ expires_at: Instant::now() + RULES_REFRESH,
+ },
+ );
+ Ok(keys)
+ }
+ Err(error) => {
+ if existing.is_none() && crate::pipeline::errors::is_dependency_unavailable(&error) {
+ return Err(error);
+ }
+ tracing::warn!(team_id, error = %error, "[logs-patterns] message keys fetch failed — falling back");
+ let fallback = existing.unwrap_or_else(default_message_keys);
+ cache.lock().unwrap().insert(
+ team_id,
+ KeysEntry {
+ keys: fallback.clone(),
+ expires_at: Instant::now() + Duration::from_secs(5),
+ },
+ );
+ Ok(fallback)
+ }
+ }
+}
+
+#[async_trait]
+impl PatternMessageKeysSource for PgPatternMessageKeys {
+ async fn get_message_keys(&self, team_id: i64) -> anyhow::Result<Vec<String>> {
+ if let Some(e) = self.cache.lock().unwrap().get(&team_id) {
+ if Instant::now() < e.expires_at {
+ return Ok(e.keys.to_vec());
+ }
+ }
+ let load = {
+ let mut inflight = self.inflight.lock().unwrap();
+ inflight
+ .entry(team_id)
+ .or_insert_with(|| {
+ let pool = self.pool.clone();
+ let cache = self.cache.clone();
+ let inflight = self.inflight.clone();
+ let task = tokio::spawn(async move {
+ let result = refresh_message_keys(pool, cache, team_id)
+ .await
+ .map_err(Arc::new);
+ inflight.lock().unwrap().remove(&team_id);
+ result
+ });
+ async move {
+ task.await.unwrap_or_else(|e| {
+ Err(Arc::new(anyhow::anyhow!("refresh task failed: {e}")))
+ })
+ }
+ .boxed()
+ .shared()
+ })
+ .clone()
+ };
+ match load.await {
+ Ok(keys) => Ok(keys.to_vec()),
+ Err(error) => Err(match error.downcast_ref::<DependencyUnavailableError>() {
+ Some(e) => anyhow::Error::new(e.clone()),
+ None => anyhow::anyhow!("{error:#}"),
+ }),
+ }
+ }
+}
+
+pub struct PgTracingConfig {
+ pool: PgPool,
+ cache: TtlCache<i64, f64>,
+}
+
+impl PgTracingConfig {
+ pub fn new(pool: PgPool) -> Self {
+ Self {
+ pool,
+ cache: TtlCache::new(),
+ }
+ }
+}
+
+#[async_trait]
+impl TracingConfigSource for PgTracingConfig {
+ async fn get_retention_days(&self, team_id: i64) -> anyhow::Result<f64> {
+ if let Some(v) = self.cache.fresh(&team_id) {
+ return Ok(*v);
+ }
+ let started = Instant::now();
+ let fetched =
+ sqlx::query("SELECT retention_days FROM tracing_teamtracingconfig WHERE team_id = $1")
+ .bind(team_id)
+ .fetch_optional(&self.pool)
+ .await
+ .map_err(classify_pg_error)
+ .and_then(|row| {
+ let stored: Option<i32> = match row {
+ Some(r) => r.try_get("retention_days")?,
+ None => None,
+ };
+ Ok(stored
+ .filter(|d| *d > 0)
+ .map(f64::from)
+ .unwrap_or(crate::pipeline::types::DEFAULT_TRACES_RETENTION_DAYS))
+ });
+ match fetched {
+ Ok(days) => {
+ self.cache.put(team_id, Arc::new(days), started);
+ Ok(days)
+ }
+ Err(error) => {
+ let existing = self.cache.stale(&team_id);
+ if existing.is_none() && crate::pipeline::errors::is_dependency_unavailable(&error)
+ {
+ return Err(error);
+ }
+ tracing::warn!(team_id, error = %error, "[traces-retention] tracing config fetch failed — falling back");
+ Ok(existing
+ .map(|v| *v)
+ .unwrap_or(crate::pipeline::types::DEFAULT_TRACES_RETENTION_DAYS))
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn only_valid_team_ids_are_queried() {
+ assert_eq!(valid_team_id("42"), Some(42));
+ assert_eq!(valid_team_id("0"), None);
+ assert_eq!(valid_team_id("9999999999"), None);
+ assert_eq!(valid_team_id("phc_abc"), None);
+ }
+
+ #[test]
+ fn connection_failures_are_dependency_unavailable() {
+ let refused = classify_pg_error(sqlx::Error::Io(std::io::Error::new(
+ std::io::ErrorKind::ConnectionRefused,
+ "refused",
+ )));
+ assert!(crate::pipeline::errors::is_dependency_unavailable(&refused));
+ let bad = classify_pg_error(sqlx::Error::RowNotFound);
+ assert!(!crate::pipeline::errors::is_dependency_unavailable(&bad));
+ }
+}
diff --git a/rust/logs-ingestion/src/service/redis_deps.rs b/rust/logs-ingestion/src/service/redis_deps.rs
new file mode 100644
index 000000000000..85a0c55a880b
--- /dev/null
+++ b/rust/logs-ingestion/src/service/redis_deps.rs
@@ -0,0 +1,230 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use futures::FutureExt;
+use redis::aio::{ConnectionLike, MultiplexedConnection};
+use redis::{
+ AsyncConnectionConfig, Client, Cmd, ConnectionAddr, ConnectionInfo, Pipeline, RedisFuture,
+ Value,
+};
+use tokio::sync::Mutex;
+
+use crate::pipeline::QuotaLimiting;
+use crate::rate_limiter::Clock;
+
+use super::config::RedisTarget;
+use super::lazy_loader::{LazyLoader, LazyLoaderOptions};
+
+/// The port, password and TLS options win over the URL, as ioredis merges its options last.
+pub fn connection_info(target: &RedisTarget) -> anyhow::Result<ConnectionInfo> {
+ use redis::IntoConnectionInfo;
+ let url = if target.url_or_host.contains("://") {
+ target.url_or_host.clone()
+ } else {
+ format!("redis://{}", target.url_or_host)
+ };
+ let mut info = url.as_str().into_connection_info()?;
+ let (host, port) = match &info.addr {
+ ConnectionAddr::Tcp(h, p)
+ | ConnectionAddr::TcpTls {
+ host: h, port: p, ..
+ } => (h.clone(), *p),
+ other => anyhow::bail!("unsupported Redis address {other:?}"),
+ };
+ let port = target.port.unwrap_or(port);
+ let tls = target.tls || matches!(info.addr, ConnectionAddr::TcpTls { .. });
+ info.addr = if tls {
+ ConnectionAddr::TcpTls {
+ host,
+ port,
+ insecure: false,
+ tls_params: None,
+ }
+ } else {
+ ConnectionAddr::Tcp(host, port)
+ };
+ if let Some(password) = &target.password {
+ info.redis.password = Some(password.clone());
+ }
+ Ok(info)
+}
+
+#[derive(Clone)]
+pub struct ReconnectingRedis {
+ client: Client,
+ conn: Arc<Mutex<Option<MultiplexedConnection>>>,
+}
+
+impl ReconnectingRedis {
+ pub fn new(info: ConnectionInfo) -> anyhow::Result<Self> {
+ Ok(Self {
+ client: Client::open(info)?,
+ conn: Arc::new(Mutex::new(None)),
+ })
+ }
+
+ async fn connection(&self) -> redis::RedisResult<MultiplexedConnection> {
+ let mut guard = self.conn.lock().await;
+ if let Some(c) = guard.as_ref() {
+ return Ok(c.clone());
+ }
+ let config = AsyncConnectionConfig::new()
+ .set_connection_timeout(Duration::from_secs(5))
+ .set_response_timeout(Duration::from_secs(5));
+ let c = self
+ .client
+ .get_multiplexed_async_connection_with_config(&config)
+ .await?;
+ *guard = Some(c.clone());
+ Ok(c)
+ }
+
+ async fn on_error(&self, error: &redis::RedisError) {
+ if error.is_io_error()
+ || error.is_connection_dropped()
+ || error.is_timeout()
+ || error.is_unrecoverable_error()
+ {
+ *self.conn.lock().await = None;
+ }
+ }
+}
+
+impl ConnectionLike for ReconnectingRedis {
+ fn req_packed_command<'a>(&'a mut self, cmd: &'a Cmd) -> RedisFuture<'a, Value> {
+ async move {
+ let mut c = self.connection().await?;
+ let result = c.req_packed_command(cmd).await;
+ if let Err(e) = &result {
+ self.on_error(e).await;
+ }
+ result
+ }
+ .boxed()
+ }
+
+ fn req_packed_commands<'a>(
+ &'a mut self,
+ cmd: &'a Pipeline,
+ offset: usize,
+ count: usize,
+ ) -> RedisFuture<'a, Vec<Value>> {
+ async move {
+ let mut c = self.connection().await?;
+ let result = c.req_packed_commands(cmd, offset, count).await;
+ if let Err(e) = &result {
+ self.on_error(e).await;
+ }
+ result
+ }
+ .boxed()
+ }
+
+ fn get_db(&self) -> i64 {
+ 0
+ }
+}
+
+pub const QUOTA_LIMITER_CACHE_KEY: &str = "@posthog/quota-limits/";
+
+/// Cached for an hour and refreshed in the background after a minute. A Redis error reads as no limited tokens.
+pub struct RedisQuotaLimiting {
+ // Shared, so a cache hit clones a pointer instead of every limited token.
+ loader: LazyLoader<Arc<HashMap<String, f64>>>,
+ clock: Clock,
+}
+
+impl RedisQuotaLimiting {
+ pub fn new(redis: ReconnectingRedis, clock: Clock) -> Self {
+ let mut options = LazyLoaderOptions::new("quota_limited_tokens", Duration::from_secs(3600));
+ options.background_age = Some(Duration::from_secs(60));
+ let load_clock = clock.clone();
+ let loader = LazyLoader::new(
+ options,
+ Arc::new(move |resources: Vec<String>| {
+ let mut redis = redis.clone();
+ let clock = load_clock.clone();
+ async move {
+ let now_seconds = clock() / 1000.0;
+ let mut out = HashMap::new();
+ for resource in resources {
+ let key = format!("{QUOTA_LIMITER_CACHE_KEY}{resource}");
+ let reply: redis::RedisResult<Vec<(String, String)>> = redis::cmd("ZRANGEBYSCORE")
+ .arg(&key)
+ .arg(crate::js::number_to_string(now_seconds))
+ .arg("+inf")
+ .arg("WITHSCORES")
+ .query_async(&mut redis)
+ .await;
+ let tokens = match reply {
+ Ok(pairs) => pairs
+ .into_iter()
+ .filter(|(token, score)| !token.is_empty() && !score.is_empty())
+ .map(|(token, score)| {
+ let ms = (crate::metric_rules::js_parse_float(
+ &crate::js::JsString::from(score),
+ ) * 1000.0)
+ .floor();
+ (token, ms)
+ })
+ .collect(),
+ Err(error) => {
+ tracing::error!(resource, error = %error, "[QuotaLimiting] Error loading limited tokens from Redis");
+ HashMap::new()
+ }
+ };
+ out.insert(resource, Some(Arc::new(tokens)));
+ }
+ Ok(out)
+ }
+ .boxed()
+ }),
+ );
+ Self { loader, clock }
+ }
+}
+
+#[async_trait]
+impl QuotaLimiting for RedisQuotaLimiting {
+ async fn is_team_token_quota_limited(
+ &self,
+ token: &str,
+ resource: &str,
+ ) -> anyhow::Result<bool> {
+ let now = (self.clock)();
+ let limited = self.loader.get(resource).await?;
+ Ok(limited
+ .and_then(|m| m.get(token).copied())
+ .is_some_and(|until| until != 0.0 && !until.is_nan() && until > now))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn redis_targets_read_like_ioredis() {
+ let info = connection_info(&RedisTarget {
+ url_or_host: "logs-redis.internal".into(),
+ port: Some(6380),
+ password: None,
+ tls: true,
+ })
+ .unwrap();
+ assert!(
+ matches!(info.addr, ConnectionAddr::TcpTls { ref host, port: 6380, .. } if host == "logs-redis.internal")
+ );
+ let info = connection_info(&RedisTarget {
+ url_or_host: "redis://127.0.0.1".into(),
+ port: None,
+ password: Some("pw".into()),
+ tls: false,
+ })
+ .unwrap();
+ assert!(matches!(info.addr, ConnectionAddr::Tcp(_, 6379)));
+ assert_eq!(info.redis.password.as_deref(), Some("pw"));
+ }
+}
diff --git a/rust/logs-ingestion/src/service/shadow.rs b/rust/logs-ingestion/src/service/shadow.rs
new file mode 100644
index 000000000000..49da2fa0335a
--- /dev/null
+++ b/rust/logs-ingestion/src/service/shadow.rs
@@ -0,0 +1,85 @@
+//! The wrappers that keep shadow mode free of billing and customer-visible effects.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+
+use crate::js::JsValue;
+use crate::metric_rules::{MetricsEmitter, OtlpTransport};
+use crate::pipeline::{Team, TeamManager, UsageRecordBatch};
+
+pub const SUPPRESSED_COUNTER: &str = "logs_ingestion_shadow_suppressed_total";
+
+fn suppressed(effect: &'static str, n: u64) {
+ metrics::counter!(SUPPRESSED_COUNTER, "effect" => effect).increment(n);
+}
+
+pub struct ReadOnlyTeamManager(pub Arc<dyn TeamManager>);
+
+#[async_trait]
+impl TeamManager for ReadOnlyTeamManager {
+ async fn get_team(&self, team_id: i64) -> anyhow::Result<Option<Team>> {
+ self.0.get_team(team_id).await
+ }
+ async fn get_team_by_token(&self, token: &str) -> anyhow::Result<Option<Team>> {
+ self.0.get_team_by_token(token).await
+ }
+ async fn set_teams_ingested_data(&self, team_ids: &[i64]) -> anyhow::Result<()> {
+ suppressed("set_teams_ingested_data", team_ids.len() as u64);
+ Ok(())
+ }
+}
+
+/// The shadow mints its own record ids, so a record it sent would bill twice.
+pub struct DiscardUsageRecords;
+
+#[async_trait]
+impl UsageRecordBatch for DiscardUsageRecords {
+ fn add(&self, _team_id: i64, _usage_key: &str, _record_id: &str, _quantity: f64, _unit: &str) {
+ suppressed("usage_record", 1);
+ }
+ async fn flush(&self) -> anyhow::Result<()> {
+ Ok(())
+ }
+}
+
+/// Present rather than absent: its presence decides whether a buffer is decoded.
+pub struct SuppressedMetricsEmitter;
+
+#[async_trait]
+impl MetricsEmitter for SuppressedMetricsEmitter {
+ async fn emit(&self, _token: &str, _team_id: i64, _payload: &JsValue) -> bool {
+ suppressed("metric_rules_otlp", 1);
+ true
+ }
+}
+
+/// `internalFetch` reaches capture-logs without a proxy, so this client opts out instead of a process-wide `NO_PROXY` entry.
+pub struct ReqwestOtlpTransport {
+ client: reqwest::Client,
+}
+
+impl ReqwestOtlpTransport {
+ pub fn new() -> anyhow::Result<Self> {
+ Ok(Self {
+ client: reqwest::Client::builder()
+ .no_proxy()
+ .timeout(std::time::Duration::from_millis(3000))
+ .build()?,
+ })
+ }
+}
+
+#[async_trait]
+impl OtlpTransport for ReqwestOtlpTransport {
+ async fn post(&self, url: &str, headers: &[(&str, String)], body: &str) -> anyhow::Result<u16> {
+ let mut request = self.client.post(url).body(body.to_string());
+ for (k, v) in headers {
+ request = request.header(*k, v);
+ }
+ let response = request.send().await?;
+ let status = response.status().as_u16();
+ response.bytes().await?;
+ Ok(status)
+ }
+}
diff --git a/rust/logs-ingestion/src/service/usage.rs b/rust/logs-ingestion/src/service/usage.rs
new file mode 100644
index 000000000000..d3be7283fe9a
--- /dev/null
+++ b/rust/logs-ingestion/src/service/usage.rs
@@ -0,0 +1,274 @@
+//! Records dedupe by (team, usage key, record id), retry on transient codes, and never fail the batch.
+
+use std::collections::HashSet;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use async_trait::async_trait;
+use tonic::transport::Channel;
+use tonic::Code;
+use usage_ingestion_proto::usage_ingestion::v1::usage_ingestion_client::UsageIngestionClient;
+use usage_ingestion_proto::usage_ingestion::v1::{BillingUsageRecord, IngestBillingUsageRequest};
+
+use crate::pipeline::UsageRecordBatch;
+use crate::property_filter::coerce::js_parse_int;
+use crate::rate_limiter::Clock;
+
+use super::config::UsageConfig;
+
+const MAX_ATTEMPTS: u32 = 3;
+const RETRY_BACKOFF_MS: u64 = 100;
+
+#[derive(Debug, Clone)]
+struct Pending {
+ team_id: i64,
+ usage_key: String,
+ record_id: String,
+ quantity: f64,
+ unit: String,
+}
+
+#[derive(Default)]
+struct Records {
+ keys: HashSet<String>,
+ list: Vec<Pending>,
+}
+
+pub struct GrpcUsageRecordBatch {
+ client: Option<UsageIngestionClient<Channel>>,
+ producer_id: Arc<str>,
+ report_teams: String,
+ max_batch_size: usize,
+ timeout: Duration,
+ records: Mutex<Records>,
+ clock: Clock,
+}
+
+/// Only an exact `*` matches all, and each item is `parseInt(item)` with no radix.
+fn report_teams_match(raw: &str, team_id: i64) -> bool {
+ if raw.trim().is_empty() {
+ return false;
+ }
+ if raw == "*" {
+ return true;
+ }
+ raw.split(',')
+ .any(|item| js_parse_int(item, None) == team_id as f64)
+}
+
+impl GrpcUsageRecordBatch {
+ /// No address or no report teams means no client.
+ pub fn new(config: &UsageConfig, producer_id: &str, clock: Clock) -> anyhow::Result<Self> {
+ let client = if config.addr.is_empty() || config.report_teams.is_empty() {
+ None
+ } else {
+ if config.tls {
+ anyhow::bail!(
+ "USAGE_INGESTION_TLS=true is not supported by the Rust logs consumer"
+ );
+ }
+ let channel = Channel::from_shared(format!("http://{}", config.addr))?
+ .timeout(config.timeout)
+ .connect_lazy();
+ Some(UsageIngestionClient::new(channel))
+ };
+ Ok(Self {
+ client,
+ producer_id: Arc::from(producer_id),
+ report_teams: config.report_teams.clone(),
+ max_batch_size: config.max_batch_size.max(1),
+ timeout: config.timeout,
+ records: Mutex::new(Records::default()),
+ clock,
+ })
+ }
+
+ fn accepts(&self, team_id: i64) -> bool {
+ self.client.is_some() && report_teams_match(&self.report_teams, team_id)
+ }
+
+ async fn ingest_chunk(
+ &self,
+ client: &UsageIngestionClient<Channel>,
+ chunk: &[BillingUsageRecord],
+ ) {
+ let mut attempt = 0;
+ loop {
+ let mut request = tonic::Request::new(IngestBillingUsageRequest {
+ records: chunk.to_vec(),
+ });
+ if let Ok(v) = self.producer_id.parse() {
+ request.metadata_mut().insert("x-client-name", v);
+ }
+ // connect-rpc's `defaultTimeoutMs` bounds the whole call, connecting included.
+ let mut client = client.clone();
+ let call = tokio::time::timeout(self.timeout, client.ingest_billing_usage(request));
+ let reply = match call.await {
+ Ok(reply) => reply,
+ Err(_) => Err(tonic::Status::deadline_exceeded(
+ "usage-ingestion call timed out",
+ )),
+ };
+ match reply {
+ Ok(response) => {
+ let accepted: HashSet<String> = response
+ .into_inner()
+ .accepted_record_ids
+ .into_iter()
+ .collect();
+ let mut rejected = 0;
+ for r in chunk {
+ let key: Arc<str> = Arc::from(r.usage_key.as_str());
+ if accepted.contains(&r.record_id) {
+ metrics::counter!("usage_ingestion_records_sent_total", "producer_id" => self.producer_id.clone(), "usage_key" => key).increment(1);
+ } else {
+ rejected += 1;
+ metrics::counter!("usage_ingestion_records_failed_total", "producer_id" => self.producer_id.clone(), "usage_key" => key, "error_code" => "rejected").increment(1);
+ }
+ }
+ if rejected > 0 {
+ tracing::warn!(producer_id = %self.producer_id, records = rejected, "usage-ingestion rejected usage records");
+ }
+ return;
+ }
+ Err(status) => {
+ let code = status.code();
+ let retryable = matches!(
+ code,
+ Code::Unavailable
+ | Code::DeadlineExceeded
+ | Code::ResourceExhausted
+ | Code::Aborted
+ | Code::Cancelled
+ | Code::Internal
+ );
+ let code_label: Arc<str> = Arc::from(format!("{code:?}"));
+ if attempt + 1 < MAX_ATTEMPTS && retryable {
+ metrics::counter!("usage_ingestion_retries_total", "producer_id" => self.producer_id.clone(), "error_code" => code_label).increment(1);
+ tokio::time::sleep(Duration::from_millis(
+ RETRY_BACKOFF_MS * 2u64.pow(attempt),
+ ))
+ .await;
+ attempt += 1;
+ continue;
+ }
+ for r in chunk {
+ let key: Arc<str> = Arc::from(r.usage_key.as_str());
+ metrics::counter!("usage_ingestion_records_failed_total", "producer_id" => self.producer_id.clone(), "usage_key" => key, "error_code" => code_label.clone()).increment(1);
+ }
+ tracing::warn!(producer_id = %self.producer_id, records = chunk.len(), error = %status, "failed to ingest usage records");
+ return;
+ }
+ }
+ }
+ }
+}
+
+#[async_trait]
+impl UsageRecordBatch for GrpcUsageRecordBatch {
+ fn add(&self, team_id: i64, usage_key: &str, record_id: &str, quantity: f64, unit: &str) {
+ if quantity <= 0.0 || !self.accepts(team_id) {
+ return;
+ }
+ let key = format!("{team_id}:{usage_key}:{record_id}");
+ let mut records = self.records.lock().unwrap();
+ if records.keys.insert(key) {
+ records.list.push(Pending {
+ team_id,
+ usage_key: usage_key.to_string(),
+ record_id: record_id.to_string(),
+ quantity,
+ unit: unit.to_string(),
+ });
+ }
+ }
+
+ async fn flush(&self) -> anyhow::Result<()> {
+ let pending = std::mem::take(&mut *self.records.lock().unwrap()).list;
+ let Some(client) = &self.client else {
+ return Ok(());
+ };
+ if pending.is_empty() {
+ return Ok(());
+ }
+ let flushed_at = (self.clock)() as i64;
+ let mut records = Vec::with_capacity(pending.len());
+ for p in pending {
+ // `BigInt(quantity)` throws on a fraction, which fails the whole flush.
+ if p.quantity.fract() != 0.0 || !p.quantity.is_finite() {
+ anyhow::bail!(
+ "The number {} cannot be converted to a BigInt because it is not an integer",
+ p.quantity
+ );
+ }
+ records.push(BillingUsageRecord {
+ team_id: p.team_id,
+ timestamp_ms: flushed_at,
+ producer_id: self.producer_id.to_string(),
+ usage_key: p.usage_key,
+ record_id: p.record_id,
+ quantity: p.quantity as i64,
+ unit: p.unit,
+ });
+ }
+ futures::future::join_all(
+ records
+ .chunks(self.max_batch_size)
+ .map(|chunk| self.ingest_chunk(client, chunk)),
+ )
+ .await;
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn report_teams_match_like_build_integer_matcher() {
+ for (raw, expected) in [
+ ("*", true),
+ (" * ", false),
+ ("*,5", true),
+ ("*,6", false),
+ ("3, 5", true),
+ ("0x5", true),
+ ("5abc", true),
+ ("", false),
+ (" ", false),
+ ("6", false),
+ ] {
+ assert_eq!(report_teams_match(raw, 5), expected, "{raw:?}");
+ }
+ }
+
+ #[tokio::test]
+ async fn a_usage_call_to_a_silent_server_ends_at_the_timeout() {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ tokio::spawn(async move {
+ let mut held = Vec::new();
+ while let Ok((socket, _)) = listener.accept().await {
+ held.push(socket);
+ }
+ });
+ let batch = GrpcUsageRecordBatch::new(
+ &UsageConfig {
+ addr: addr.to_string(),
+ tls: false,
+ timeout: Duration::from_millis(50),
+ max_batch_size: 10,
+ report_teams: "*".to_string(),
+ },
+ "logs",
+ crate::rate_limiter::system_clock(),
+ )
+ .unwrap();
+ batch.add(1, "logs_bytes", "record-1", 5.0, "bytes");
+ tokio::time::timeout(Duration::from_secs(5), batch.flush())
+ .await
+ .expect("each attempt must end at its timeout")
+ .unwrap();
+ }
+}
diff --git a/rust/logs-ingestion/tests/avro_parity.rs b/rust/logs-ingestion/tests/avro_parity.rs
index f3443aee6fad..213035d64dd6 100644
--- a/rust/logs-ingestion/tests/avro_parity.rs
+++ b/rust/logs-ingestion/tests/avro_parity.rs
@@ -158,6 +158,12 @@ fn a_deep_chain_of_named_records_is_rejected_without_overflow() {
20_000,
));
assert!(Schema::parse_json(&format!("[{}]", redefined.join(","))).is_err());
+
+ // A shallow redefinition passes the depth check, so only the duplicate name stops it, as in avsc.
+ let duplicate =
+ r#"[{"type":"record","name":"R","fields":[]},{"type":"record","name":"R","fields":[]}]"#;
+ let err = Schema::parse_json(duplicate).unwrap_err().to_string();
+ assert!(err.contains("duplicate type name: R"), "{err}");
}
#[test]
@@ -269,6 +275,22 @@ fn empty_array_items_share_one_budget_per_container() {
assert!(decode_container(&zstd_container(schema, &[block(4_000_000_000)])).is_err());
// Each block stays under the cap, and the two together pass it.
assert!(decode_container(&zstd_container(schema, &[block(600_000), block(600_000)])).is_err());
+
+ // Each record holds two of the one before, so 31 levels expand one record into 2^31 empty values.
+ let mut records = vec![r#"{"type":"record","name":"R0","fields":[]}"#.to_string()];
+ for i in 1..=30 {
+ records.push(format!(
+ r#"{{"type":"record","name":"R{i}","fields":[{{"name":"a","type":"R{p}"}},{{"name":"b","type":"R{p}"}}]}}"#,
+ p = i - 1
+ ));
+ }
+ let doubling = format!(
+ r#"{{"type":"record","name":"Top","fields":[{{"name":"u","type":[{}]}}]}}"#,
+ records.join(",")
+ );
+ let last_branch = encode_long(30);
+ let block = (1, zstd::bulk::compress(&last_branch, 1).unwrap());
+ assert!(decode_container(&zstd_container(doubling.as_bytes(), &[block])).is_err());
}
#[test]
diff --git a/rust/logs-ingestion/tests/common/mod.rs b/rust/logs-ingestion/tests/common/mod.rs
index 0d7cdf1b27c7..a4b2c2abf0ec 100644
--- a/rust/logs-ingestion/tests/common/mod.rs
+++ b/rust/logs-ingestion/tests/common/mod.rs
@@ -17,8 +17,11 @@ pub fn modules_dir() -> PathBuf {
repo_path("nodejs/src/logs/parity/fixtures/modules")
}
+/// `PARITY_SCENARIOS_DIR` points the corpus test at other fixtures, such as recorded bench workloads.
pub fn scenarios_dir() -> PathBuf {
- repo_path("nodejs/src/logs/parity/fixtures/scenarios")
+ std::env::var_os("PARITY_SCENARIOS_DIR")
+ .map(PathBuf::from)
+ .unwrap_or_else(|| repo_path("nodejs/src/logs/parity/fixtures/scenarios"))
}
pub fn load_json(path: &PathBuf) -> JsValue {
@@ -108,3 +111,67 @@ pub fn canonical(value: &AvroValue) -> JsValue {
pub fn canonical_text(records: &[AvroValue]) -> String {
json_stringify(&JsValue::Array(records.iter().map(canonical).collect()))
}
+
+pub fn avro_from_canonical(
+ schema: &logs_ingestion::avro::Schema,
+ value: &JsValue,
+) -> logs_ingestion::avro::AvroValue {
+ use logs_ingestion::avro::{AvroMap, AvroValue, Schema};
+ use std::sync::Arc;
+ let field = |v: &JsValue, k: &str| match v {
+ JsValue::Object(o) => o.get(k).cloned().unwrap_or(JsValue::Null),
+ _ => JsValue::Null,
+ };
+ match (schema, value) {
+ (Schema::Union(branches), v) => {
+ let pick = branches.iter().find(|b| {
+ matches!(
+ (b, v),
+ (Schema::Null, JsValue::Null)
+ | (Schema::Boolean, JsValue::Bool(_))
+ | (Schema::String | Schema::Enum(_), JsValue::String(_))
+ | (
+ Schema::Int | Schema::Long | Schema::Float | Schema::Double,
+ JsValue::Number(_)
+ )
+ | (Schema::Array(_), JsValue::Array(_))
+ | (
+ Schema::Bytes | Schema::Fixed(_) | Schema::Map(_) | Schema::Record(_),
+ JsValue::Object(_)
+ )
+ )
+ });
+ avro_from_canonical(pick.unwrap_or_else(|| panic!("no branch for {v:?}")), v)
+ }
+ (Schema::Null, _) => AvroValue::Null,
+ (Schema::Boolean, JsValue::Bool(b)) => AvroValue::Boolean(*b),
+ (Schema::Int, JsValue::Number(n)) => AvroValue::Int(*n as i64),
+ (Schema::Long, JsValue::Number(n)) => AvroValue::Long(*n as i64),
+ (Schema::Float, JsValue::Number(n)) => AvroValue::Float(*n),
+ (Schema::Double, JsValue::Number(n)) => AvroValue::Double(*n),
+ (Schema::String, JsValue::String(s)) => AvroValue::String(s.to_string()),
+ (Schema::Bytes, v) => AvroValue::Bytes(hex::decode(string(&field(v, "$bytes"))).unwrap()),
+ (Schema::Array(items), JsValue::Array(a)) => {
+ AvroValue::Array(a.iter().map(|v| avro_from_canonical(items, v)).collect())
+ }
+ (Schema::Map(values), JsValue::Object(o)) => {
+ let mut map = AvroMap::new();
+ for (k, v) in o.iter() {
+ map.assign(k.to_string(), avro_from_canonical(values, v));
+ }
+ AvroValue::Map(map)
+ }
+ (Schema::Record(r), v) => AvroValue::Record(
+ r.fields
+ .iter()
+ .map(|f| {
+ (
+ Arc::from(&*f.name),
+ avro_from_canonical(&f.schema, &field(v, &f.name)),
+ )
+ })
+ .collect(),
+ ),
+ (s, v) => panic!("cannot rebuild {v:?} as {s:?}"),
+ }
+}
diff --git a/rust/logs-ingestion/tests/parity_corpus.rs b/rust/logs-ingestion/tests/parity_corpus.rs
index c2ee5730174c..dc93611a659e 100644
--- a/rust/logs-ingestion/tests/parity_corpus.rs
+++ b/rust/logs-ingestion/tests/parity_corpus.rs
@@ -13,7 +13,8 @@ use logs_ingestion::pipeline::scenario::{
use logs_ingestion::rate_limiter::{RedisTokenBucket, TokenBucketRedis};
use sha2::{Digest, Sha256};
-const REDIS_URL: &str = "redis://127.0.0.1:6379";
+// Database 9 keeps the FLUSHDB away from the dev stack, which uses database 0.
+const REDIS_URL: &str = "redis://127.0.0.1:6379/9";
fn obj(entries: Vec<(&str, JsValue)>) -> JsValue {
let mut o = JsObject::new();
diff --git a/rust/logs-ingestion/tests/service_kafka.rs b/rust/logs-ingestion/tests/service_kafka.rs
new file mode 100644
index 000000000000..ed10d674ac44
--- /dev/null
+++ b/rust/logs-ingestion/tests/service_kafka.rs
@@ -0,0 +1,604 @@
+mod common;
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use async_trait::async_trait;
+use common::{array, canonical, get, js_sort_cmp, load_json, scenarios_dir, string};
+use futures::FutureExt;
+use ingestion_framework::{Headers, IngestionOutputs, KafkaMessage, OutputMessage};
+use logs_ingestion::avro::decode_container;
+use logs_ingestion::js::{json_parse, json_stringify, JsObject, JsString, JsValue};
+use logs_ingestion::pipeline::scenario::{kafka_message, scenario_parts, PinnedClock};
+use logs_ingestion::pipeline::LogsIngestionConsumer;
+use logs_ingestion::rate_limiter::{RedisTokenBucket, TokenBucketRedis};
+use logs_ingestion::service::config::{ConsumerLoopConfig, ServiceConfig};
+use logs_ingestion::service::consumer_loop::{
+ run_consumer_loop, BackgroundTask, BatchProcessor, BatchSource,
+};
+use logs_ingestion::service::kafka_outputs::KafkaOutputs;
+use logs_ingestion::service::kafka_source::{recv_batch, KafkaBatchSource};
+use logs_ingestion::service::{create_consumer, create_producers, shadow_deps};
+use rdkafka::consumer::{Consumer, StreamConsumer};
+use rdkafka::message::{Header, OwnedHeaders};
+use rdkafka::mocking::MockCluster;
+use rdkafka::producer::{DefaultProducerContext, FutureProducer, FutureRecord};
+use rdkafka::{ClientConfig, Offset, TopicPartitionList};
+use tokio::sync::Notify;
+
+const INPUT: &str = "logs_ingestion";
+const SHADOW_LOGS: &str = "clickhouse_logs_rust_shadow";
+const SHADOW_DLQ: &str = "logs_ingestion_dlq_rust_shadow";
+const SHADOW_METRICS: &str = "clickhouse_app_metrics2_rust_shadow";
+const PRODUCTION: &[&str] = &[
+ "clickhouse_logs",
+ "logs_ingestion_dlq",
+ "clickhouse_app_metrics2",
+ "log_entries",
+];
+const GROUP: &str = "ingestion-logs-rust-shadow";
+
+fn env_for(bootstrap: &str) -> HashMap<String, String> {
+ [
+ ("PLUGIN_SERVER_MODE", "ingestion-logs"),
+ ("DATABASE_URL", "postgres://unused"),
+ ("DATABASE_READONLY_URL", "postgres://unused-replica"),
+ ("LOGS_REDIS_HOST", "127.0.0.1"),
+ ("KAFKA_CONSUMER_METADATA_BROKER_LIST", bootstrap),
+ ("KAFKA_CONSUMER_AUTO_OFFSET_RESET", "earliest"),
+ ("KAFKA_CONSUMER_ENABLE_PARTITION_EOF", "false"),
+ (
+ "KAFKA_WARPSTREAM_LOGS_PRODUCER_METADATA_BROKER_LIST",
+ bootstrap,
+ ),
+ (
+ "KAFKA_WARPSTREAM_INGESTION_PRODUCER_METADATA_BROKER_LIST",
+ bootstrap,
+ ),
+ ("KAFKA_WARPSTREAM_LOGS_PRODUCER_LINGER_MS", "0"),
+ ("KAFKA_WARPSTREAM_INGESTION_PRODUCER_LINGER_MS", "0"),
+ ]
+ .iter()
+ .map(|(k, v)| (k.to_string(), v.to_string()))
+ .collect()
+}
+
+fn mock_cluster() -> MockCluster<'static, DefaultProducerContext> {
+ let cluster = MockCluster::new(1).expect("mock cluster");
+ cluster.create_topic(INPUT, 4, 1).unwrap();
+ for t in [
+ SHADOW_LOGS,
+ SHADOW_DLQ,
+ SHADOW_METRICS,
+ "node_logs",
+ "node_dlq",
+ ]
+ .iter()
+ .chain(PRODUCTION)
+ {
+ cluster.create_topic(t, 1, 1).unwrap();
+ }
+ cluster
+}
+
+fn producer(bootstrap: &str) -> FutureProducer {
+ ClientConfig::new()
+ .set("bootstrap.servers", bootstrap)
+ .set("linger.ms", "0")
+ .create()
+ .unwrap()
+}
+
+async fn produce(
+ p: &FutureProducer,
+ topic: &str,
+ partition: Option<i32>,
+ m: &KafkaMessage,
+) -> (i32, i64) {
+ let mut headers = OwnedHeaders::new();
+ for (k, v) in &m.headers {
+ headers = headers.insert(Header {
+ key: k,
+ value: v.as_deref(),
+ });
+ }
+ let mut record: FutureRecord<'_, [u8], [u8]> = FutureRecord::to(topic).headers(headers);
+ if let Some(p) = partition {
+ record = record.partition(p);
+ }
+ if let Some(k) = &m.key {
+ record = record.key(k.as_slice());
+ }
+ if let Some(v) = &m.value {
+ record = record.payload(v.as_ref());
+ }
+ if let Some(ts) = m.timestamp {
+ record = record.timestamp(ts);
+ }
+ p.send(record, Duration::from_secs(5))
+ .await
+ .map_err(|(e, _)| e)
+ .unwrap()
+}
+
+async fn read_all(bootstrap: &str, topics: &[&str], expected: usize) -> Vec<KafkaMessage> {
+ let consumer: StreamConsumer = ClientConfig::new()
+ .set("bootstrap.servers", bootstrap)
+ .set("group.id", format!("reader-{}", uuid::Uuid::now_v7()))
+ .set("auto.offset.reset", "earliest")
+ .set("enable.auto.commit", "false")
+ .create()
+ .unwrap();
+ consumer.subscribe(topics).unwrap();
+ let deadline = Instant::now() + Duration::from_secs(40);
+ let mut out = Vec::new();
+ let mut grace: Option<Instant> = None;
+ while Instant::now() < deadline && grace.is_none_or(|g| Instant::now() < g) {
+ out.extend(
+ recv_batch(&consumer, 1000, Duration::from_millis(300))
+ .await
+ .unwrap(),
+ );
+ if out.len() >= expected && grace.is_none() {
+ grace = Some(Instant::now() + Duration::from_secs(1));
+ }
+ }
+ out
+}
+
+fn high_watermark(bootstrap: &str, topic: &str) -> i64 {
+ let consumer: StreamConsumer = ClientConfig::new()
+ .set("bootstrap.servers", bootstrap)
+ .set("group.id", "watermarks")
+ .create()
+ .unwrap();
+ consumer
+ .fetch_watermarks(topic, 0, Duration::from_secs(5))
+ .unwrap()
+ .1
+}
+
+fn committed(bootstrap: &str, partitions: i32) -> Vec<i64> {
+ let consumer: StreamConsumer = ClientConfig::new()
+ .set("bootstrap.servers", bootstrap)
+ .set("group.id", GROUP)
+ .create()
+ .unwrap();
+ let mut tpl = TopicPartitionList::new();
+ for p in 0..partitions {
+ tpl.add_partition(INPUT, p);
+ }
+ let c = consumer
+ .committed_offsets(tpl, Duration::from_secs(5))
+ .unwrap();
+ (0..partitions)
+ .map(|p| match c.find_partition(INPUT, p).unwrap().offset() {
+ Offset::Offset(o) => o,
+ _ => -1,
+ })
+ .collect()
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn kafka_outputs_route_by_output_name_and_wait_for_delivery() {
+ let cluster = mock_cluster();
+ let bootstrap = cluster.bootstrap_servers();
+ let config = ServiceConfig::from_env(&env_for(&bootstrap)).unwrap();
+ let outputs = KafkaOutputs::new(&config.routes, &create_producers(&config).unwrap()).unwrap();
+ let mut headers = Headers::new();
+ headers.set("token", "phc_x");
+ headers.set("team_id", "7");
+ let message = OutputMessage {
+ value: Some(b"payload".to_vec().into()),
+ key: Some(b"k".to_vec()),
+ headers,
+ };
+ outputs
+ .queue_messages("logs", vec![message.clone(), message])
+ .await
+ .unwrap();
+ outputs
+ .produce("app_metrics", OutputMessage::default())
+ .await
+ .unwrap();
+ assert!(outputs.queue_messages("log_entries", vec![]).await.is_err());
+
+ let read = read_all(&bootstrap, &[SHADOW_LOGS, SHADOW_METRICS], 3).await;
+ let logs: Vec<_> = read.iter().filter(|m| m.topic == SHADOW_LOGS).collect();
+ assert_eq!(logs.len(), 2);
+ assert_eq!(logs[0].value.as_deref(), Some(&b"payload"[..]));
+ assert_eq!(logs[0].key.as_deref(), Some(&b"k"[..]));
+ assert_eq!(
+ logs[0].headers,
+ vec![
+ ("token".into(), Some(b"phc_x".to_vec())),
+ ("team_id".into(), Some(b"7".to_vec()))
+ ]
+ );
+ assert_eq!(read.iter().filter(|m| m.topic == SHADOW_METRICS).count(), 1);
+ for t in PRODUCTION {
+ assert_eq!(high_watermark(&bootstrap, t), 0);
+ }
+
+ let mut missing = env_for(&bootstrap);
+ missing.insert("LOGS_INGESTION_CONSUMER_DLQ_TOPIC".into(), "missing".into());
+ let missing = ServiceConfig::from_env(&missing).unwrap();
+ // The second call fails too only if the first one did not create the topic.
+ for _ in 0..2 {
+ let error = match create_producers(&missing) {
+ Ok(_) => panic!("a missing shadow topic must fail startup"),
+ Err(e) => e.to_string(),
+ };
+ assert!(error.contains("missing_rust_shadow"), "{error}");
+ }
+}
+
+struct FailingProcessor {
+ fail_offset: i64,
+ batches: Arc<Notify>,
+}
+
+#[async_trait]
+impl BatchProcessor for FailingProcessor {
+ async fn process(&self, messages: Vec<KafkaMessage>) -> anyhow::Result<Option<BackgroundTask>> {
+ let fail = messages.iter().any(|m| m.offset == self.fail_offset);
+ self.batches.notify_one();
+ Ok(Some(
+ async move {
+ if fail {
+ anyhow::bail!("produce failed")
+ } else {
+ Ok(())
+ }
+ }
+ .boxed(),
+ ))
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_fatal_consumer_error_fails_the_next_batch() {
+ let cluster = mock_cluster();
+ let config = ServiceConfig::from_env(&env_for(&cluster.bootstrap_servers())).unwrap();
+ let mut source = KafkaBatchSource::new(create_consumer(&config).unwrap());
+ let reason = std::ffi::CString::new("test fatal error").unwrap();
+ // nosemgrep: rust.lang.security.unsafe-usage.unsafe-usage
+ unsafe {
+ rdkafka::bindings::rd_kafka_test_fatal_error(
+ source.consumer().client().native_ptr(),
+ rdkafka::bindings::rd_kafka_resp_err_t::RD_KAFKA_RESP_ERR__FENCED,
+ reason.as_ptr(),
+ );
+ }
+ let error = source
+ .next_batch(10, Duration::from_millis(200))
+ .await
+ .unwrap_err();
+ assert!(error.to_string().contains("fatal"), "{error}");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn offsets_commit_only_after_the_background_task_succeeds() {
+ let cluster = mock_cluster();
+ let bootstrap = cluster.bootstrap_servers();
+ let config = ServiceConfig::from_env(&env_for(&bootstrap)).unwrap();
+ let p = producer(&bootstrap);
+ for _ in 0..3 {
+ produce(
+ &p,
+ INPUT,
+ Some(0),
+ &KafkaMessage {
+ value: Some(b"x".to_vec().into()),
+ ..Default::default()
+ },
+ )
+ .await;
+ }
+ let mut source = KafkaBatchSource::new(create_consumer(&config).unwrap());
+ let processor = FailingProcessor {
+ fail_offset: 2,
+ batches: Arc::new(Notify::new()),
+ };
+ let loop_config = ConsumerLoopConfig {
+ batch_size: 2,
+ batch_timeout: Duration::from_secs(20),
+ max_background_tasks: 1,
+ background_task_timeout: Duration::from_secs(5),
+ };
+ let result = run_consumer_loop(
+ &mut source,
+ &processor,
+ &loop_config,
+ futures::future::pending(),
+ &|| {},
+ )
+ .await;
+ assert_eq!(result.unwrap_err().to_string(), "produce failed");
+ drop(source);
+ assert_eq!(
+ committed(&bootstrap, 1),
+ vec![2],
+ "the failed batch at offset 2 must replay"
+ );
+}
+
+struct OneBatch<'a> {
+ consumer: &'a LogsIngestionConsumer,
+ done: Arc<Notify>,
+}
+
+#[async_trait]
+impl BatchProcessor for OneBatch<'_> {
+ async fn process(&self, messages: Vec<KafkaMessage>) -> anyhow::Result<Option<BackgroundTask>> {
+ let task = self
+ .consumer
+ .process_kafka_batch(messages)
+ .await?
+ .background_task;
+ self.done.notify_one();
+ Ok(task)
+ }
+}
+
+struct Input {
+ message: KafkaMessage,
+ scenario_offset: i64,
+ actual_offset: i64,
+}
+
+fn sorted_texts(values: Vec<JsValue>) -> Vec<String> {
+ let mut t: Vec<String> = values.iter().map(json_stringify).collect();
+ t.sort_by(|a, b| js_sort_cmp(a, b));
+ t
+}
+
+fn obj(entries: Vec<(&str, JsValue)>) -> JsValue {
+ let mut o = JsObject::new();
+ for (k, v) in entries {
+ o.insert_str(k, v);
+ }
+ JsValue::Object(o)
+}
+
+/// The DLQ `source_offset` maps back from the mock cluster's offsets to the scenario's.
+fn canonical_output(output: &str, m: &KafkaMessage, inputs: &[Input]) -> JsValue {
+ let mut headers: Vec<(String, String)> = m
+ .headers
+ .iter()
+ .filter(|(k, _)| output != "logs" || k != "source_offset")
+ .map(|(k, v)| {
+ (
+ k.clone(),
+ String::from_utf8_lossy(v.as_deref().unwrap_or_default()).into_owned(),
+ )
+ })
+ .collect();
+ let partition = m.header("source_partition").unwrap_or_default();
+ for (k, v) in headers.iter_mut() {
+ if output == "logs_dlq" && k == "source_offset" {
+ let actual: i64 = v.parse().unwrap();
+ let input = inputs
+ .iter()
+ .find(|i| i.actual_offset == actual && i.message.partition.to_string() == partition)
+ .unwrap();
+ *v = input.scenario_offset.to_string();
+ }
+ }
+ headers.sort_by(|a, b| js_sort_cmp(&a.0, &b.0));
+ let mut h = JsObject::new();
+ for (k, v) in headers {
+ h.insert(JsString::from(k), JsValue::string(&v));
+ }
+ let bytes = m.value.as_deref().unwrap_or_default();
+ let value = if let Some(i) = inputs
+ .iter()
+ .position(|i| i.message.value.as_deref() == Some(bytes))
+ {
+ obj(vec![
+ ("kind", JsValue::string("passthrough")),
+ ("batch", JsValue::Number(0.0)),
+ ("index", JsValue::Number(i as f64)),
+ ])
+ } else if output == "app_metrics" {
+ let raw = String::from_utf8_lossy(bytes).into_owned();
+ let parsed = json_parse(&raw).unwrap();
+ let JsValue::Object(o) = &parsed else {
+ panic!()
+ };
+ let mut entries: Vec<_> = o.iter().map(|(k, v)| (k.to_string(), v.clone())).collect();
+ entries.sort_by(|a, b| js_sort_cmp(&a.0, &b.0));
+ let mut sorted = JsObject::new();
+ for (k, v) in entries {
+ sorted.insert(JsString::from(k), v);
+ }
+ obj(vec![
+ ("kind", JsValue::string("json")),
+ ("raw", JsValue::string(&raw)),
+ ("json", JsValue::Object(sorted)),
+ ])
+ } else {
+ let c = decode_container(bytes).unwrap();
+ obj(vec![
+ ("kind", JsValue::string("avro")),
+ ("schema", json_parse(&c.schema_json).unwrap()),
+ ("codec", JsValue::string(&c.codec)),
+ (
+ "records",
+ JsValue::Array(c.records.iter().map(canonical).collect()),
+ ),
+ ])
+ };
+ obj(vec![
+ ("batch", JsValue::Number(0.0)),
+ ("output", JsValue::string(output)),
+ (
+ "key",
+ m.key
+ .as_ref()
+ .map(|k| JsValue::string(&String::from_utf8_lossy(k)))
+ .unwrap_or(JsValue::Null),
+ ),
+ ("headers", JsValue::Object(h)),
+ ("value", value),
+ ])
+}
+
+async fn shadow_scenario(name: &str) {
+ let client = redis::Client::open("redis://127.0.0.1:6379").unwrap();
+ let mut conn = match client.get_multiplexed_async_connection().await {
+ Ok(c) => c,
+ Err(e) => {
+ eprintln!("skipping: no Redis: {e}");
+ return;
+ }
+ };
+ let keys: Vec<String> = redis::cmd("KEYS")
+ .arg("@posthog-logs-rust-shadow/*")
+ .query_async(&mut conn)
+ .await
+ .unwrap();
+ if !keys.is_empty() {
+ let _: () = redis::cmd("DEL")
+ .arg(&keys)
+ .query_async(&mut conn)
+ .await
+ .unwrap();
+ }
+ let redis: Arc<dyn TokenBucketRedis> = Arc::new(RedisTokenBucket::new(conn));
+
+ let fixture = load_json(&scenarios_dir().join(format!("{name}.json")));
+ let scenario = get(&fixture, "scenario");
+ let expected = get(&fixture, "expected");
+ assert_eq!(array(get(scenario, "batches")).len(), 1);
+
+ let cluster = mock_cluster();
+ let bootstrap = cluster.bootstrap_servers();
+ let config = ServiceConfig::from_env(&env_for(&bootstrap)).unwrap();
+ let p = producer(&bootstrap);
+ let mut inputs = Vec::new();
+ for m in array(&array(get(scenario, "batches"))[0]) {
+ let message = kafka_message(m);
+ let (_, actual_offset) = produce(&p, INPUT, Some(message.partition), &message).await;
+ inputs.push(Input {
+ scenario_offset: message.offset,
+ actual_offset,
+ message,
+ });
+ }
+
+ let clock = PinnedClock::new(match &array(get(scenario, "batch_now_ms"))[0] {
+ JsValue::Number(n) => *n,
+ _ => 0.0,
+ });
+ let mut parts = scenario_parts(scenario, redis, clock.clock(), false);
+ let outputs =
+ Arc::new(KafkaOutputs::new(&config.routes, &create_producers(&config).unwrap()).unwrap());
+ parts.deps.outputs = outputs.clone();
+ let consumer =
+ LogsIngestionConsumer::new(parts.identity, parts.config, shadow_deps(parts.deps));
+ let mut source = KafkaBatchSource::new(create_consumer(&config).unwrap());
+ let done = Arc::new(Notify::new());
+ let processor = OneBatch {
+ consumer: &consumer,
+ done: done.clone(),
+ };
+ let loop_config = ConsumerLoopConfig {
+ batch_size: inputs.len(),
+ batch_timeout: Duration::from_secs(30),
+ max_background_tasks: 1,
+ background_task_timeout: Duration::from_secs(30),
+ };
+ let stats = run_consumer_loop(
+ &mut source,
+ &processor,
+ &loop_config,
+ done.notified(),
+ &|| {},
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ (stats.batches, stats.messages),
+ (1, inputs.len() as u64),
+ "{name}: one Kafka batch"
+ );
+ drop(source);
+
+ let mut per_partition = [0i64; 4];
+ for i in &inputs {
+ per_partition[i.message.partition as usize] =
+ per_partition[i.message.partition as usize].max(i.actual_offset + 1);
+ }
+ let committed = committed(&bootstrap, 4);
+ for p in 0..4 {
+ if per_partition[p] > 0 {
+ assert_eq!(
+ committed[p], per_partition[p],
+ "{name}: partition {p} committed"
+ );
+ }
+ }
+
+ let expected_outputs = array(get(expected, "outputs"));
+ let read = read_all(
+ &bootstrap,
+ &[SHADOW_LOGS, SHADOW_DLQ, SHADOW_METRICS],
+ expected_outputs.len(),
+ )
+ .await;
+ for (output, topic) in [
+ ("logs", SHADOW_LOGS),
+ ("logs_dlq", SHADOW_DLQ),
+ ("app_metrics", SHADOW_METRICS),
+ ] {
+ let actual: Vec<&KafkaMessage> = read.iter().filter(|m| m.topic == topic).collect();
+ if output == "logs" {
+ for m in &actual {
+ let offset: i64 = m
+ .header("source_offset")
+ .expect("shadow logs carry source_offset")
+ .parse()
+ .unwrap();
+ let partition = m.header("source_partition").unwrap();
+ assert!(inputs
+ .iter()
+ .any(|i| i.actual_offset == offset
+ && i.message.partition.to_string() == partition));
+ }
+ }
+ let want: Vec<JsValue> = expected_outputs
+ .iter()
+ .filter(|o| string(get(o, "output")) == output)
+ .cloned()
+ .collect();
+ assert_eq!(
+ sorted_texts(
+ actual
+ .iter()
+ .map(|m| canonical_output(output, m, &inputs))
+ .collect()
+ ),
+ sorted_texts(want),
+ "{name}: {output}"
+ );
+ }
+ for t in PRODUCTION {
+ assert_eq!(
+ high_watermark(&bootstrap, t),
+ 0,
+ "{name}: nothing reaches {t}"
+ );
+ }
+ assert!(parts.recorded.usage_rows.lock().unwrap().is_empty());
+ assert!(parts.recorded.teams_ingested.lock().unwrap().is_empty());
+ assert!(parts.recorded.metric_emits.lock().unwrap().is_empty());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn shadow_mode_reproduces_the_corpus_on_shadow_topics_only() {
+ for name in ["passthrough", "parse-failures", "codecs"] {
+ shadow_scenario(name).await;
+ }
+}
diff --git a/rust/logs-ingestion/tests/service_postgres.rs b/rust/logs-ingestion/tests/service_postgres.rs
new file mode 100644
index 000000000000..2d3295d4a09d
--- /dev/null
+++ b/rust/logs-ingestion/tests/service_postgres.rs
@@ -0,0 +1,339 @@
+//! Each test writes its own organization, project and team and removes them. Skips without Postgres.
+
+use std::sync::Arc;
+
+use logs_ingestion::metric_rules::MetricRulesSource;
+use logs_ingestion::pipeline::errors::is_dependency_unavailable;
+use logs_ingestion::pipeline::{
+ PatternMessageKeysSource, RetentionRulesSource, SamplingRulesSource, TeamManager,
+ TracingConfigSource,
+};
+use logs_ingestion::service::postgres::{
+ PgMetricRules, PgPatternMessageKeys, PgRetentionRules, PgSamplingRules, PgTeamManager,
+ PgTracingConfig,
+};
+use sqlx::postgres::PgPoolOptions;
+use sqlx::PgPool;
+
+const DATABASE_URL: &str = "postgres://posthog:posthog@localhost:5432/test_posthog";
+
+struct SeededTeam {
+ pool: PgPool,
+ id: i32,
+ organization_id: uuid::Uuid,
+ token: String,
+}
+
+impl SeededTeam {
+ async fn create(pool: PgPool, logs_settings: &str) -> Self {
+ let mut tx = pool.begin().await.unwrap();
+ let id: i32 = sqlx::query_scalar("SELECT nextval('posthog_team_id_seq')::int")
+ .fetch_one(&mut *tx)
+ .await
+ .unwrap();
+ let organization_id = uuid::Uuid::now_v7();
+ let token = format!("phc_rust_logs_test_{id}");
+ sqlx::query(
+ r#"INSERT INTO posthog_organization
+ (id, name, slug, created_at, updated_at, plugins_access_level, for_internal_metrics, is_member_join_email_enabled, enforce_2fa, customer_id, available_product_features, personalization, setup_section_2_completed, domain_whitelist, members_can_use_personal_api_keys, allow_publicly_shared_resources, default_anonymize_ips)
+ VALUES ($1, 'Rust logs test', $2, now(), now(), 9, false, true, NULL, NULL, '{}', '{}', true, '{}', true, true, false)"#,
+ )
+ .bind(organization_id)
+ .bind(format!("rust-logs-test-{}", organization_id.simple()))
+ .execute(&mut *tx)
+ .await
+ .unwrap();
+ sqlx::query(
+ "INSERT INTO posthog_project (id, organization_id, name, created_at) VALUES ($1, $2, 'Rust logs test', now())",
+ )
+ .bind(id)
+ .bind(organization_id)
+ .execute(&mut *tx)
+ .await
+ .unwrap();
+ sqlx::query(
+ r#"INSERT INTO posthog_team
+ (id, uuid, organization_id, project_id, api_token, name, created_at, updated_at, app_urls, anonymize_ips, completed_snippet_onboarding, ingested_event, session_recording_opt_in, is_demo, access_control, test_account_filters, timezone, data_attributes, plugins_opt_in, opt_out_capture, event_names, event_names_with_usage, event_properties, event_properties_with_usage, event_properties_numerical, cookieless_server_hash_mode, base_currency, session_recording_retention_period, web_analytics_pre_aggregated_tables_enabled, logs_settings)
+ VALUES ($1, $2, $3, $1, $4, 'Rust logs test', now(), now(), '{}', false, false, false, false, false, false, '{}', 'UTC', '["data-attr"]', false, false, '[]', '[]', '[]', '[]', '[]', 0, 'USD', '30d', false, $5::jsonb)"#,
+ )
+ .bind(id)
+ .bind(uuid::Uuid::now_v7())
+ .bind(organization_id)
+ .bind(&token)
+ .bind(logs_settings)
+ .execute(&mut *tx)
+ .await
+ .unwrap();
+ tx.commit().await.unwrap();
+ Self {
+ pool,
+ id,
+ organization_id,
+ token,
+ }
+ }
+
+ async fn cleanup(&self) {
+ for table in [
+ "logs_logsexclusionrule",
+ "logs_logsretentionrule",
+ "tracing_tracesretentionrule",
+ "logs_logsmetricrule",
+ "logs_teamlogsconfig",
+ "tracing_teamtracingconfig",
+ ] {
+ sqlx::query(&format!("DELETE FROM {table} WHERE team_id = $1"))
+ .bind(self.id)
+ .execute(&self.pool)
+ .await
+ .unwrap();
+ }
+ for (sql, id) in [
+ ("DELETE FROM posthog_team WHERE id = $1", self.id),
+ ("DELETE FROM posthog_project WHERE id = $1", self.id),
+ ] {
+ sqlx::query(sql).bind(id).execute(&self.pool).await.unwrap();
+ }
+ sqlx::query("DELETE FROM posthog_organization WHERE id = $1")
+ .bind(self.organization_id)
+ .execute(&self.pool)
+ .await
+ .unwrap();
+ }
+}
+
+async fn pool() -> Option<PgPool> {
+ match PgPoolOptions::new()
+ .max_connections(4)
+ .connect(DATABASE_URL)
+ .await
+ {
+ Ok(p) => Some(p),
+ Err(e) => {
+ eprintln!("skipping: no Postgres at {DATABASE_URL}: {e}");
+ None
+ }
+ }
+}
+
+#[tokio::test]
+async fn team_manager_reads_teams_and_writes_ingested_event_once() {
+ let Some(pool) = pool().await else { return };
+ let team = SeededTeam::create(
+ pool.clone(),
+ r#"{"json_parse_logs": true, "retention_days": 30}"#,
+ )
+ .await;
+ let manager = PgTeamManager::new(pool.clone(), Some(pool.clone()));
+ let by_token = manager
+ .get_team_by_token(&team.token)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(by_token.id, team.id as i64);
+ let settings = by_token.logs_settings.clone().unwrap();
+ assert_eq!(settings.json_parse_logs, Some(true));
+ assert_eq!(settings.retention_days, Some(30.0));
+ assert_eq!(
+ manager.get_team(team.id as i64).await.unwrap(),
+ Some(by_token)
+ );
+ assert_eq!(
+ manager
+ .get_team_by_token("phc_rust_logs_no_such_token")
+ .await
+ .unwrap(),
+ None
+ );
+ assert_eq!(manager.get_team(0).await.unwrap(), None);
+ // The team is cached by id now, and its id must still not work as a token.
+ assert_eq!(
+ manager
+ .get_team_by_token(&team.id.to_string())
+ .await
+ .unwrap(),
+ None
+ );
+
+ let read_only = PgTeamManager::new(pool.clone(), None);
+ read_only
+ .set_teams_ingested_data(&[team.id as i64])
+ .await
+ .unwrap();
+ let ingested: bool =
+ sqlx::query_scalar("SELECT ingested_event FROM posthog_team WHERE id = $1")
+ .bind(team.id)
+ .fetch_one(&pool)
+ .await
+ .unwrap();
+ assert!(!ingested, "a manager without a writer must not write");
+ manager
+ .set_teams_ingested_data(&[team.id as i64, team.id as i64])
+ .await
+ .unwrap();
+ let ingested: bool =
+ sqlx::query_scalar("SELECT ingested_event FROM posthog_team WHERE id = $1")
+ .bind(team.id)
+ .fetch_one(&pool)
+ .await
+ .unwrap();
+ assert!(ingested);
+ team.cleanup().await;
+}
+
+#[tokio::test]
+async fn rule_caches_read_the_same_rows_as_node() {
+ let Some(pool) = pool().await else { return };
+ let team = SeededTeam::create(pool.clone(), "null").await;
+ let id = team.id;
+ for (priority, rule_type, enabled) in [
+ (2, "path_drop", true),
+ (1, "severity_sampling", true),
+ (0, "path_drop", false),
+ ] {
+ sqlx::query(
+ "INSERT INTO logs_logsexclusionrule (id, team_id, created_at, name, enabled, priority, rule_type, \
+ scope_service, scope_attribute_filters, config, version) \
+ VALUES (gen_random_uuid(), $1, now(), 'r', $2, $3, $4, 'svc', '[]', $5::jsonb, 3)",
+ )
+ .bind(id)
+ .bind(enabled)
+ .bind(priority)
+ .bind(rule_type)
+ .bind(r#"{"actions": {"DEBUG": {"type": "drop"}}, "patterns": []}"#)
+ .execute(&pool)
+ .await
+ .unwrap();
+ }
+ let sampling = PgSamplingRules::new(pool.clone());
+ let set = sampling.get_compiled_rule_set(id as i64).await.unwrap();
+ assert_eq!(set.rules.len(), 2, "disabled rules are skipped");
+ assert!(
+ Arc::ptr_eq(
+ &set,
+ &sampling.get_compiled_rule_set(id as i64).await.unwrap()
+ ),
+ "cached for 30s"
+ );
+
+ sqlx::query(
+ "INSERT INTO logs_logsretentionrule (id, team_id, created_at, name, enabled, priority, config, version) \
+ VALUES (gen_random_uuid(), $1, now(), 'keep', true, 0, '{\"retention_days\": 30}', 1), \
+ (gen_random_uuid(), $1, now(), 'bad tier', true, 1, '{\"retention_days\": 31}', 1)",
+ )
+ .bind(id)
+ .execute(&pool)
+ .await
+ .unwrap();
+ let retention = PgRetentionRules::new(pool.clone());
+ let logs_rules = retention
+ .get_compiled_rule_set(id as i64, "logs")
+ .await
+ .unwrap();
+ assert_eq!(logs_rules.rules.len(), 1);
+ assert_eq!(logs_rules.rules[0].retention_days, 30);
+ assert!(retention
+ .get_compiled_rule_set(id as i64, "spans")
+ .await
+ .unwrap()
+ .rules
+ .is_empty());
+
+ sqlx::query(
+ "INSERT INTO logs_logsmetricrule (id, team_id, created_at, name, metric_name, enabled, filter_group, \
+ value_attribute, group_by, source, version) \
+ VALUES (gen_random_uuid(), $1, now(), 'm', 'errors_total', true, NULL, NULL, ARRAY['service_name'], 'logs', 1)",
+ )
+ .bind(id)
+ .execute(&pool)
+ .await
+ .unwrap();
+ let metric = PgMetricRules::new(pool.clone())
+ .get_compiled_rules(id as i64)
+ .await
+ .unwrap();
+ assert_eq!(metric.len(), 1);
+ assert_eq!(metric[0].metric_name.to_string(), "errors_total");
+ assert_eq!(
+ metric[0]
+ .group_by
+ .iter()
+ .map(|g| g.to_string())
+ .collect::<Vec<_>>(),
+ vec!["service_name"]
+ );
+
+ let keys = PgPatternMessageKeys::new(pool.clone());
+ assert_eq!(
+ keys.get_message_keys(id as i64).await.unwrap(),
+ vec!["message", "msg", "event"]
+ );
+ sqlx::query("INSERT INTO logs_teamlogsconfig (team_id, logs_pattern_message_keys) VALUES ($1, ARRAY['text'])")
+ .bind(id)
+ .execute(&pool)
+ .await
+ .unwrap();
+ assert_eq!(
+ PgPatternMessageKeys::new(pool.clone())
+ .get_message_keys(id as i64)
+ .await
+ .unwrap(),
+ vec!["text"]
+ );
+
+ let tracing_config = PgTracingConfig::new(pool.clone());
+ assert_eq!(
+ tracing_config.get_retention_days(id as i64).await.unwrap(),
+ 14.0
+ );
+ sqlx::query("INSERT INTO tracing_teamtracingconfig (team_id, retention_days) VALUES ($1, 90)")
+ .bind(id)
+ .execute(&pool)
+ .await
+ .unwrap();
+ assert_eq!(
+ PgTracingConfig::new(pool.clone())
+ .get_retention_days(id as i64)
+ .await
+ .unwrap(),
+ 90.0
+ );
+ team.cleanup().await;
+}
+
+#[tokio::test]
+async fn an_unreachable_database_is_dependency_unavailable_where_node_throws() {
+ let pool = PgPoolOptions::new()
+ .acquire_timeout(std::time::Duration::from_millis(500))
+ .connect_lazy("postgres://x:[email protected]:1/x")
+ .unwrap();
+ let teams = PgTeamManager::new(pool.clone(), None);
+ assert!(is_dependency_unavailable(
+ &teams.get_team_by_token("t").await.unwrap_err()
+ ));
+ assert!(is_dependency_unavailable(
+ &PgSamplingRules::new(pool.clone())
+ .get_compiled_rule_set(1)
+ .await
+ .unwrap_err()
+ ));
+ // Retention fails open; tracing config and pattern keys throw only with nothing cached.
+ assert!(PgRetentionRules::new(pool.clone())
+ .get_compiled_rule_set(1, "logs")
+ .await
+ .unwrap()
+ .rules
+ .is_empty());
+ assert!(is_dependency_unavailable(
+ &PgTracingConfig::new(pool.clone())
+ .get_retention_days(1)
+ .await
+ .unwrap_err()
+ ));
+ assert!(is_dependency_unavailable(
+ &PgPatternMessageKeys::new(pool.clone())
+ .get_message_keys(1)
+ .await
+ .unwrap_err()
+ ));
+}