:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): relay the changelog to clickhouse_person (#121249)
Zach Brown9ab1d055f023
diff --git a/.github/rust-images.yml b/.github/rust-images.ymlindex 73c4a758dca0..d8a7fc379c1c 100644--- a/.github/rust-images.yml+++ b/.github/rust-images.yml@@ -150,6 +150,10 @@dockerfile: ./rust/Dockerfileproject: hj9nd3cg9w+- image: personhog-changelog-relay+ dockerfile: ./rust/Dockerfile+ project: lvsrh6c64c+- image: personhog-test-harnessdockerfile: ./rust/Dockerfileproject: 6tl5lmb8l8diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 6198b40dedcc..ca095f155a7f 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -8931,6 +8931,34 @@ version = "0.1.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3637c05577168127568a64e9dc5a6887da720efef07b3d9472d45f63ab191166"+[[package]]+name = "personhog-changelog-relay"+version = "0.1.0"+dependencies = [+ "assignment-coordination",+ "axum 0.7.9",+ "chrono",+ "common-alloc",+ "common-kafka",+ "common-metrics",+ "envconfig",+ "futures",+ "lifecycle",+ "metrics",+ "metrics-util",+ "personhog-coordination",+ "personhog-proto",+ "prost 0.13.5",+ "rdkafka",+ "serde",+ "serde_json",+ "tokio",+ "tokio-util",+ "tracing",+ "tracing-subscriber",+ "uuid",+]+[[package]]name = "personhog-common"version = "0.1.0"diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex e269164773d9..45496c032fbf 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -62,6 +62,7 @@ members = ["ingestion-control-plane","ingestion-worker-proto","kafka-deduplicator",+ "personhog-changelog-relay","personhog-common","personhog-coordination","personhog-identity",diff --git a/rust/owners.yaml b/rust/owners.yamlindex db0987225df1..511de6aaa92d 100644--- a/rust/owners.yaml+++ b/rust/owners.yaml@@ -20,6 +20,7 @@ rules:- '/kafka-assigner-proto/'- '/kafka-assigner/'- '/kafka-deduplicator/'+ - '/personhog-changelog-relay/'- '/personhog-common/'- '/personhog-coordination/'- '/personhog-leader/'diff --git a/rust/personhog-changelog-relay/Cargo.toml b/rust/personhog-changelog-relay/Cargo.tomlnew file mode 100644index 000000000000..273d15895790--- /dev/null+++ b/rust/personhog-changelog-relay/Cargo.toml@@ -0,0 +1,35 @@+[package]+name = "personhog-changelog-relay"+version = "0.1.0"+edition = "2021"+publish = false++[dependencies]+personhog-proto = { path = "../personhog-proto" }+personhog-coordination = { path = "../personhog-coordination" }+assignment-coordination = { path = "../common/assignment-coordination" }+common-alloc = { path = "../common/alloc" }+common-kafka = { path = "../common/kafka" }+common-metrics = { path = "../common/metrics" }+lifecycle = { path = "../common/lifecycle" }++axum = { workspace = true }+chrono = { workspace = true }+envconfig = { workspace = true }+futures = { workspace = true }+metrics = { workspace = true }+prost = { workspace = true }+rdkafka = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+tokio = { workspace = true }+tokio-util = { workspace = true }+tracing = { workspace = true }+tracing-subscriber = { workspace = true }+uuid = { workspace = true }++[dev-dependencies]+metrics-util = { workspace = true }++[lints]+workspace = truediff --git a/rust/personhog-changelog-relay/README.md b/rust/personhog-changelog-relay/README.mdnew file mode 100644index 000000000000..4b41ae41b77d--- /dev/null+++ b/rust/personhog-changelog-relay/README.md@@ -0,0 +1,38 @@+# personhog-changelog-relay++Relays the personhog changelog to the ClickHouse person topic. It consumes `personhog.types.v1.Person` records from the changelog on MSK and produces each as the JSON row Node's `generateKafkaPersonUpdateMessage` writes, unkeyed and without headers, to `OUTPUT_TOPIC` (`clickhouse_person`) on the output cluster.++## Following the changelog++`config/changelog` must list the relay's group in `retire_groups`, added from the control plane's changelog panel, or the relay refuses to start. Retire then waits for this group as well as the writer's to read a generation to the end, so a lagging relay never loses an old topic's tail: the relay writes a drain report once it proves its group has, and the coordinator retires on the reports of every named group.++The subscription follows `config/changelog` through the same follower as the writer: the oldest draining generation its group has not read to the end, then the live topic. Every topic starts from its first record when the group has no commit on it, so a migration's new topic is read from the flip.++## Delivery++A batch is up to `BATCH_SIZE` records or what arrives within `BATCH_TIMEOUT_MS`. Every record in it is produced and acked before the group commits past it, so delivery is at least once; ClickHouse keeps the highest version of each person, so a repeated row changes nothing. A failed produce retries until the broker acks and never advances the offset.++A record that can never become a row goes to `DLQ_TOPIC` with its original bytes and the `source_topic`, `source_partition`, `source_offset`, `reason` and `error` headers: no payload, not a `Person` proto, a uuid that does not parse, properties that are not UTF-8, or a timestamp out of range. A row the output rejects as too large goes there too, as its changelog record. That copy fits because the output producer's `KAFKA_PRODUCER_MESSAGE_MAX_BYTES` defaults to 64 MB, well above the changelog topic's limit; keep any override above it as well.++## Configuration++| Variable | Default | |+| --- | --- | --- |+| `KAFKA_HOSTS`, `KAFKA_TLS` | | The changelog's cluster; the `KAFKA_PRODUCER_*` settings tune the output producer |+| `KAFKA_CONSUMER_GROUP` | `personhog-changelog-relay` | |+| `OUTPUT_KAFKA_HOSTS`, `OUTPUT_KAFKA_TLS` | | The output cluster |+| `OUTPUT_TOPIC` | `clickhouse_person` | |+| `DLQ_TOPIC` | `personhog_changelog_relay_dlq` | On the output cluster |+| `BATCH_SIZE`, `BATCH_TIMEOUT_MS` | `1000`, `500` | |+| `ETCD_ENDPOINTS`, `ETCD_PREFIX` | `http://localhost:2379`, `/personhog/` | |+| `METRICS_PORT` | `9106` | |++## Metrics++- `personhog_changelog_relay_records_total{outcome="relayed"|"dlq", reason}`+- `personhog_changelog_relay_produce_retries_total{topic}`+- `personhog_changelog_relay_lag_seconds`: the age of the oldest record in the last batch, or 0 when caught up. It climbs while a delivery retries, and after a failed read it shows the age of the last record relayed. Alert on it together with the rate of `personhog_changelog_relay_produce_retries_total`; the relay keeps reporting healthy during an outage, since a restart would not fix one+- `personhog_changelog_relay_batch_duration_ms`, `personhog_changelog_relay_batch_records`+- `personhog_changelog_relay_draining`: 1 while the subscription reads a sealed generation+- `personhog_changelog_relay_commit_errors_total`, `personhog_changelog_relay_consumer_errors_total`+- `personhog_changelog_relay_resubscriptions_total`diff --git a/rust/personhog-changelog-relay/src/config.rs b/rust/personhog-changelog-relay/src/config.rsnew file mode 100644index 000000000000..d83bf566d611--- /dev/null+++ b/rust/personhog-changelog-relay/src/config.rs@@ -0,0 +1,107 @@+use std::time::Duration;++use common_kafka::config::KafkaConfig;+use envconfig::Envconfig;++#[derive(Envconfig, Clone)]+pub struct Config {+ /// The changelog's cluster; the producer settings in it tune the output producer.+ #[envconfig(nested = true)]+ pub kafka: KafkaConfig,++ #[envconfig(default = "personhog-changelog-relay")]+ pub kafka_consumer_group: String,++ /// The output cluster, which can differ from the changelog's.+ pub output_kafka_hosts: String,++ #[envconfig(default = "false")]+ pub output_kafka_tls: bool,++ #[envconfig(default = "clickhouse_person")]+ pub output_topic: String,++ #[envconfig(default = "personhog_changelog_relay_dlq")]+ pub dlq_topic: String,++ #[envconfig(default = "1000")]+ pub batch_size: usize,++ #[envconfig(default = "500")]+ pub batch_timeout_ms: u64,++ #[envconfig(default = "http://localhost:2379")]+ pub etcd_endpoints: String,++ #[envconfig(default = "/personhog/")]+ pub etcd_prefix: String,++ #[envconfig(default = "9106")]+ pub metrics_port: u16,+}++/// The output producer's default `message.max.bytes`, Node's WarpStream setting.+/// A rejected row goes to the DLQ as its changelog record, so this must stay+/// well above the changelog topic's limit for that copy to fit.+pub const OUTPUT_MESSAGE_MAX_BYTES: u32 = 64_000_000;++impl Config {+ pub fn output_kafka(&self) -> KafkaConfig {+ KafkaConfig {+ kafka_hosts: self.output_kafka_hosts.clone(),+ kafka_tls: self.output_kafka_tls,+ kafka_producer_message_max_bytes: Some(+ self.kafka+ .kafka_producer_message_max_bytes+ .unwrap_or(OUTPUT_MESSAGE_MAX_BYTES),+ ),+ ..self.kafka.clone()+ }+ }++ pub fn batch_timeout(&self) -> Duration {+ Duration::from_millis(self.batch_timeout_ms)+ }++ pub fn etcd_endpoint_list(&self) -> Vec<String> {+ self.etcd_endpoints+ .split(',')+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .map(String::from)+ .collect()+ }+}++#[cfg(test)]+mod tests {+ use std::collections::HashMap;++ use super::*;++ #[test]+ fn the_output_producer_defaults_to_a_limit_above_any_changelog_record() {+ let cases = [+ ("unset", None, OUTPUT_MESSAGE_MAX_BYTES),+ ("overridden", Some("2000000"), 2_000_000),+ ];+ for (name, setting, expected) in cases {+ let mut env = HashMap::from([(+ "OUTPUT_KAFKA_HOSTS".to_string(),+ "localhost:9092".to_string(),+ )]);+ if let Some(value) = setting {+ env.insert(+ "KAFKA_PRODUCER_MESSAGE_MAX_BYTES".to_string(),+ value.to_string(),+ );+ }+ let config = Config::init_from_hashmap(&env).unwrap();+ assert_eq!(+ config.output_kafka().kafka_producer_message_max_bytes,+ Some(expected),+ "{name}"+ );+ }+ }+}diff --git a/rust/personhog-changelog-relay/src/consumer.rs b/rust/personhog-changelog-relay/src/consumer.rsnew file mode 100644index 000000000000..89c8ce26da34--- /dev/null+++ b/rust/personhog-changelog-relay/src/consumer.rs@@ -0,0 +1,135 @@+use std::collections::{HashMap, HashSet};+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::RwLock;++use common_kafka::config::KafkaConfig;+use personhog_coordination::follow::ChangelogSubscription;+use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};+use rdkafka::error::KafkaError;+use rdkafka::message::{BorrowedMessage, Message};+use rdkafka::{ClientConfig, Offset, TopicPartitionList};++/// Broker address and transport settings alone.+pub fn base_client_config(kafka: &KafkaConfig) -> ClientConfig {+ let mut config = ClientConfig::new();+ config.set("bootstrap.servers", &kafka.kafka_hosts);+ if kafka.kafka_tls {+ config+ .set("security.protocol", "ssl")+ .set("enable.ssl.certificate.verification", "false");+ }+ if !kafka.kafka_client_rack.is_empty() {+ config.set("client.rack", &kafka.kafka_client_rack);+ }+ config+}++pub fn consumer_client_config(kafka: &KafkaConfig, group: &str) -> ClientConfig {+ let mut config = base_client_config(kafka);+ config+ .set("group.id", group)+ .set("enable.auto.commit", "false")+ .set("enable.auto.offset.store", "false")+ // A generation the group never committed on is read from its first record.+ .set("auto.offset.reset", "earliest")+ // The leader produces transactionally; aborted windows must never reach ClickHouse.+ .set("isolation.level", "read_committed")+ .set("partition.assignment.strategy", "cooperative-sticky");+ config+}++/// One changelog record, owned so it outlives the consumer's borrow.+pub struct Record {+ pub topic: String,+ pub partition: i32,+ pub offset: i64,+ /// When the leader produced it, in epoch milliseconds.+ pub timestamp_ms: Option<i64>,+ pub payload: Option<Vec<u8>>,+}++impl Record {+ fn from_message(message: &BorrowedMessage<'_>) -> Self {+ Self {+ topic: message.topic().to_string(),+ partition: message.partition(),+ offset: message.offset(),+ timestamp_ms: message.timestamp().to_millis(),+ payload: message.payload().map(<[u8]>::to_vec),+ }+ }+}++pub struct ChangelogConsumer {+ consumer: StreamConsumer,+ topic: RwLock<String>,+ draining: AtomicBool,+}++impl ChangelogConsumer {+ pub fn new(config: &ClientConfig, topic: String) -> Result<Self, KafkaError> {+ let consumer: StreamConsumer = config.create()?;+ consumer.subscribe(&[&topic])?;+ Ok(Self {+ consumer,+ topic: RwLock::new(topic),+ draining: AtomicBool::new(false),+ })+ }++ pub async fn recv(&self) -> Result<Record, KafkaError> {+ self.consumer+ .recv()+ .await+ .map(|message| Record::from_message(&message))+ }++ /// Commits each partition's next offset after the highest relayed one. A+ /// partition this pod no longer owns is skipped: its new owner relays from+ /// the last commit, so the records repeat rather than go missing.+ pub fn commit(&self, highest: &HashMap<(String, i32), i64>) -> Result<(), KafkaError> {+ let assigned: HashSet<(String, i32)> = self+ .consumer+ .assignment()?+ .elements()+ .iter()+ .map(|e| (e.topic().to_string(), e.partition()))+ .collect();+ let mut tpl = TopicPartitionList::new();+ for ((topic, partition), offset) in highest {+ if assigned.contains(&(topic.clone(), *partition)) {+ tpl.add_partition_offset(topic, *partition, Offset::Offset(offset + 1))?;+ }+ }+ if tpl.count() == 0 {+ return Ok(());+ }+ self.consumer.commit(&tpl, CommitMode::Async)+ }++ pub fn fatal_error(&self) -> Option<(rdkafka::types::RDKafkaErrorCode, String)> {+ self.consumer.client().fatal_error()+ }+}++impl ChangelogSubscription for ChangelogConsumer {+ fn topic(&self) -> String {+ self.topic.read().expect("topic lock poisoned").clone()+ }++ fn resubscribe(&self, topic: &str) -> Result<(), String> {+ self.consumer+ .subscribe(&[topic])+ .map_err(|e| e.to_string())?;+ *self.topic.write().expect("topic lock poisoned") = topic.to_string();+ Ok(())+ }++ fn set_draining(&self, draining: bool) {+ self.draining.store(draining, Ordering::SeqCst);+ }++ fn is_draining(&self) -> bool {+ self.draining.load(Ordering::SeqCst)+ }+}diff --git a/rust/personhog-changelog-relay/src/lib.rs b/rust/personhog-changelog-relay/src/lib.rsnew file mode 100644index 000000000000..64258218f7a6--- /dev/null+++ b/rust/personhog-changelog-relay/src/lib.rs@@ -0,0 +1,4 @@+pub mod config;+pub mod consumer;+pub mod relay;+pub mod transcode;diff --git a/rust/personhog-changelog-relay/src/main.rs b/rust/personhog-changelog-relay/src/main.rsnew file mode 100644index 000000000000..5eec6709a903--- /dev/null+++ b/rust/personhog-changelog-relay/src/main.rs@@ -0,0 +1,178 @@+use std::sync::Arc;+use std::time::Duration;++use assignment_coordination::store::{EtcdStore, StoreConfig};+use axum::{routing::get, Router};+use common_kafka::kafka_producer::{create_kafka_producer, spawn_bounded_flush_on_shutdown};+use common_metrics::setup_metrics_routes;+use envconfig::Envconfig;+use lifecycle::{ComponentOptions, Manager};+use metrics::counter;+use personhog_coordination::changelog::{load_changelog_config, CHANGELOG_BOOT_POLL_INTERVAL};+use personhog_coordination::follow::{follow_changelog, ChangelogSubscription, Follower, Reporter};+use personhog_coordination::store::PersonhogStore;+use tokio_util::sync::CancellationToken;+use tracing::level_filters::LevelFilter;+use tracing_subscriber::layer::SubscriberExt;+use tracing_subscriber::util::SubscriberInitExt;+use tracing_subscriber::{fmt, EnvFilter};++use personhog_changelog_relay::config::Config;+use personhog_changelog_relay::consumer::{+ base_client_config, consumer_client_config, ChangelogConsumer,+};+use personhog_changelog_relay::relay::Relay;++common_alloc::used!();++const PRODUCER_FLUSH_BOUND: Duration = Duration::from_secs(10);++#[tokio::main]+async fn main() -> Result<(), Box<dyn std::error::Error>> {+ let config = Config::init_from_env().expect("Invalid configuration");+ tracing_subscriber::registry()+ .with(fmt::layer().with_target(true).with_level(true))+ .with(+ EnvFilter::builder()+ .with_default_directive(LevelFilter::INFO.into())+ .from_env_lossy(),+ )+ .init();+ tracing::info!(+ group = %config.kafka_consumer_group,+ output_topic = %config.output_topic,+ dlq_topic = %config.dlq_topic,+ "starting personhog-changelog-relay"+ );++ let mut manager = Manager::builder("personhog-changelog-relay")+ .with_global_shutdown_timeout(Duration::from_secs(30))+ .build();+ let relay_handle = manager.register(+ "relay",+ ComponentOptions::new()+ .with_graceful_shutdown(Duration::from_secs(15))+ .with_liveness_deadline(Duration::from_secs(30)),+ );+ // Phase 1: the producer flushes only after the relay loop has stopped.+ let producer_handle = manager.register(+ "kafka-producer",+ ComponentOptions::new()+ .with_graceful_shutdown(PRODUCER_FLUSH_BOUND + Duration::from_secs(2))+ .with_shutdown_phase(1),+ );+ let changelog_handle = manager.register("changelog-config", ComponentOptions::new());+ let metrics_handle = manager.register(+ "metrics-server",+ ComponentOptions::new().is_observability(true),+ );+ let readiness = manager.readiness_handler();+ let liveness = manager.liveness_handler();+ let monitor = manager.monitor_background();++ // Installs the process-wide recorder, so it precedes every metric.+ let metrics_router = setup_metrics_routes(+ Router::new()+ .route(+ "/_readiness",+ get(move || {+ let r = readiness.clone();+ async move { r.check().await }+ }),+ )+ .route("/_liveness", get(move || async move { liveness.check() })),+ );+ let metrics_port = config.metrics_port;+ tokio::spawn(async move {+ let _guard = metrics_handle.process_scope();+ let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{metrics_port}"))+ .await+ .expect("Failed to bind metrics port");+ axum::serve(listener, metrics_router)+ .with_graceful_shutdown(metrics_handle.shutdown_signal())+ .await+ .expect("Metrics server error");+ });++ let store = EtcdStore::connect(StoreConfig {+ endpoints: config.etcd_endpoint_list(),+ prefix: config.etcd_prefix.clone(),+ })+ .await?;+ let store = Arc::new(PersonhogStore::new(store));+ let loaded = load_changelog_config(&store, CHANGELOG_BOOT_POLL_INTERVAL).await?;+ // Retire waits only on the groups the config lists, so relaying under any+ // other group could see a generation deleted before it is read.+ if !loaded.retire_groups.contains(&config.kafka_consumer_group) {+ return Err(format!(+ "config/changelog's retire_groups {:?} do not include KAFKA_CONSUMER_GROUP {}; add it from the control plane first",+ loaded.retire_groups, config.kafka_consumer_group+ )+ .into());+ }+ // Oldest undrained generation first, so a restart mid-drain resumes it.+ let mut follower = Follower::default();+ let topic = follower+ .select_topic(&loaded, |topic| drained(&config, topic))+ .await?;+ let consumer = Arc::new(ChangelogConsumer::new(+ &consumer_client_config(&config.kafka, &config.kafka_consumer_group),+ topic.clone(),+ )?);+ consumer.set_draining(topic != loaded.topic);+ tracing::info!(topic = %topic, draining = consumer.is_draining(), "subscribed to the changelog");++ let producer = create_kafka_producer(&config.output_kafka(), producer_handle.clone()).await?;+ spawn_bounded_flush_on_shutdown(producer.clone(), producer_handle, PRODUCER_FLUSH_BOUND);++ {+ let consumer = Arc::clone(&consumer);+ let config = config.clone();+ tokio::spawn(async move {+ let _guard = changelog_handle.process_scope();+ let cancel = CancellationToken::new();+ let follow = follow_changelog(+ store,+ loaded,+ follower,+ consumer,+ |topic: String| drained(&config, topic),+ Reporter::for_group(&config.kafka_consumer_group),+ |_, _| counter!("personhog_changelog_relay_resubscriptions_total").increment(1),+ cancel.clone(),+ );+ tokio::select! {+ _ = changelog_handle.shutdown_signal() => cancel.cancel(),+ _ = follow => {}+ }+ });+ }++ let relay = Relay::new(+ consumer,+ producer,+ config.output_topic.clone(),+ config.dlq_topic.clone(),+ config.batch_size,+ config.batch_timeout(),+ );+ tokio::spawn(async move {+ let _guard = relay_handle.process_scope();+ relay.run(&relay_handle).await;+ });++ monitor.wait().await?;+ Ok(())+}++/// Whether the relay's group has read every partition of `topic` to its end.+async fn drained(config: &Config, topic: String) -> Result<bool, String> {+ let base = base_client_config(&config.kafka);+ let group = config.kafka_consumer_group.clone();+ tokio::task::spawn_blocking(move || {+ common_kafka::probe::group_drained(&base, &group, &topic, Duration::from_secs(10))+ })+ .await+ .map_err(|e| format!("drain probe join: {e}"))?+ .map_err(|e| format!("drain probe: {e}"))+}diff --git a/rust/personhog-changelog-relay/src/relay.rs b/rust/personhog-changelog-relay/src/relay.rsnew file mode 100644index 000000000000..b3d1af95996e--- /dev/null+++ b/rust/personhog-changelog-relay/src/relay.rs@@ -0,0 +1,287 @@+use std::collections::HashMap;+use std::sync::Arc;+use std::time::{Duration, Instant};++use common_kafka::kafka_producer::KafkaContext;+use futures::future::join_all;+use lifecycle::Handle;+use metrics::{counter, gauge, histogram, Gauge};+use personhog_coordination::follow::ChangelogSubscription;+use rdkafka::error::KafkaError;+use rdkafka::message::{Header, OwnedHeaders};+use rdkafka::producer::{FutureProducer, FutureRecord};+use rdkafka::types::RDKafkaErrorCode;+use rdkafka::util::Timeout;+use tracing::warn;++use crate::consumer::{ChangelogConsumer, Record};+use crate::transcode::person_row;++const RETRY_BACKOFF_START: Duration = Duration::from_millis(100);+const RETRY_BACKOFF_MAX: Duration = Duration::from_secs(5);++pub struct Relay {+ consumer: Arc<ChangelogConsumer>,+ producer: FutureProducer<KafkaContext>,+ output_topic: String,+ dlq_topic: String,+ batch_size: usize,+ batch_timeout: Duration,+ /// How far `clickhouse_person` trails the leaders, in seconds.+ lag: Gauge,+}++impl Relay {+ pub fn new(+ consumer: Arc<ChangelogConsumer>,+ producer: FutureProducer<KafkaContext>,+ output_topic: String,+ dlq_topic: String,+ batch_size: usize,+ batch_timeout: Duration,+ ) -> Self {+ Self {+ consumer,+ producer,+ output_topic,+ dlq_topic,+ batch_size: batch_size.max(1),+ batch_timeout,+ lag: gauge!("personhog_changelog_relay_lag_seconds"),+ }+ }++ fn set_lag_since(&self, timestamp_ms: i64) {+ let now = chrono::Utc::now().timestamp_millis();+ self.lag.set((now - timestamp_ms).max(0) as f64 / 1000.0);+ }++ pub async fn run(&self, handle: &Handle) {+ let mut last_relayed_ms = None;+ while !handle.is_shutting_down() {+ let Some((batch, consumer_failed)) = self.collect(handle).await else {+ return;+ };+ handle.report_healthy();+ gauge!("personhog_changelog_relay_draining").set(if self.consumer.is_draining() {+ 1.0+ } else {+ 0.0+ });+ histogram!("personhog_changelog_relay_batch_records").record(batch.len() as f64);+ if batch.is_empty() {+ // An empty batch is caught up only if the consumer could read;+ // otherwise the lag is at most the age of the last record relayed.+ match (consumer_failed, last_relayed_ms) {+ (false, _) => self.lag.set(0.0),+ (true, Some(relayed)) => self.set_lag_since(relayed),+ (true, None) => {}+ }+ continue;+ }+ let started = Instant::now();+ if !self.relay_batch(&batch, handle).await {+ return;+ }+ histogram!("personhog_changelog_relay_batch_duration_ms")+ .record(started.elapsed().as_secs_f64() * 1000.0);+ if let Some(oldest) = batch.iter().filter_map(|r| r.timestamp_ms).min() {+ self.set_lag_since(oldest);+ }+ last_relayed_ms = batch.iter().filter_map(|r| r.timestamp_ms).max();+ }+ }++ /// Up to a batch of records, or what arrived within the batch timeout, and+ /// whether a read failed meanwhile. `None` when the consumer is fatally broken.+ async fn collect(&self, handle: &Handle) -> Option<(Vec<Record>, bool)> {+ let deadline = tokio::time::Instant::now() + self.batch_timeout;+ let mut batch = Vec::new();+ let mut failed = false;+ while batch.len() < self.batch_size {+ let received = tokio::select! {+ _ = handle.shutdown_recv() => return Some((Vec::new(), false)),+ received = tokio::time::timeout_at(deadline, self.consumer.recv()) => received,+ };+ match received {+ Err(_) => break,+ Ok(Ok(record)) => batch.push(record),+ Ok(Err(e)) => {+ if let Some((code, reason)) = self.consumer.fatal_error() {+ handle.signal_failure(format!(+ "Kafka consumer entered a fatal state ({code:?}): {reason}"+ ));+ return None;+ }+ counter!("personhog_changelog_relay_consumer_errors_total").increment(1);+ warn!(error = %e, "Kafka recv error");+ failed = true;+ }+ }+ }+ Some((batch, failed))+ }++ /// Delivers every record, then commits past them. Returns false when+ /// shutdown interrupted a delivery, leaving the batch uncommitted.+ pub async fn relay_batch(&self, batch: &[Record], handle: &Handle) -> bool {+ let delivered = join_all(batch.iter().map(|record| self.deliver(record, handle))).await;+ if !delivered.into_iter().all(|ok| ok) {+ return false;+ }+ let mut highest: HashMap<(String, i32), i64> = HashMap::new();+ for record in batch {+ let offset = highest+ .entry((record.topic.clone(), record.partition))+ .or_insert(record.offset);+ *offset = (*offset).max(record.offset);+ }+ if let Err(e) = self.consumer.commit(&highest) {+ counter!("personhog_changelog_relay_commit_errors_total").increment(1);+ warn!(error = %e, "offset commit failed; the next batch commits past it");+ }+ true+ }++ async fn deliver(&self, record: &Record, handle: &Handle) -> bool {+ let (reason, error) = match person_row(record.payload.as_deref()) {+ Ok(row) => match self+ .send(+ &self.output_topic,+ &row,+ None,+ record.timestamp_ms,+ handle,+ false,+ )+ .await+ {+ Sent::Delivered => {+ counter!("personhog_changelog_relay_records_total", "outcome" => "relayed")+ .increment(1);+ return true;+ }+ Sent::Shutdown => return false,+ Sent::Rejected(e) => ("rejected", format!("the output rejected the row: {e}")),+ },+ Err(poison) => (poison.reason(), poison.to_string()),+ };+ warn!(+ topic = %record.topic,+ partition = record.partition,+ offset = record.offset,+ reason,+ error = %error,+ "unrelayable changelog record; sending it to the DLQ"+ );+ let headers = dlq_headers(record, reason, &error);+ let payload = record.payload.as_deref().unwrap_or_default();+ // The record already fit on the changelog, so the DLQ retries even a rejection.+ match self+ .send(+ &self.dlq_topic,+ payload,+ Some(headers),+ record.timestamp_ms,+ handle,+ true,+ )+ .await+ {+ Sent::Delivered => {+ counter!(+ "personhog_changelog_relay_records_total",+ "outcome" => "dlq",+ "reason" => reason+ )+ .increment(1);+ true+ }+ Sent::Rejected(_) | Sent::Shutdown => false,+ }+ }++ /// Retries until the broker acks, so an outage holds the relay's offsets+ /// where they are. A rejection no retry can fix returns unless+ /// `retry_rejections` is set.+ async fn send(+ &self,+ topic: &str,+ payload: &[u8],+ headers: Option<OwnedHeaders>,+ timestamp_ms: Option<i64>,+ handle: &Handle,+ retry_rejections: bool,+ ) -> Sent {+ let mut backoff = RETRY_BACKOFF_START;+ loop {+ let mut record = FutureRecord::<(), [u8]>::to(topic).payload(payload);+ if let Some(headers) = &headers {+ record = record.headers(headers.clone());+ }+ match self.producer.send(record, Timeout::Never).await {+ Ok(_) => return Sent::Delivered,+ Err((e, _)) if !retry_rejections && is_rejection(&e) => return Sent::Rejected(e),+ Err((e, _)) => {+ counter!("personhog_changelog_relay_produce_retries_total", "topic" => topic.to_string())+ .increment(1);+ warn!(topic, error = %e, "produce failed; retrying");+ }+ }+ // A broker outage is not a stuck process, but the lag shows it.+ handle.report_healthy();+ if let Some(timestamp_ms) = timestamp_ms {+ self.set_lag_since(timestamp_ms);+ }+ tokio::select! {+ _ = handle.shutdown_recv() => return Sent::Shutdown,+ _ = tokio::time::sleep(backoff) => {}+ }+ backoff = (backoff * 2).min(RETRY_BACKOFF_MAX);+ }+ }+}++enum Sent {+ Delivered,+ Rejected(KafkaError),+ Shutdown,+}++/// Errors about the record itself, which no retry can fix.+fn is_rejection(e: &KafkaError) -> bool {+ matches!(+ e,+ KafkaError::MessageProduction(+ RDKafkaErrorCode::MessageSizeTooLarge+ | RDKafkaErrorCode::MessageBatchTooLarge+ | RDKafkaErrorCode::InvalidRecord+ )+ )+}++fn dlq_headers(record: &Record, reason: &str, error: &str) -> OwnedHeaders {+ let partition = record.partition.to_string();+ let offset = record.offset.to_string();+ OwnedHeaders::new()+ .insert(Header {+ key: "source_topic",+ value: Some(record.topic.as_str()),+ })+ .insert(Header {+ key: "source_partition",+ value: Some(partition.as_str()),+ })+ .insert(Header {+ key: "source_offset",+ value: Some(offset.as_str()),+ })+ .insert(Header {+ key: "reason",+ value: Some(reason),+ })+ .insert(Header {+ key: "error",+ value: Some(error),+ })+}diff --git a/rust/personhog-changelog-relay/src/transcode.rs b/rust/personhog-changelog-relay/src/transcode.rsnew file mode 100644index 000000000000..c314a792c70f--- /dev/null+++ b/rust/personhog-changelog-relay/src/transcode.rs@@ -0,0 +1,167 @@+//! A changelog record as the `clickhouse_person` row Node's+//! `generateKafkaPersonUpdateMessage` produces.++use chrono::DateTime;+use personhog_proto::personhog::types::v1::Person;+use prost::Message;+use serde::Serialize;++/// Why a record cannot become a row. These records go to the DLQ, since+/// retrying them can never succeed.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum Poison {+ Empty,+ Decode(String),+ Uuid(String),+ Properties,+ Timestamp(i64),+}++impl Poison {+ pub fn reason(&self) -> &'static str {+ match self {+ Poison::Empty => "empty",+ Poison::Decode(_) => "decode",+ Poison::Uuid(_) => "uuid",+ Poison::Properties => "properties",+ Poison::Timestamp(_) => "timestamp",+ }+ }+}++impl std::fmt::Display for Poison {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Poison::Empty => write!(f, "record has no payload"),+ Poison::Decode(e) => write!(f, "not a Person proto: {e}"),+ Poison::Uuid(uuid) => write!(f, "uuid {uuid:?} does not parse"),+ Poison::Properties => write!(f, "properties are not UTF-8"),+ Poison::Timestamp(ms) => write!(f, "timestamp {ms} ms is out of range"),+ }+ }+}++#[derive(Serialize)]+struct PersonRow<'a> {+ id: &'a str,+ created_at: String,+ properties: &'a str,+ team_id: i64,+ is_identified: u8,+ is_deleted: u8,+ version: i64,+ last_seen_at: Option<String>,+}++/// ClickHouse second precision in UTC, as Node writes it.+fn clickhouse_seconds(ms: i64) -> Result<String, Poison> {+ DateTime::from_timestamp_millis(ms)+ .map(|t| t.format("%Y-%m-%d %H:%M:%S").to_string())+ .ok_or(Poison::Timestamp(ms))+}++pub fn person_row(payload: Option<&[u8]>) -> Result<Vec<u8>, Poison> {+ let payload = payload.ok_or(Poison::Empty)?;+ let person = Person::decode(payload).map_err(|e| Poison::Decode(e.to_string()))?;+ // ClickHouse stops its whole Kafka consumer on a row whose id is not a UUID.+ uuid::Uuid::parse_str(&person.uuid).map_err(|_| Poison::Uuid(person.uuid.clone()))?;+ let properties = match person.properties.as_slice() {+ [] => "{}",+ bytes => std::str::from_utf8(bytes).map_err(|_| Poison::Properties)?,+ };+ let row = PersonRow {+ id: &person.uuid,+ created_at: clickhouse_seconds(person.created_at)?,+ properties,+ team_id: person.team_id,+ is_identified: u8::from(person.is_identified),+ is_deleted: u8::from(person.is_deleted),+ version: person.version,+ last_seen_at: person.last_seen_at.map(clickhouse_seconds).transpose()?,+ };+ Ok(serde_json::to_vec(&row).expect("a row of strings and integers serializes"))+}++#[cfg(test)]+mod tests {+ use super::*;++ const UUID: &str = "0192f3a1-7c4e-7a3b-9d2e-5f6a7b8c9d0e";++ fn person() -> Person {+ Person {+ id: 42,+ uuid: UUID.to_string(),+ team_id: 7,+ properties: br#"{"email":"[email protected]"}"#.to_vec(),+ created_at: 1_700_000_000_123,+ version: 3,+ is_identified: true,+ last_seen_at: Some(1_700_000_100_999),+ ..Default::default()+ }+ }++ fn row(person: &Person) -> Result<String, Poison> {+ person_row(Some(&person.encode_to_vec())).map(|b| String::from_utf8(b).unwrap())+ }++ #[test]+ fn a_person_becomes_nodes_row() {+ assert_eq!(+ row(&person()).unwrap(),+ r#"{"id":"0192f3a1-7c4e-7a3b-9d2e-5f6a7b8c9d0e","created_at":"2023-11-14 22:13:20","properties":"{\"email\":\"[email protected]\"}","team_id":7,"is_identified":1,"is_deleted":0,"version":3,"last_seen_at":"2023-11-14 22:15:00"}"#+ );+ }++ #[test]+ fn a_death_document_is_a_deleted_row_with_empty_properties() {+ let dead = Person {+ properties: Vec::new(),+ is_deleted: true,+ is_identified: false,+ last_seen_at: None,+ version: 4,+ ..person()+ };+ assert_eq!(+ row(&dead).unwrap(),+ r#"{"id":"0192f3a1-7c4e-7a3b-9d2e-5f6a7b8c9d0e","created_at":"2023-11-14 22:13:20","properties":"{}","team_id":7,"is_identified":0,"is_deleted":1,"version":4,"last_seen_at":null}"#+ );+ }++ #[test]+ fn unrelayable_records_are_poison() {+ let cases: [(&str, Result<String, Poison>, &str); 4] = [+ (+ "no payload",+ person_row(None).map(|_| String::new()),+ "empty",+ ),+ (+ "not a proto",+ person_row(Some(&[0xFF; 8])).map(|_| String::new()),+ "decode",+ ),+ (+ "bad uuid",+ row(&Person {+ uuid: "nope".to_string(),+ ..person()+ }),+ "uuid",+ ),+ (+ "bad properties",+ row(&Person {+ properties: vec![0xC3, 0x28],+ ..person()+ }),+ "properties",+ ),+ ];+ for (name, result, reason) in cases {+ assert_eq!(result.unwrap_err().reason(), reason, "{name}");+ }+ }+}diff --git a/rust/personhog-changelog-relay/tests/relay.rs b/rust/personhog-changelog-relay/tests/relay.rsnew file mode 100644index 000000000000..2ba5fdd39ad7--- /dev/null+++ b/rust/personhog-changelog-relay/tests/relay.rs@@ -0,0 +1,406 @@+use std::collections::HashMap;+use std::sync::Arc;+use std::time::Duration;++use common_kafka::config::KafkaConfig;+use envconfig::Envconfig;+use metrics_util::debugging::{DebugValue, DebuggingRecorder};+use personhog_changelog_relay::consumer::{consumer_client_config, ChangelogConsumer};+use personhog_changelog_relay::relay::Relay;+use personhog_coordination::follow::Follower;+use personhog_coordination::types::{ChangelogConfig, Generation};+use personhog_proto::personhog::types::v1::Person;+use prost::Message as _;+use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};+use rdkafka::client::DefaultClientContext;+use rdkafka::consumer::{BaseConsumer, Consumer};+use rdkafka::message::{Headers, Message};+use rdkafka::producer::FutureRecord;+use rdkafka::{ClientConfig, Offset, TopicPartitionList};++const CHANGELOG: &str = "personhog_updates_test";+const OUTPUT: &str = "clickhouse_person_test";+const DLQ: &str = "relay_dlq_test";+const GROUP: &str = "relay-test";++fn person(id: i64) -> Person {+ Person {+ id,+ uuid: format!("0192f3a1-7c4e-7a3b-9d2e-{id:012}"),+ team_id: 1,+ properties: b"{}".to_vec(),+ created_at: 1_700_000_000_000,+ version: 1,+ ..Default::default()+ }+}++/// Every record on `topic`, read from the beginning until `count` arrive.+fn read_all(bootstrap: &str, topic: &str, count: usize) -> Vec<rdkafka::message::OwnedMessage> {+ let consumer: BaseConsumer = ClientConfig::new()+ .set("bootstrap.servers", bootstrap)+ .set("group.id", format!("reader-{topic}"))+ .create()+ .unwrap();+ let mut tpl = TopicPartitionList::new();+ tpl.add_partition_offset(topic, 0, Offset::Beginning)+ .unwrap();+ consumer.assign(&tpl).unwrap();+ let mut out = Vec::new();+ let deadline = std::time::Instant::now() + Duration::from_secs(10);+ while out.len() < count && std::time::Instant::now() < deadline {+ if let Some(Ok(message)) = consumer.poll(Duration::from_millis(100)) {+ out.push(message.detach());+ }+ }+ out+}++/// Good records reach the output as Node's rows with no key; a record that is+/// not a proto, and one whose row the output rejects as too large, reach the+/// DLQ as their original bytes; the group commits past all of them.+#[tokio::test]+async fn a_batch_is_relayed_dead_lettered_and_committed() {+ let (cluster, producer) = common_kafka::test::create_mock_kafka().await;+ for topic in [CHANGELOG, OUTPUT, DLQ] {+ cluster.create_topic(topic, 1, 1).unwrap();+ }+ let bootstrap = cluster.bootstrap_servers();+ // Escaping roughly doubles these properties in the row, past the output's+ // size limit, while the record itself stays under it.+ let oversized = Person {+ properties: format!(r#"{{"k":"{}"}}"#, r#"\""#.repeat(400)).into_bytes(),+ ..person(3)+ };+ let payloads = [+ person(1).encode_to_vec(),+ vec![0xFF; 8],+ person(2).encode_to_vec(),+ oversized.encode_to_vec(),+ ];+ for payload in &payloads {+ producer+ .send(+ FutureRecord::<(), _>::to(CHANGELOG)+ .partition(0)+ .payload(payload),+ Duration::from_secs(5),+ )+ .await+ .unwrap();+ }++ let kafka = KafkaConfig::init_from_hashmap(&HashMap::from([+ ("KAFKA_HOSTS".to_string(), bootstrap.clone()),+ (+ "KAFKA_PRODUCER_MESSAGE_MAX_BYTES".to_string(),+ "1500".to_string(),+ ),+ ]))+ .unwrap();+ let consumer = Arc::new(+ ChangelogConsumer::new(+ &consumer_client_config(&kafka, GROUP),+ CHANGELOG.to_string(),+ )+ .unwrap(),+ );+ let mut manager = lifecycle::Manager::builder("test")+ .with_trap_signals(false)+ .build();+ let handle = manager.register(+ "relay",+ lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),+ );+ let _monitor = manager.monitor_background();+ let output = common_kafka::kafka_producer::create_kafka_producer(&kafka, handle.clone())+ .await+ .unwrap();+ let relay = Relay::new(+ Arc::clone(&consumer),+ output,+ OUTPUT.to_string(),+ DLQ.to_string(),+ 10,+ Duration::from_millis(500),+ );+ let run_handle = handle.clone();+ let running = tokio::spawn(async move { relay.run(&run_handle).await });++ let rows = tokio::task::spawn_blocking({+ let bootstrap = bootstrap.clone();+ move || read_all(&bootstrap, OUTPUT, 2)+ })+ .await+ .unwrap();+ let dead = tokio::task::spawn_blocking({+ let bootstrap = bootstrap.clone();+ move || read_all(&bootstrap, DLQ, 2)+ })+ .await+ .unwrap();++ assert_eq!(rows.len(), 2);+ for row in &rows {+ assert!(row.key().is_none());+ assert!(row.headers().is_none());+ let value: serde_json::Value = serde_json::from_slice(row.payload().unwrap()).unwrap();+ assert_eq!(value["created_at"], "2023-11-14 22:13:20");+ }+ assert_eq!(dead.len(), 2);+ let headers = |message: &rdkafka::message::OwnedMessage| -> HashMap<String, Vec<u8>> {+ message+ .headers()+ .unwrap()+ .iter()+ .map(|h| (h.key.to_string(), h.value.unwrap_or_default().to_vec()))+ .collect()+ };+ for (message, payload, offset, reason) in [+ (&dead[0], &payloads[1], "1", "decode"),+ (&dead[1], &payloads[3], "3", "rejected"),+ ] {+ assert_eq!(message.payload(), Some(&payload[..]));+ let headers = headers(message);+ assert_eq!(headers["source_topic"], CHANGELOG.as_bytes());+ assert_eq!(headers["source_offset"], offset.as_bytes());+ assert_eq!(headers["reason"], reason.as_bytes());+ }++ let committed = tokio::task::spawn_blocking(move || {+ let deadline = std::time::Instant::now() + Duration::from_secs(10);+ loop {+ let mut tpl = TopicPartitionList::new();+ tpl.add_partition(CHANGELOG, 0);+ let probe: BaseConsumer = ClientConfig::new()+ .set("bootstrap.servers", &bootstrap)+ .set("group.id", GROUP)+ .create()+ .unwrap();+ let offset = probe+ .committed_offsets(tpl, Duration::from_secs(5))+ .unwrap()+ .find_partition(CHANGELOG, 0)+ .map(|e| e.offset());+ if offset == Some(Offset::Offset(4)) || std::time::Instant::now() > deadline {+ return offset;+ }+ std::thread::sleep(Duration::from_millis(100));+ }+ })+ .await+ .unwrap();+ assert_eq!(committed, Some(Offset::Offset(4)));++ handle.request_shutdown();+ tokio::time::timeout(Duration::from_secs(5), running)+ .await+ .expect("the relay stops on shutdown")+ .unwrap();+}++/// The local broker (docker), which the writer's suites use too: the mock+/// broker never answers the rejoin a changed subscription sends.+const BROKER: &str = "localhost:9092";++async fn broker_topics(topics: &[&str]) {+ let admin: AdminClient<DefaultClientContext> = ClientConfig::new()+ .set("bootstrap.servers", BROKER)+ .create()+ .unwrap();+ let new: Vec<_> = topics+ .iter()+ .map(|t| NewTopic::new(t, 1, TopicReplication::Fixed(1)))+ .collect();+ for result in admin+ .create_topics(new.iter(), &AdminOptions::new())+ .await+ .unwrap()+ {+ result.unwrap();+ }+}++/// Once the old generation is drained the subscription moves to the live+/// topic, and relays what the leaders produced there before the move.+#[tokio::test]+async fn a_move_to_the_live_topic_relays_it_from_its_first_record() {+ let tag = uuid::Uuid::new_v4().simple().to_string();+ let (old, new, output, dlq, group) = (+ format!("relay_old_{tag}"),+ format!("relay_new_{tag}"),+ format!("relay_out_{tag}"),+ format!("relay_dlq_{tag}"),+ format!("relay_{tag}"),+ );+ broker_topics(&[&old, &new, &output, &dlq]).await;++ let kafka = KafkaConfig::init_from_hashmap(&HashMap::from([(+ "KAFKA_HOSTS".to_string(),+ BROKER.to_string(),+ )]))+ .unwrap();+ let mut manager = lifecycle::Manager::builder("test")+ .with_trap_signals(false)+ .build();+ let handle = manager.register(+ "relay",+ lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),+ );+ let _monitor = manager.monitor_background();+ let producer = common_kafka::kafka_producer::create_kafka_producer(&kafka, handle.clone())+ .await+ .unwrap();+ for (topic, id) in [(&old, 1), (&new, 2)] {+ producer+ .send(+ FutureRecord::<(), _>::to(topic)+ .partition(0)+ .payload(&person(id).encode_to_vec()),+ Duration::from_secs(5),+ )+ .await+ .unwrap();+ }++ let consumer = Arc::new(+ ChangelogConsumer::new(&consumer_client_config(&kafka, &group), old.clone()).unwrap(),+ );+ let relay = Relay::new(+ Arc::clone(&consumer),+ producer,+ output.clone(),+ dlq,+ 10,+ Duration::from_millis(200),+ );+ let run_handle = handle.clone();+ let running = tokio::spawn(async move { relay.run(&run_handle).await });+ let read = |count: usize| {+ let output = output.clone();+ tokio::task::spawn_blocking(move || {+ read_all(BROKER, &output, count)+ .iter()+ .map(|row| {+ let value: serde_json::Value =+ serde_json::from_slice(row.payload().unwrap()).unwrap();+ value["id"].as_str().unwrap().to_string()+ })+ .collect::<Vec<_>>()+ })+ };++ assert_eq!(read(1).await.unwrap(), vec![person(1).uuid]);++ let flipped = ChangelogConfig {+ topic: new.clone(),+ partitions: 1,+ epoch: 2,+ writer_group: "writer".to_string(),+ retire_groups: vec![group],+ draining: vec![Generation {+ topic: old.clone(),+ partitions: 1,+ epoch: 1,+ }],+ };+ let moved = Follower::default()+ .follow_once(consumer.as_ref(), &flipped, |_| async { Ok(true) })+ .await+ .unwrap();+ assert_eq!(moved, Some(old));++ assert_eq!(read(2).await.unwrap(), vec![person(1).uuid, person(2).uuid]);++ handle.request_shutdown();+ tokio::time::timeout(Duration::from_secs(5), running)+ .await+ .expect("the relay stops on shutdown")+ .unwrap();+}++/// While a delivery retries, here into a DLQ too small for the record, the+/// lag gauge climbs from the stuck record's age instead of freezing.+#[tokio::test]+async fn the_lag_climbs_while_a_delivery_retries() {+ let (cluster, producer) = common_kafka::test::create_mock_kafka().await;+ for topic in [CHANGELOG, OUTPUT, DLQ] {+ cluster.create_topic(topic, 1, 1).unwrap();+ }+ let produced_at = chrono::Utc::now().timestamp_millis() - 60_000;+ producer+ .send(+ FutureRecord::<(), _>::to(CHANGELOG)+ .partition(0)+ .timestamp(produced_at)+ .payload(&vec![0xFF; 2_000]),+ Duration::from_secs(5),+ )+ .await+ .unwrap();++ let kafka = KafkaConfig::init_from_hashmap(&HashMap::from([+ ("KAFKA_HOSTS".to_string(), cluster.bootstrap_servers()),+ (+ "KAFKA_PRODUCER_MESSAGE_MAX_BYTES".to_string(),+ "1500".to_string(),+ ),+ ]))+ .unwrap();+ let consumer = Arc::new(+ ChangelogConsumer::new(+ &consumer_client_config(&kafka, GROUP),+ CHANGELOG.to_string(),+ )+ .unwrap(),+ );+ let mut manager = lifecycle::Manager::builder("test")+ .with_trap_signals(false)+ .build();+ let handle = manager.register(+ "relay",+ lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),+ );+ let _monitor = manager.monitor_background();+ let output = common_kafka::kafka_producer::create_kafka_producer(&kafka, handle.clone())+ .await+ .unwrap();+ let recorder = DebuggingRecorder::new();+ let snapshotter = recorder.snapshotter();+ let relay = metrics::with_local_recorder(&recorder, || {+ Relay::new(+ Arc::clone(&consumer),+ output,+ OUTPUT.to_string(),+ DLQ.to_string(),+ 10,+ Duration::from_millis(200),+ )+ });+ let run_handle = handle.clone();+ let running = tokio::spawn(async move { relay.run(&run_handle).await });++ let lag = || {+ snapshotter+ .snapshot()+ .into_vec()+ .into_iter()+ .find(|(key, _, _, _)| key.key().name() == "personhog_changelog_relay_lag_seconds")+ .map(|(_, _, _, value)| match value {+ DebugValue::Gauge(value) => value.into_inner(),+ _ => panic!("the lag is a gauge"),+ })+ .unwrap_or_default()+ };+ let deadline = std::time::Instant::now() + Duration::from_secs(10);+ while lag() < 60.0 && std::time::Instant::now() < deadline {+ tokio::time::sleep(Duration::from_millis(50)).await;+ }+ assert!(lag() >= 60.0, "lag {} while the DLQ send retries", lag());++ handle.request_shutdown();+ tokio::time::timeout(Duration::from_secs(5), running)+ .await+ .expect("the relay stops on shutdown")+ .unwrap();+}