:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::fs;
use std::net::SocketAddr;
use std::num::NonZeroU32;
use std::time::Duration;
use common_ingestion_warnings::RowWarningsSettings;
use common_kafka::config::KafkaConfig;
use envconfig::Envconfig;
use personhog_common::h2_window::{Http2Windows, WindowSize};
use personhog_coordination::authority::AuthorityClock;
use personhog_coordination::pod::{
PodConfig, DRAIN_SETUP_BOUND, REVOKE_TIMEOUT, SHUTDOWN_FENCE_BOUND,
};
#[derive(Envconfig, Clone)]
pub struct Config {
#[envconfig(default = "127.0.0.1:50053")]
pub grpc_address: SocketAddr,
/// Per-partition person-cache capacity in bytes. Entries are weighed
/// by their stored size, so this bounds memory, not
/// entry count. Sized against full ownership: a lone survivor owns
/// every partition, so the worst-case cache footprint is this value
/// times the partition count (16 MiB × 16 partitions = 256 MiB).
#[envconfig(default = "16777216")]
pub cache_memory_capacity_bytes: usize,
/// Trades CPU on every cache read and write for more persons per byte.
/// The codec's samples, dictionary and per-thread contexts sit outside the capacity bound.
#[envconfig(default = "false")]
pub cache_properties_compression_enabled: bool,
/// How long a fencing transaction window admits joining writes
/// before committing, when it does not fill first (see
/// FENCING_WINDOW_MAX_WRITES). Amortizes the commit round trip
/// across concurrent same-partition writes.
#[envconfig(default = "5")]
pub fencing_window_ms: u64,
/// Closes a fencing window early once this many writes have joined,
/// rather than holding it open for the rest of FENCING_WINDOW_MS.
/// Under a backlog, windows fill and commit at once, so a drain
/// cycle is bounded by the commit round trip instead of the window;
/// the window still bounds ack latency at light load. A close
/// trigger, not a hard cap: writes arriving while the close
/// propagates still ride the window.
#[envconfig(default = "32")]
pub fencing_window_max_writes: usize,
/// Transactional producers per partition; writes rotate across them.
/// A takeover fences one id per lane, so the count binds fleet-wide:
/// a successor on a smaller count leaves a predecessor's extra lanes
/// unfenced.
#[envconfig(default = "4")]
pub fencing_lanes: usize,
/// Timeout for transactional init (fencing acquisition) and
/// commit/abort operations.
#[envconfig(default = "0")]
pub fencing_txn_timeout_ms: u64,
/// `message.timeout.ms` for the fenced changelog producer only. It is
/// separate from the shared producer's because a fenced write's total
/// must fit inside the lease self-fence runway — see
/// [`Config::validate_fencing_timescales`].
#[envconfig(default = "0")]
pub fencing_message_timeout_ms: u32,
#[envconfig(default = "9102")]
pub metrics_port: u16,
/// Maximum concurrent partition warms. Warms are broker-bound reads
/// on MSK, so this can sit well above the S3-era default of 4.
#[envconfig(default = "8")]
pub warm_concurrency: usize,
// ── gRPC server ──────────────────────────────────────────────
/// Interval between HTTP/2 keepalive pings sent by the gRPC server (0 = disabled)
#[envconfig(default = "30")]
pub grpc_keepalive_interval_secs: u64,
/// Timeout for a keepalive ping ack before considering the connection dead
#[envconfig(default = "10")]
pub grpc_keepalive_timeout_secs: u64,
#[envconfig(default = "0")]
pub grpc_initial_stream_window_bytes: WindowSize,
#[envconfig(default = "0")]
pub grpc_initial_connection_window_bytes: WindowSize,
/// Maximum gRPC message size to encode (send), in bytes. Defaults to 128 MiB.
#[envconfig(default = "134217728")]
pub grpc_max_send_message_size: usize,
/// Maximum gRPC message size to decode (receive), in bytes.
#[envconfig(default = "134217728")]
pub grpc_max_recv_message_size: usize,
/// Maximum age of a gRPC server connection before it is gracefully
/// closed (GOAWAY), guarding against half-dead long-lived connections.
/// 0 = disabled (connections live indefinitely).
#[envconfig(default = "300")]
pub grpc_max_connection_age_secs: u64,
/// Maximum concurrent in-flight gRPC requests before the server sheds
/// load with RESOURCE_EXHAUSTED. That code is terminal end to end —
/// the router treats a delivered response as an outcome and the
/// client does not retry it — so this is backpressure to the caller,
/// not a redirect. 0 = disabled.
#[envconfig(default = "0")]
pub max_concurrent_requests: usize,
// ── Response compression ─────────────────────────────────────
/// When true, gzip-compress responses for clients that advertise gzip
/// in `grpc-accept-encoding`. Compression runs on a blocking thread
/// pool instead of the tokio runtime.
#[envconfig(default = "false")]
pub gzip_response_compression: bool,
/// Gzip compression level (1–9). Lower is faster, higher compresses more.
#[envconfig(default = "6")]
pub gzip_compression_level: u32,
/// Minimum response payload size in bytes to compress. Payloads smaller
/// than this pass through uncompressed.
#[envconfig(default = "256")]
pub gzip_min_payload_size: usize,
/// Log a warning when a response exceeds this size in bytes, even
/// for uncompressed passthrough. 0 = disabled. Default 4 MiB.
#[envconfig(default = "4194304")]
pub gzip_max_response_size: usize,
/// When true, responses exceeding `gzip_max_response_size` are rejected
/// with RESOURCE_EXHAUSTED; when false, the oversized
/// response is delivered normally (monitor mode).
#[envconfig(default = "false")]
pub gzip_max_response_size_enforce: bool,
// ── Kafka durability ─────────────────────────────────────────
#[envconfig(nested = true)]
pub kafka: KafkaConfig,
// ── Warming ──────────────────────────────────────────────────
/// Consumer group name used by the writer pod. The leader queries this
/// group's committed offsets during warming: any Kafka message at or
/// after the writer's committed offset has not yet been persisted to PG,
/// so it must live in the leader's cache to avoid stale PG fallback
/// reads.
#[envconfig(default = "personhog-writer")]
pub writer_consumer_group: String,
/// Timeout for the OffsetFetch round-trip that asks the writer's
/// consumer group for its committed offset.
#[envconfig(default = "5")]
pub warm_committed_offsets_timeout_secs: u64,
/// Timeout for the per-partition `fetch_watermarks` metadata call.
#[envconfig(default = "5")]
pub warm_fetch_watermarks_timeout_secs: u64,
/// Per-message receive timeout while consuming the warming range. If
/// hit, warming aborts with the offsets seen so far so the partition
/// can be retried fresh.
#[envconfig(default = "10")]
pub warm_recv_timeout_secs: u64,
/// Maximum attempts for retryable warming metadata calls
/// (committed-offset query, fetch-watermarks).
#[envconfig(default = "3")]
pub warm_retry_max_attempts: u32,
/// Initial backoff between warming-step retries; doubles each attempt
/// up to `warm_retry_max_backoff_ms`.
#[envconfig(default = "500")]
pub warm_retry_initial_backoff_ms: u64,
/// Cap on the exponential backoff between warming-step retries.
#[envconfig(default = "5000")]
pub warm_retry_max_backoff_ms: u64,
// ── Property size admission ──────────────────────────────────
/// Ceiling for a person's properties, measured exactly as the
/// `check_properties_size` constraint on `posthog_person` measures it:
/// `pg_column_size(properties)`, the JSONB binary size. An update
/// that would newly push a within-limit row over this is rejected; a
/// row already over it (predating the constraint, or from another
/// writer) is remediated by trimming to the target below, discarding
/// the triggering update — mirroring the Node pipeline's policy. So
/// every acked record is applyable by the writer verbatim: the cache,
/// the changelog, and Postgres can never disagree about an acked row.
#[envconfig(default = "655360")]
pub properties_size_threshold: usize,
/// Size to trim already-oversized properties down to during
/// remediation, comfortably below the threshold so remediated rows
/// keep headroom under the constraint.
#[envconfig(default = "524288")]
pub properties_trim_target: usize,
#[envconfig(default = "false")]
pub ingestion_warnings_enabled: bool,
#[envconfig(default = "")]
pub ingestion_warnings_kafka_hosts: String,
#[envconfig(default = "false")]
pub ingestion_warnings_kafka_tls: bool,
#[envconfig(default = "clickhouse_ingestion_warnings")]
pub ingestion_warnings_topic: String,
#[envconfig(default = "16")]
pub ingestion_warnings_kafka_queue_mib: u32,
#[envconfig(default = "1048576")]
pub ingestion_warnings_kafka_message_max_bytes: u32,
/// The defaults below are fire-and-forget: a warning is worth less than
/// the cost of retrying it, and a slow warnings topic must not hold an update.
#[envconfig(default = "1")]
pub ingestion_warnings_kafka_acks: String,
#[envconfig(default = "0")]
pub ingestion_warnings_kafka_retries: u32,
#[envconfig(default = "100")]
pub ingestion_warnings_kafka_linger_ms: u32,
#[envconfig(default = "10000")]
pub ingestion_warnings_kafka_queue_messages: u32,
#[envconfig(default = "5000")]
pub ingestion_warnings_kafka_message_timeout_ms: u32,
/// Warnings of one type a minute that one pod sends, shared by every team
/// on it; the rest are dropped as `type_budget_exhausted`.
#[envconfig(default = "60")]
pub ingestion_warnings_type_budget_per_minute: NonZeroU32,
#[envconfig(default = "1000000")]
pub ingestion_warnings_max_tracked_scopes: usize,
// ── Dirty index / changelog recovery ─────────────────────────
/// How often to poll the writer's committed offsets and prune dirty
/// index marks the writer has applied to PG. A tick costs one batched
/// OffsetFetch plus work proportional to the marks actually reclaimed
/// (the index is never scanned), so a short interval is cheap — and it
/// bounds how long an applied-but-unpruned mark keeps sending reads to
/// the changelog for state PG already has.
#[envconfig(default = "1")]
pub dirty_index_prune_interval_secs: u64,
/// Overall deadline for recovering one evicted dirty person from the
/// changelog, including transient-failure retries. A point read that
/// hasn't returned in a few seconds isn't going to, and each recovery
/// occupies a pooled consumer for its whole duration — a long deadline
/// amplifies a broker blip into pool exhaustion.
#[envconfig(default = "5")]
pub recovery_recv_timeout_secs: u64,
/// Number of pooled changelog-recovery consumers, bounding concurrent
/// recoveries the way a DB connection pool bounds queries. Each is a
/// full Kafka client (its own connections and background threads), but
/// even 16 is fewer than the per-partition consumers this pool
/// replaced. Under a benchmarked writer outage a pool of 4 queued
/// recoveries for ~10ms on average and tripled write p99; 16 zeroed
/// the queueing. The `personhog_leader_recovery_pool_wait_ms`
/// histogram shows when this is undersized.
#[envconfig(default = "16")]
pub recovery_pool_size: usize,
/// Soft bound on dirty index entries (~100 bytes each). The index
/// grows one mark per unique person written since the writer's
/// committed offset, so this bound is the memory runway a writer
/// outage gets before new-person writes shed with RESOURCE_EXHAUSTED.
/// The default (~1 GB worst case) buys hours at heavy churn.
#[envconfig(default = "10000000")]
pub dirty_index_max_entries: usize,
/// Bound on the unresolved-version floors, its own knob rather than
/// borrowing the dirty index's. The two grow under different
/// failures: dirty marks accumulate whenever the writer lags, floors
/// only when writes end without an answer — cancellations and
/// ambiguous commits — which is orders of magnitude rarer. Past the
/// bound, floors spill to one coarse per-partition value (safety
/// survives; precision degrades), so this sizes memory, not
/// correctness. The default is a fraction of the dirty index's.
#[envconfig(default = "1000000")]
pub emitted_versions_max_entries: usize,
/// Hard cap on live lifecycle fences (~100 bytes each). The fence map
/// grows one entry per person frozen by a lifecycle op and has no
/// eviction, so this is the memory fuse against a surge of ops from a
/// huge customer: at the cap, new FencePerson calls shed with
/// RESOURCE_EXHAUSTED — backpressure the saga's retries absorb —
/// while re-seals of already-fenced persons and the takeover scan
/// (marks already live; the freeze must hold) are exempt.
#[envconfig(default = "250000")]
pub fence_map_max_entries: usize,
// ── PG fallback ───────────────────────────────────────────────
/// Postgres URL for cache miss fallback. If empty, cache misses
/// return NotFound without querying PG. Must point at the primary:
/// the dirty index prunes a mark as soon as the writer's committed
/// offset shows the primary has the row, so reading an async replica
/// here would serve stale rows for unmarked persons and silently
/// break read-your-write. Lifecycle mark verifications read this pool
/// too and fail closed on replica lag.
#[envconfig(default = "")]
pub fallback_database_url: String,
/// Table the fallback reads. Must be the table the writer maintains
/// (its PG_TARGET_TABLE): the dirty index treats an unmarked person's
/// PG row as current, which is only true of the writer's own target.
/// Prod pairs posthog_person on both sides; the dev validation stack
/// pairs personhog_person_tmp on both — flip them together at cutover.
#[envconfig(default = "posthog_person")]
pub fallback_table: String,
/// Saga state tables the fence checks read (the takeover scan, the mark
/// verifications, the ghost-fence healer). Must be the pair identity
/// writes: its LIFECYCLE_OP_TABLE and LIFECYCLE_OP_PERSON_TABLE.
#[envconfig(default = "lifecycle_op")]
pub lifecycle_op_table: String,
#[envconfig(default = "lifecycle_op_person")]
pub lifecycle_op_person_table: String,
#[envconfig(default = "5")]
pub fallback_pg_max_connections: u32,
/// Keep at least this many connections warm so the first cache-miss
/// after a quiet period doesn't pay the TCP+auth handshake tax.
#[envconfig(default = "1")]
pub fallback_pg_min_connections: u32,
// ── etcd coordination ────────────────────────────────────────
#[envconfig(default = "http://localhost:2379")]
pub etcd_endpoints: String,
#[envconfig(default = "/personhog/")]
pub etcd_prefix: String,
/// Pod name for etcd registration (typically set from K8s downward API)
#[envconfig(default = "leader-0")]
pub pod_name: String,
/// Pod IP from the K8s downward API (`status.podIP`), injected by the
/// chart. Used to derive the advertised gRPC address when binding a
/// wildcard. Unset in local runs, which bind a concrete address.
#[envconfig(default = "")]
pub pod_ip: String,
/// Enable K8s awareness: at startup the leader discovers its owning
/// controller (Deployment) and generation (pod-template-hash) and
/// registers them, so the coordinator can steer placement away from
/// old-generation pods during rollouts instead of handing partitions
/// to pods that are about to be replaced. Requires RBAC to read
/// pods, replicasets, and deployments in the pod's namespace.
#[envconfig(default = "false")]
pub k8s_awareness_enabled: bool,
/// Kubernetes namespace for controller discovery. If empty,
/// auto-reads from the service account mount.
#[envconfig(default = "")]
pub k8s_namespace: String,
#[envconfig(default = "30")]
pub lease_ttl: i64,
#[envconfig(default = "10")]
pub heartbeat_interval_secs: u64,
// ── Shutdown budgets ─────────────────────────────────────────
// The lifecycle manager's per-phase windows. Configurable because
// the terms validated against them — the drain timeout, the
// heartbeat interval — are, and a fixed ceiling under adjustable
// terms is a configuration an operator cannot resolve. Their
// relations are checked by `validate_shutdown_budgets` at startup.
/// How long the lifecycle manager lets the coordination component
/// exit gracefully. The pod's whole teardown — drain setup, drain,
/// fence, keepalive join, revoke — must fit inside it, which
/// `validate_lease_timescales` enforces.
#[envconfig(default = "55")]
pub coordination_graceful_shutdown_secs: u64,
/// The phase-1 components' shared budget: the gRPC server and the
/// producer stop in parallel after coordination finishes.
#[envconfig(default = "15")]
pub phase1_graceful_shutdown_secs: u64,
/// Phase 0 (coordination) plus phase 1, with slack. Must stay under
/// the chart's termination grace period so shutdown concludes
/// process-side.
#[envconfig(default = "75")]
pub global_shutdown_timeout_secs: u64,
}
/// How many times a window's commit is attempted in total, counting the
/// first.
pub const FENCING_COMMIT_ATTEMPTS: u32 = 2;
/// How many times a window's abort is attempted in total.
///
/// One, deliberately. An abort that does not land leaves the producer in
/// a state it cannot begin another transaction from — which is now a
/// condemned fence, given up on the next write and re-taken by the
/// healing pass, whose `init_transactions` aborts the pending
/// transaction at the broker as a side effect. Retrying the abort is
/// therefore a slower, less reliable version of a recovery that already
/// exists, and it is not free: every attempt is bounded by the
/// transaction timeout and has to fit the same runway.
pub const FENCING_ABORT_ATTEMPTS: u32 = 1;
/// Every call bounded by the transaction timeout that one window can
/// make. Both the runway bound and the broker's own patience are sized
/// from this, so the two cannot drift apart.
pub const FENCING_TXN_CALLS: u32 = FENCING_COMMIT_ATTEMPTS + FENCING_ABORT_ATTEMPTS;
/// Sanity ceiling on lanes: each is a producer, a connection and a thread
/// per acquisition.
pub const MAX_FENCING_LANES: usize = 64;
/// librdkafka's documented minimum for `transaction.timeout.ms`.
const MIN_TXN_TIMEOUT: Duration = Duration::from_millis(1000);
/// A floor for the send timeout; zero means *no timeout* to librdkafka,
/// the opposite of what this bound exists to express.
const MIN_MESSAGE_TIMEOUT: Duration = Duration::from_millis(250);
/// Apache Kafka's default `transaction.max.timeout.ms`. A producer that
/// asks for longer is refused at `init_transactions` rather than trimmed.
const MAX_BROKER_TXN_TIMEOUT: Duration = Duration::from_secs(900);
impl Config {
/// The runway the keepalive reserves for the local fence: it
/// declares lease loss after two thirds of the TTL, leaving the
/// final third for the fence to land before the coordinator can
/// treat the lease as expired.
pub fn lease_fence_runway(&self) -> Duration {
Duration::from_millis((self.lease_ttl.max(0) as u64).saturating_mul(1000) / 3)
}
/// Slack the drain spends outside the writes themselves: the settle
/// wait that follows `wait_until_empty`, plus one poll interval of
/// granularity on the wait (the coordination drain polls at 50ms).
/// Subtracted from the runway before sizing the write budget — sizing
/// against the full runway left the lease-loss drain oversubscribed
/// by construction, so worst-case queueing failed the drain and cost
/// a restart to load rather than to failure.
fn fencing_drain_slack(&self) -> Duration {
self.fencing_settle_budget() + Duration::from_millis(50)
}
/// What one fenced write may spend and still resolve inside the runway
/// the lease reserves for self-fencing: net of the settle that follows
/// a drain, and halved, because a write can park behind a committing
/// window and pay that window's send and commit before its own.
fn fencing_budget(&self) -> Duration {
self.lease_fence_runway()
.saturating_sub(Duration::from_millis(self.fencing_window_ms))
.saturating_sub(self.fencing_drain_slack())
/ 2
}
/// How long a fenced send may take: what the transaction calls leave.
pub fn fencing_message_timeout(&self) -> Duration {
if self.fencing_message_timeout_ms > 0 {
return Duration::from_millis(u64::from(self.fencing_message_timeout_ms));
}
self.fencing_budget()
.saturating_sub(self.fencing_txn_timeout() * FENCING_TXN_CALLS)
.max(MIN_MESSAGE_TIMEOUT)
}
/// How long a transaction init, commit, or abort may take: an even
/// share of the budget with the send, never under librdkafka's floor.
pub fn fencing_txn_timeout(&self) -> Duration {
if self.fencing_txn_timeout_ms > 0 {
return Duration::from_millis(self.fencing_txn_timeout_ms);
}
(self.fencing_budget() / (FENCING_TXN_CALLS + 1)).max(MIN_TXN_TIMEOUT)
}
/// How long fence acquisition may take.
///
/// Deliberately not the transaction timeout. That one is sized so
/// that a *queued write* still resolves inside the lease runway, and
/// acquisition is neither queued nor a write — it is two broker round
/// trips, one of which makes the coordinator abort a predecessor's
/// open transaction. Tying it to the runway meant re-budgeting the
/// write path silently tightened acquisition, and acquisition happens
/// fleet-wide during a deploy, which is the worst moment to have
/// shortened it.
///
/// It still has to fit inside the runway as a whole: a warm that
/// outlives the lease it is warming for has nothing to serve.
pub fn fencing_init_timeout(&self) -> Duration {
const MIN_INIT_TIMEOUT: Duration = Duration::from_secs(2);
self.fencing_txn_timeout()
.max(MIN_INIT_TIMEOUT)
.min(self.lease_fence_runway())
}
/// Ceiling on the drain's wait for an open window to commit.
///
/// The wait exists to catch a committer about to fire anyway — it
/// sleeps for the window, then commits — so a longer budget only
/// helps when the commit is retrying, which is the case where the
/// successor's abort is the right answer regardless.
///
/// Absolute rather than derived from the lease, because what it has
/// to fit inside is absolute: the pre-revoke self-fence allows three
/// seconds for a whole drain. Bounding it by the broker's own
/// patience instead would be inert — `fencing_txn_timeout` floors at
/// a second and the window can make three such calls, so that bound
/// never falls below four and a half, and every accepted lease would
/// leave the cap doing all the work. Before the drain slack was paid
/// out of the write budget it sat at 7.5s at the production lease
/// (about 6s now), truncating on every shutdown with an open window
/// and reporting the drain as failed.
///
/// Residual: a `drain_timeout` configured under two seconds shrinks
/// that allowance below this budget, and the leader cannot see the
/// coordination-side value to derive against. Truncating is safe —
/// the wait is best-effort — so the cost is a spurious drain
/// failure, not a correctness one.
pub fn fencing_settle_budget(&self) -> Duration {
Duration::from_secs(2)
}
/// How long the broker may hold one of this pod's transactions open
/// before abandoning it.
///
/// A window lives from `begin_transaction` through its admission
/// interval, its sends, and its commit, so the broker's patience has
/// to cover all three — bounding it by the commit alone would let the
/// broker abort a window this pod is still legitimately filling, and
/// the resulting epoch bump reads exactly like a fence from a real
/// successor.
pub fn fencing_broker_txn_timeout(&self) -> Duration {
// Every transaction-timeout-bounded call the window can make, not
// just the first: a commit that retries, or a commit followed by
// an abort, keeps the transaction open for all of them. A bound
// that covers only one call lets the broker abandon a window this
// pod is still legitimately working, and that epoch bump is
// indistinguishable from a real successor's fence — so a
// retriable blip would read as "this pod's claim is stale" and
// cost the partition its fence.
//
// Half again on top, because the arithmetic is a floor: the
// window sleep can overshoot and each blocking call waits its
// turn on the blocking pool.
let lifetime = Duration::from_millis(self.fencing_window_ms)
+ self.fencing_message_timeout()
+ self.fencing_txn_timeout() * FENCING_TXN_CALLS;
lifetime + lifetime / 2
}
/// The lease relations that hold whether or not fencing is on.
///
/// `PodHandle::new` asserts that the heartbeat fits inside the
/// keepalive's renewal margin, but it does so several hundred lines
/// into startup — after etcd, Kafka and the Postgres pool are
/// established — and its message names the heartbeat rather than the
/// TTL that decides the margin. Checking it here turns a late panic
/// into an early refusal that names both.
pub fn validate_lease_timescales(&self) -> Result<(), String> {
let margin = AuthorityClock::renewal_margin(self.lease_ttl);
let heartbeat = self.heartbeat_interval();
// Zero is under every margin, so the comparison below waves it
// through — but the keepalive uses this interval as the *timeout*
// for each renewal round, so a zero one times out instantly,
// exhausts the margin through its retry pace, and ends the
// session. The pod then releases every partition and starts over,
// for as long as it runs, without ever serving or crashing.
if heartbeat.is_zero() {
return Err(
"HEARTBEAT_INTERVAL_SECS must be greater than zero: the keepalive uses it as \
the timeout for each renewal round, so a zero interval fences the pod against \
healthy etcd in a loop it cannot leave"
.to_string(),
);
}
if heartbeat >= margin {
return Err(format!(
"HEARTBEAT_INTERVAL_SECS ({heartbeat:?}) must be well under the keepalive \
renewal margin ({margin:?} = 2/3 of LEASE_TTL {}s): the sleep between \
renewals would exhaust the margin on its own, and the pod would fence \
itself against healthy etcd",
self.lease_ttl,
));
}
// The pod's graceful exit — drain setup, drain, shutdown-path
// fence, one keepalive round's join, bounded revoke — has to
// fit the coordination budget, or the lifecycle manager
// abandons it mid-teardown while this pod is still the
// registered owner. The drain term reads the same
// `base_pod_config` the running pod is built from, so a future
// knob cannot decouple the validated sum from the deployed one.
let drain = self.base_pod_config().drain_timeout;
let teardown =
DRAIN_SETUP_BOUND + drain + SHUTDOWN_FENCE_BOUND + heartbeat + REVOKE_TIMEOUT;
let budget = self.coordination_graceful_shutdown();
if teardown >= budget {
return Err(format!(
"the pod's teardown ({teardown:?} = setup {DRAIN_SETUP_BOUND:?} + drain \
{drain:?} + fence {SHUTDOWN_FENCE_BOUND:?} + a {heartbeat:?} keepalive join \
+ revoke {REVOKE_TIMEOUT:?}) must finish inside the coordination \
component's {budget:?} graceful shutdown budget; lower \
HEARTBEAT_INTERVAL_SECS or raise COORDINATION_GRACEFUL_SHUTDOWN_SECS"
));
}
Ok(())
}
/// The lifecycle manager's phases must fit the window that
/// supervises them, or its own deadline fires before theirs.
///
/// Checked at startup rather than compile time because the budgets
/// are configuration: a fixed ceiling under adjustable phase
/// timings is a configuration an operator cannot resolve.
pub fn validate_shutdown_budgets(&self) -> Result<(), String> {
let phases = self.coordination_graceful_shutdown() + self.phase1_graceful_shutdown();
let global = self.global_shutdown_timeout();
if phases >= global {
return Err(format!(
"the shutdown phases ({phases:?} = a {:?} coordination drain plus a {:?} \
server and producer stop) must finish inside the {global:?} global window \
with room to spare; raise GLOBAL_SHUTDOWN_TIMEOUT_SECS or lower the phases",
self.coordination_graceful_shutdown(),
self.phase1_graceful_shutdown(),
));
}
Ok(())
}
pub fn coordination_graceful_shutdown(&self) -> Duration {
Duration::from_secs(self.coordination_graceful_shutdown_secs)
}
pub fn phase1_graceful_shutdown(&self) -> Duration {
Duration::from_secs(self.phase1_graceful_shutdown_secs)
}
pub fn global_shutdown_timeout(&self) -> Duration {
Duration::from_secs(self.global_shutdown_timeout_secs)
}
/// The coordination-relevant half of the pod's configuration, shared
/// by `main`'s construction and the teardown validation above so the
/// two cannot drift: a drain or heartbeat knob added here is summed
/// by the validation automatically, where one added at the
/// construction site would be invisible to it.
pub fn base_pod_config(&self) -> PodConfig {
PodConfig {
lease_ttl: self.lease_ttl,
heartbeat_interval: self.heartbeat_interval(),
// Zero would park every warm on an unobtainable permit and
// wedge handoffs; treat it as fully sequential instead.
warm_concurrency: self.warm_concurrency.max(1),
..Default::default()
}
}
/// Every relation the fenced produce path depends on, checked at
/// startup: the derivation satisfies them wherever the lease TTL
/// leaves room, and an operator can override either knob.
pub fn validate_fencing_timescales(&self) -> Result<(), String> {
if self.fencing_lanes < 1 {
return Err("FENCING_LANES must be at least 1".to_string());
}
if self.fencing_lanes > MAX_FENCING_LANES {
return Err(format!(
"FENCING_LANES ({}) must be at most {MAX_FENCING_LANES}",
self.fencing_lanes
));
}
if self.fencing_window_max_writes == 0 {
return Err(
"FENCING_WINDOW_MAX_WRITES must be at least 1: zero would close every \
window at its first write, which is the per-write-commit shape the \
window exists to avoid"
.to_string(),
);
}
let (message, txn, runway, window) = (
self.fencing_message_timeout(),
self.fencing_txn_timeout(),
self.lease_fence_runway(),
Duration::from_millis(self.fencing_window_ms),
);
// librdkafka fails `init_transactions` outright when this
// exceeds the broker's `transaction.max.timeout.ms`, and it does
// so on every partition, forever, with nothing but a heal-failure
// counter to say why. The default broker ceiling is fifteen
// minutes; a lease TTL of an hour or more derives past it.
let broker_txn = self.fencing_broker_txn_timeout();
if broker_txn > MAX_BROKER_TXN_TIMEOUT {
return Err(format!(
"the derived broker transaction timeout ({broker_txn:?}) exceeds the usual \
broker ceiling ({MAX_BROKER_TXN_TIMEOUT:?} = transaction.max.timeout.ms): \
init_transactions would fail on every partition; lower LEASE_TTL"
));
}
if txn < MIN_TXN_TIMEOUT {
return Err(format!(
"fencing transaction timeout ({txn:?}) is below librdkafka's minimum \
({MIN_TXN_TIMEOUT:?}); the producer would not start"
));
}
if message < MIN_MESSAGE_TIMEOUT {
return Err(format!(
"fencing message timeout ({message:?}) is below {MIN_MESSAGE_TIMEOUT:?}; \
zero means no timeout at all to librdkafka"
));
}
if message > txn {
return Err(format!(
"fencing message timeout ({message:?}) exceeds the transaction timeout \
({txn:?}); librdkafka rejects the producer outright"
));
}
// Acquisition spends its timeout twice — once on the metadata
// ping, once on init_transactions — and both have to land inside
// the runway the keepalive reserves. The queued-write bound below
// caps the derived value transitively at the defaults, but an
// override can move the inputs, so the two-call bound is checked
// by name.
if self.fencing_init_timeout() * 2 > runway {
return Err(format!(
"fencing acquisition spends its timeout ({:?}) on two broker calls, which \
does not fit the lease fence runway ({runway:?}); raise LEASE_TTL or lower \
the transaction-timeout override",
self.fencing_init_timeout()
));
}
// A write parked behind a committing window pays that window's
// send and commit before its own, so the runway has to cover
// two — each of them including every commit attempt the code
// will make, not just the first.
let queued_worst_case = window + (message + txn * FENCING_TXN_CALLS) * 2;
let drain_room = runway.saturating_sub(self.fencing_drain_slack());
if queued_worst_case > drain_room {
return Err(format!(
"a fenced write queued behind another can take {queued_worst_case:?} \
(window {window:?} + 2 × (send {message:?} + {FENCING_TXN_CALLS} × commit/abort \
{txn:?})), longer than the lease self-fence runway ({runway:?} = LEASE_TTL \
{}s / 3); raise LEASE_TTL or lower the fencing timeouts",
self.lease_ttl,
));
}
Ok(())
}
pub fn grpc_keepalive_interval(&self) -> Option<Duration> {
if self.grpc_keepalive_interval_secs == 0 {
None
} else {
Some(Duration::from_secs(self.grpc_keepalive_interval_secs))
}
}
pub fn grpc_keepalive_timeout(&self) -> Option<Duration> {
if self.grpc_keepalive_timeout_secs == 0 {
None
} else {
Some(Duration::from_secs(self.grpc_keepalive_timeout_secs))
}
}
pub fn grpc_http2_windows(&self) -> Http2Windows {
Http2Windows::new(
self.grpc_initial_stream_window_bytes,
self.grpc_initial_connection_window_bytes,
)
}
pub fn grpc_max_connection_age(&self) -> Option<Duration> {
if self.grpc_max_connection_age_secs == 0 {
None
} else {
Some(Duration::from_secs(self.grpc_max_connection_age_secs))
}
}
pub fn etcd_endpoint_list(&self) -> Vec<String> {
self.etcd_endpoints
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect()
}
pub fn heartbeat_interval(&self) -> Duration {
Duration::from_secs(self.heartbeat_interval_secs)
}
pub fn ingestion_warnings_settings(&self) -> RowWarningsSettings {
RowWarningsSettings {
enabled: self.ingestion_warnings_enabled,
kafka_hosts: self.ingestion_warnings_kafka_hosts.clone(),
kafka_tls: self.ingestion_warnings_kafka_tls,
topic: self.ingestion_warnings_topic.clone(),
kafka_queue_mib: self.ingestion_warnings_kafka_queue_mib,
kafka_message_max_bytes: self.ingestion_warnings_kafka_message_max_bytes,
kafka_acks: self.ingestion_warnings_kafka_acks.clone(),
kafka_retries: self.ingestion_warnings_kafka_retries,
kafka_linger_ms: self.ingestion_warnings_kafka_linger_ms,
kafka_queue_messages: self.ingestion_warnings_kafka_queue_messages,
kafka_message_timeout_ms: self.ingestion_warnings_kafka_message_timeout_ms,
type_budget_per_minute: self.ingestion_warnings_type_budget_per_minute,
max_tracked_scopes: self.ingestion_warnings_max_tracked_scopes,
}
}
/// Resolve the K8s namespace from config or the service account mount.
pub fn resolve_k8s_namespace(&self) -> Result<String, String> {
if !self.k8s_namespace.is_empty() {
return Ok(self.k8s_namespace.clone());
}
fs::read_to_string("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
.map(|s| s.trim().to_string())
.map_err(|e| {
format!("k8s_namespace not set and failed to read from service account: {e}")
})
}
}
/// Derive the `host:port` this leader should advertise for routing.
///
/// The advertised port is always the serving port (taken from the bind
/// address), so it cannot drift from reality. The host is the bind host
/// when it is concrete (local runs bind `127.0.0.1:<port>`), or POD_IP
/// when binding a wildcard (deployments bind `0.0.0.0`). Wildcard with no
/// POD_IP fails closed: a leader that cannot say where it is reachable
/// must not register and claim partitions.
pub fn derive_advertise_address(
grpc_address: &std::net::SocketAddr,
pod_ip: &str,
) -> Result<String, String> {
if !grpc_address.ip().is_unspecified() {
return Ok(grpc_address.to_string());
}
if pod_ip.is_empty() {
return Err(format!(
"cannot derive an advertise address: GRPC_ADDRESS binds the wildcard \
{grpc_address} and POD_IP is not set — routers would have nowhere to dial"
));
}
Ok(format!("{pod_ip}:{}", grpc_address.port()))
}
#[cfg(test)]
mod tests {
use super::derive_advertise_address;
#[test]
fn advertise_address_prefers_concrete_bind_and_requires_pod_ip_for_wildcards() {
let concrete = "127.0.0.1:50060".parse().unwrap();
assert_eq!(
derive_advertise_address(&concrete, "").unwrap(),
"127.0.0.1:50060"
);
let wildcard = "0.0.0.0:50053".parse().unwrap();
assert_eq!(
derive_advertise_address(&wildcard, "10.1.2.3").unwrap(),
"10.1.2.3:50053"
);
assert!(derive_advertise_address(&wildcard, "").is_err());
let wildcard6 = "[::]:50053".parse().unwrap();
assert!(derive_advertise_address(&wildcard6, "").is_err());
}
}
#[cfg(test)]
mod lease_timescale_tests {
use super::*;
/// The phase budgets have to fit the window supervising them. This
/// held at compile time while they were constants; as configuration
/// it is a startup refusal, and the defaults must still satisfy it.
#[test]
fn shutdown_phases_that_overrun_the_global_window_are_refused() {
let config =
Config::init_from_hashmap(&std::collections::HashMap::new()).expect("defaults");
assert!(
config.validate_shutdown_budgets().is_ok(),
"the defaults must satisfy their own relations"
);
let mut overrun =
Config::init_from_hashmap(&std::collections::HashMap::new()).expect("defaults");
overrun.global_shutdown_timeout_secs =
overrun.coordination_graceful_shutdown_secs + overrun.phase1_graceful_shutdown_secs;
assert!(
overrun.validate_shutdown_budgets().is_err(),
"phases summing to the whole global window leave the manager no slack"
);
}
/// The pod's teardown must fit the coordination component's budget,
/// and the keepalive join is the one term an operator can move. A
/// heartbeat the renewal margin accepts can still blow the budget —
/// that band is exactly what a check on the margin alone missed.
#[test]
fn a_teardown_the_shutdown_budget_cannot_fit_is_refused() {
let mut config =
Config::init_from_hashmap(&std::collections::HashMap::new()).expect("defaults");
config.lease_ttl = 30;
// Inside the 20s renewal margin, so the pair check passes — but
// setup (5s) + drain (30s) + fence (3s) + a 16s join + revoke
// (5s) = 59s overruns the 55s budget. Sixteen, not a rounder
// number, because it discriminates on every term: without the
// setup bound the sum is 54s, which a broken validation would
// accept — a 17s join sums to exactly 55s either way and pins
// nothing about the setup term.
config.heartbeat_interval_secs = 16;
assert!(config.validate_lease_timescales().is_err());
config.heartbeat_interval_secs = 10;
assert!(
config.validate_lease_timescales().is_ok(),
"the default heartbeat must fit the budget"
);
}
}
#[cfg(test)]
mod fencing_timescale_tests {
use super::*;
/// The envconfig defaults with no environment behind them, so an
/// ambient variable in a developer's shell cannot change what these
/// tests assert.
fn fenced(lease_ttl: i64) -> Config {
let mut config =
Config::init_from_hashmap(&std::collections::HashMap::new()).expect("defaults");
config.lease_ttl = lease_ttl;
config.fencing_txn_timeout_ms = 0;
config.fencing_message_timeout_ms = 0;
config
}
/// At any lease TTL the derivation must either satisfy every
/// relation or be rejected — never produce a config that starts and
/// then violates the runway, and never one librdkafka refuses.
#[test]
fn derived_timeouts_are_either_valid_or_rejected() {
// 16 and 20 are the band where the bound's attempt count is the
// only thing that decides acceptance — without them the sweep
// cannot tell a two-call model from a three-call one.
for lease_ttl in [0, 1, 5, 10, 16, 20, 21, 30, 60, 300] {
let config = fenced(lease_ttl);
if config.validate_fencing_timescales().is_ok() {
let (message, txn) = (
config.fencing_message_timeout(),
config.fencing_txn_timeout(),
);
assert!(txn >= MIN_TXN_TIMEOUT, "LEASE_TTL={lease_ttl}");
assert!(message >= MIN_MESSAGE_TIMEOUT, "LEASE_TTL={lease_ttl}");
assert!(message <= txn, "LEASE_TTL={lease_ttl}");
// Mirrors the production bound, retries included: a
// check that models fewer attempts than the code makes
// would accept exactly the configurations that break it.
let queued = Duration::from_millis(config.fencing_window_ms)
+ (message + txn * FENCING_TXN_CALLS) * 2;
assert!(
queued <= config.lease_fence_runway(),
"LEASE_TTL={lease_ttl}: queued worst case {queued:?} exceeds runway"
);
}
}
}
/// The attempt count decides the split, so budget the calls do not
/// spend is a derivation bug, not slack.
#[test]
fn the_production_ttl_affords_every_transaction_call() {
let config = fenced(30);
let (message, txn, runway, window) = (
config.fencing_message_timeout(),
config.fencing_txn_timeout(),
config.lease_fence_runway(),
Duration::from_millis(config.fencing_window_ms),
);
let queued = window + (message + txn * FENCING_TXN_CALLS) * 2;
assert!(
queued <= runway,
"{FENCING_TXN_CALLS} transaction calls need {queued:?}, runway is {runway:?}: \
lower the attempt counts"
);
assert_eq!(
message + txn * FENCING_TXN_CALLS,
config.fencing_budget(),
"the derivation leaves budget unspent"
);
}
/// The transaction calls take only their floor; the send gets the rest.
#[test]
fn the_send_gets_what_the_transaction_calls_leave() {
let txn = fenced(30).fencing_txn_timeout();
assert!(
txn <= MIN_TXN_TIMEOUT + MIN_TXN_TIMEOUT / 10,
"transaction timeout {txn:?} sits well above the floor; that runway belongs to the send"
);
}
/// The gap that made the fencing check misleading: a TTL it accepts
/// can still be one the pod refuses to start on, and that refusal
/// arrived hundreds of lines later, blamed the heartbeat, and never
/// mentioned the TTL that actually decides the margin.
///
/// The fencing floor and the heartbeat happen not to overlap at the
/// default heartbeat today, so the pairs below raise it — which is
/// the point: the two checks constrain different things, and nothing
/// keeps a future change to the timeout shares from moving the
/// fencing floor back under the heartbeat.
#[test]
fn a_lease_ttl_the_heartbeat_cannot_fit_is_refused_up_front() {
for (lease_ttl, heartbeat_secs) in [(27, 18), (30, 20), (60, 40)] {
let mut config = fenced(lease_ttl);
config.heartbeat_interval_secs = heartbeat_secs;
assert!(
config.validate_fencing_timescales().is_ok(),
"LEASE_TTL={lease_ttl} is meant to pass the fencing check"
);
let err = config
.validate_lease_timescales()
.expect_err("but must not pass the lease check");
assert!(
err.contains("LEASE_TTL"),
"the refusal must name the knob to change, got: {err}"
);
}
}
/// Zero passes the margin comparison — it is under every margin — but
/// the keepalive uses the interval as each round's timeout, so it
/// fences the pod against healthy etcd forever.
#[test]
fn a_zero_heartbeat_is_refused() {
let mut config = fenced(30);
config.heartbeat_interval_secs = 0;
let err = config
.validate_lease_timescales()
.expect_err("a zero heartbeat must not start");
assert!(
err.contains("greater than zero"),
"the refusal must name the constraint, got: {err}"
);
}
/// And the production pairing must survive it, or the check would
/// refuse the fleet it was written to protect.
#[test]
fn the_production_lease_and_heartbeat_agree() {
let mut config = fenced(30);
config.heartbeat_interval_secs = 10;
config
.validate_lease_timescales()
.expect("LEASE_TTL=30 with a 10s heartbeat must start");
}
/// The broker's patience has to cover the whole window, not the first
/// call into it. A bound shorter than the lifetime lets the broker
/// abandon a window this pod is still working, and that epoch bump
/// is indistinguishable from a real successor's fence — so a
/// retriable blip would cost the partition its fence and read as a
/// stale claim.
#[test]
fn the_broker_waits_out_the_whole_window() {
for lease_ttl in [21, 30, 60, 300] {
let config = fenced(lease_ttl);
if config.validate_fencing_timescales().is_err() {
continue;
}
let lifetime = Duration::from_millis(config.fencing_window_ms)
+ config.fencing_message_timeout()
+ config.fencing_txn_timeout() * FENCING_TXN_CALLS;
assert!(
config.fencing_broker_txn_timeout() >= lifetime + lifetime / 2,
"LEASE_TTL={lease_ttl}: broker bound {:?} leaves no slack over the window's \
own worst case {lifetime:?} — the sleep overshoots and each blocking call \
waits its turn on the pool",
config.fencing_broker_txn_timeout(),
);
}
}
/// The fenced path creates one producer per owned partition, so the
/// shared producer's queue limits are an aggregate to divide rather
/// than a per-producer figure to copy. Inheriting them let a lone
/// survivor owning every partition claim the whole product — several
/// gigabytes on a pod sized for one — which a client can reach by
/// spreading large property updates across partitions while Kafka is
/// slow.
#[test]
fn fenced_producer_queues_do_not_scale_with_partition_count() {
let config = fenced(30);
let shared_mib = config.kafka.kafka_producer_queue_mib;
let shared_messages = config.kafka.kafka_producer_queue_messages;
for partitions in [1, 8, 16, 50, 64, 256, 1024, 4096] {
let per_partition_mib = queue_mib(&config, partitions);
let per_partition_messages = queue_messages(&config, partitions);
assert!(
per_partition_mib <= shared_mib,
"partitions={partitions}: a single fenced producer must never be entitled \
to more than the shared producer's whole queue"
);
// The aggregate is the invariant: every individual value
// looked reasonable while the total scaled with the partition
// count, which is how the original defect hid.
// Either the whole fleet of producers fits the aggregate, or
// every one of them is already at the smallest queue
// librdkafka will take — past that point the budget cannot be
// honoured at all, and the honest answer is the floor rather
// than a number that quietly multiplies.
let aggregate_mib = per_partition_mib * partitions;
assert!(
aggregate_mib <= shared_mib || per_partition_mib == 1,
"partitions={partitions}: fenced producers together claim {aggregate_mib} MiB \
against a shared budget of {shared_mib} MiB, with room to divide further"
);
let aggregate_messages = per_partition_messages as u64 * partitions as u64;
assert!(
aggregate_messages <= shared_messages as u64 || per_partition_messages == 1,
"partitions={partitions}: fenced producers together claim \
{aggregate_messages} queued messages against {shared_messages}"
);
}
}
/// Dividing must not round a producer down to a depth that cannot
/// hold a window's worth of writes.
/// Overrides are the only way to reach the librdkafka limits the
/// derivation floors away from, so they are the only way these two
/// checks ever fire — and without them a pod starts with a producer
/// librdkafka refuses to create, or with no send timeout at all.
#[test]
fn overrides_outside_librdkafkas_limits_are_refused() {
for (txn_ms, message_ms, fragment) in [
(500u64, 0u32, "below librdkafka's minimum"),
(2000, 100, "is below"),
] {
let mut config = fenced(30);
config.fencing_txn_timeout_ms = txn_ms;
config.fencing_message_timeout_ms = message_ms;
let err = config
.validate_fencing_timescales()
.expect_err("an override outside librdkafka's limits must not start");
assert!(
err.contains(fragment),
"the refusal must say which limit it trips, got: {err}"
);
}
}
/// The pre-revoke self-fence allows three seconds for a whole drain,
/// and the drain is `wait_until_empty` *then* this wait. A budget
/// that ate the whole allowance would turn every shutdown with an
/// open window into a reported drain failure — while a budget under
/// the admission window cannot outwait the window it exists to
/// settle.
#[test]
fn settling_fits_inside_the_shutdown_fence_bound() {
for lease_ttl in [21, 30, 60, 300, 3599] {
let config = fenced(lease_ttl);
if config.validate_fencing_timescales().is_err() {
continue;
}
assert!(
config.fencing_settle_budget() < Duration::from_secs(3),
"LEASE_TTL={lease_ttl}: settle budget {:?} does not leave the drain room \
inside the self-fence's three-second allowance",
config.fencing_settle_budget()
);
// The wait exists to catch a committer that sleeps for the
// window and then commits, so a budget at or under the window
// can never see one fire: every drain with an open window
// would abandon the records it was meant to commit.
assert!(
config.fencing_settle_budget() > Duration::from_millis(config.fencing_window_ms),
"LEASE_TTL={lease_ttl}: settle budget {:?} cannot outwait the {}ms admission \
window it exists to settle",
config.fencing_settle_budget(),
config.fencing_window_ms
);
}
}
/// Acquisition is two broker round trips and happens fleet-wide
/// during a deploy. Nothing asserted on its budget, and tying it to
/// the write path's once already shortened it by a quarter as a side
/// effect of re-budgeting writes.
#[test]
fn acquisition_keeps_its_own_budget() {
for lease_ttl in [21, 30, 60, 300] {
let config = fenced(lease_ttl);
if config.validate_fencing_timescales().is_err() {
continue;
}
assert!(
config.fencing_init_timeout() >= Duration::from_secs(2),
"LEASE_TTL={lease_ttl}: acquisition needs room for a broker round trip"
);
// The timeout bounds each of the two calls acquisition makes
// — `fetch_metadata` then `init_transactions` — so the budget
// the lease has to cover is twice what it names.
assert!(
config.fencing_init_timeout() * 2 <= config.lease_fence_runway(),
"LEASE_TTL={lease_ttl}: a warm cannot outlive the lease it warms for"
);
}
}
fn queue_mib(config: &Config, partitions: u32) -> u32 {
let producers = crate::fencing::fenced_producer_count(partitions, config.fencing_lanes);
crate::fencing::queue_share(config.kafka.kafka_producer_queue_mib, producers)
}
fn queue_messages(config: &Config, partitions: u32) -> u32 {
let producers = crate::fencing::fenced_producer_count(partitions, config.fencing_lanes);
crate::fencing::queue_share(config.kafka.kafka_producer_queue_messages, producers)
}
#[test]
fn a_high_partition_count_still_leaves_a_workable_queue() {
let config = fenced(30);
// The budget is divided among every fenced producer, four lanes
// per partition at the default, and never rounds to a value
// librdkafka would reject.
assert_eq!(queue_mib(&config, 16), 6);
assert_eq!(queue_messages(&config, 16), 156_250);
assert!(queue_mib(&config, 1024) >= 1);
assert!(queue_messages(&config, 1024) >= 1);
assert!(
queue_mib(&config, 0) >= 1,
"partitions=0 must not divide by zero"
);
}
/// Zero lanes would leave a partition with nothing to produce on, and
/// an absurd count would open that many connections per acquisition.
#[test]
fn a_lane_count_outside_its_bounds_is_refused() {
for lanes in [0, MAX_FENCING_LANES + 1] {
let mut config = fenced(30);
config.fencing_lanes = lanes;
let err = config
.validate_fencing_timescales()
.expect_err("a lane count outside its bounds must be refused");
assert!(err.contains("FENCING_LANES"), "{err}");
}
}
/// A lease TTL long enough to derive past the broker's own ceiling
/// must be refused at startup, not discovered as an acquisition that
/// fails on every partition with only a counter to explain it.
#[test]
fn a_lease_ttl_that_outruns_the_broker_ceiling_is_refused() {
let err = fenced(3607)
.validate_fencing_timescales()
.expect_err("an hour-long lease derives a transaction timeout no broker accepts");
assert!(
err.contains("transaction.max.timeout.ms"),
"the refusal must name the broker setting it would trip, got: {err}"
);
// The band's edges, both sides: the last accepted TTL under the
// broker ceiling, and the first accepted one above the librdkafka
// floors once the drain slack is paid.
fenced(3606)
.validate_fencing_timescales()
.expect("LEASE_TTL=3606 sits just inside the broker ceiling");
fenced(26)
.validate_fencing_timescales()
.expect("LEASE_TTL=26 is the acceptance floor");
assert!(
fenced(25).validate_fencing_timescales().is_err(),
"below the floor, the librdkafka minimums cannot fit the drain room"
);
// And the production value must stay comfortably inside it.
fenced(30)
.validate_fencing_timescales()
.expect("LEASE_TTL=30 must remain usable");
}
/// The production lease TTL must actually be usable with fencing on,
/// or the flag could never be enabled.
#[test]
fn the_production_lease_ttl_supports_fencing() {
fenced(30)
.validate_fencing_timescales()
.expect("LEASE_TTL=30 must support fencing with derived timeouts");
}
#[test]
fn a_lease_ttl_too_short_for_fencing_is_rejected() {
assert!(fenced(5).validate_fencing_timescales().is_err());
}
#[test]
fn an_override_that_can_outlive_the_fence_is_rejected() {
let mut config = fenced(30);
config.fencing_txn_timeout_ms = 60_000;
assert!(config.validate_fencing_timescales().is_err());
}
#[test]
fn an_override_librdkafka_would_reject_is_caught() {
let mut config = fenced(30);
config.fencing_txn_timeout_ms = 1_000;
config.fencing_message_timeout_ms = 2_000;
let err = config
.validate_fencing_timescales()
.expect_err("must reject");
assert!(err.contains("librdkafka"), "got: {err}");
}
}