:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! `flags-cache-builder` — long-running Kafka consumer that rebuilds per-team
//! feature-flag caches off `flags_cache_invalidation` messages.
//!
//! Replaces the Django signal → Celery → Python build hot path with
//! Django signal → Kafka → Rust build. The producer (merged in PR #62817) is a
//! dumb "team X changed, rebuild it" trigger; this consumer reads fresh DB state
//! at build time, so out-of-order delivery can never stamp stale content.
//!
//! Loop shape (architecture doc, "coalesce within a small window"):
//! 1. Batch-fetch up to N messages, bounded by a ~500ms coalesce window.
//! 2. Dedupe by `team_id` — an edit session firing 20 saves becomes one build.
//! 3. Build each unique team once, with bounded retries.
//! 4. Route teams that exhaust their retry budget to the DLQ.
//! 5. Commit offsets as a batch (only after the build outcome is decided).
//!
//! The lazy request-path fill stays as the final safety net: a stuck consumer
//! degrades latency, not correctness.
//!
//! Messages marked `shadow: true` take a separate path: build the payload as
//! usual, then diff it against the live Redis entry instead of writing —
//! parity telemetry for teams the Python (Celery) builder still owns. Shadow
//! work never writes, never DLQs, and never touches the real-build metrics;
//! see `flags::cache_shadow`.
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};
use axum::{routing::get, Router};
use chrono::{DateTime, Utc};
use common_database::{get_pool, PostgresReader};
use common_hypercache::writer::HyperCacheWriter;
use common_hypercache::{HyperCacheError, HyperCacheReader, KeyType};
use common_kafka::config::{ConsumerConfig, KafkaConfig};
use common_kafka::kafka_consumer::{Offset, RecvErr, SingleTopicConsumer};
use common_kafka::kafka_producer::{
create_kafka_producer, send_keyed_iter_to_kafka_with_headers, KafkaContext,
};
use common_metrics::{setup_metrics_routes_for_product_with_overrides, Matcher};
use common_redis::CompressionConfig;
use common_types::TeamId;
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Handle, Manager};
use rdkafka::message::{Header, OwnedHeaders};
use rdkafka::producer::FutureProducer;
use strum::{EnumIter, IntoEnumIterator};
use tokio_util::sync::CancellationToken;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::EnvFilter;
use feature_flags::flags::cache_builder::build_flags_cache;
use feature_flags::flags::cache_invalidation::{FlagsCacheInvalidation, Source};
use feature_flags::flags::cache_shadow::{
diff_live_entry, summarize_diffs, MismatchTracker, ShadowIssueType, ShadowLiveEntry,
ShadowObservation, TrackerStoreOp,
};
use feature_flags::flags::cache_writer::{self, persist_flags_cache, PersistOutcome};
use feature_flags::server::create_redis_client;
common_alloc::used!();
const SERVICE_NAME: &str = "flags-cache-builder";
/// Defaults used when `KAFKA_CONSUMER_GROUP` / `KAFKA_CONSUMER_TOPIC` are unset.
/// The charts deployment sets both explicitly; these keep local dev runnable.
const DEFAULT_CONSUMER_GROUP: &str = "flags-cache-builder";
const DEFAULT_CONSUMER_TOPIC: &str = "flags_cache_invalidation";
/// Logging tag passed to `create_redis_client`; appears in connection-retry logs.
const REDIS_CLIENT_TYPE: &str = "flags-cache-builder";
const REDIS_CONNECT_RETRIES: u32 = 3;
/// Backoff base for build retries. Attempts sleep base·4^(n-1): 200ms, 800ms.
const RETRY_BASE_MS: u64 = 200;
const RETRY_BACKOFF_MULTIPLIER: u64 = 4;
/// Pause after a batch that yielded only Kafka receive errors, to avoid hot-looping
/// (and log/CPU spam) while the broker is unreachable.
const KAFKA_RECV_ERROR_BACKOFF: Duration = Duration::from_millis(500);
/// Cap on the `x-dlq-error` header so an unusually long error can't blow Kafka's
/// header/message size limit and fail the DLQ produce. The full error is logged.
const DLQ_ERROR_HEADER_MAX: usize = 1024;
// Metric names. End-to-end latency buckets follow the architecture doc.
const MESSAGES_RECEIVED: &str = "flags_cache_builder_messages_received_total";
const BUILDS_TOTAL: &str = "flags_cache_builder_builds_total";
const BUILD_RETRIES: &str = "flags_cache_builder_build_retries_total";
const BUILD_DURATION_SECONDS: &str = "flags_cache_builder_build_duration_seconds";
const E2E_LATENCY_SECONDS: &str = "flags_cache_builder_end_to_end_latency_seconds";
const PARSE_ERRORS: &str = "flags_cache_builder_parse_errors_total";
const KAFKA_RECV_ERRORS: &str = "flags_cache_builder_kafka_recv_errors_total";
const DLQ_PRODUCED: &str = "flags_cache_builder_dlq_produced_total";
const COALESCED_TEAMS: &str = "flags_cache_builder_coalesced_teams";
// The `result` label values on BUILDS_TOTAL and DLQ_PRODUCED. Shared with
// `precreate_counters` so a pre-created series and the series its emitter
// writes to cannot differ in spelling.
const RESULT_SUCCESS: &str = "success";
const RESULT_FAILURE: &str = "failure";
// Shadow-compare metrics. Deliberately disjoint from the real-build metrics:
// BUILDS_TOTAL{result=failure} feeds the FlagsCacheBuilderBuildFailureRate page,
// and a shadow build failing must never page — the team is still served by the
// Python builder. Every shadow build increments exactly one SHADOW_BUILDS outcome
// (match / mismatch_confirmed / mismatch_suppressed / live_entry_missing / error),
// so the unlabelled sum is the processed count.
const SHADOW_BUILDS: &str = "flags_cache_shadow_builds_total";
const SHADOW_MISMATCH: &str = "flags_cache_shadow_mismatch_total";
const SHADOW_MISMATCH_FIRST_SIGHT: &str = "flags_cache_shadow_mismatch_first_sight_total";
const SHADOW_FAILURES: &str = "flags_cache_shadow_build_failures_total";
// Failures of the mismatch tracker's Redis state, with an `op` label for which
// access failed (read / write / clear). Every one of them costs confirmations and
// none of them fail a build, so like the rest of the shadow telemetry this must
// not feed the real-build failure alert. A sustained rate here means
// SHADOW_MISMATCH is undercounting.
const SHADOW_TRACKER_STORE_ERRORS: &str = "flags_cache_shadow_tracker_store_errors_total";
// Per-team wall time for one shadow compare (build + live read + diff). Shadow
// teams run sequentially after the batch's real builds, so this is the quantity
// that sets how long a batch of shadow work delays the next real invalidation —
// the head-of-line span to watch during the producer-side ramp. Seconds-shaped,
// same buckets as the real-build duration histogram.
const SHADOW_BUILD_DURATION_SECONDS: &str = "flags_cache_shadow_build_duration_seconds";
/// Caps on the mismatch log lines (see `summarize_diffs`).
const SHADOW_LOG_MAX_ENTRIES: usize = 20;
const SHADOW_LOG_MAX_BYTES: usize = 4096;
/// Seconds buckets for end-to-end latency. The ladder runs to an hour because a
/// refresh message waits for the sweep that produced it to pace through its
/// batch, and an unbounded top bucket makes every `source="refresh"` quantile
/// read as `+Inf`. Every boundary at or below 60 s is unchanged, so edit-path
/// quantiles stay comparable across this change.
const E2E_LATENCY_BUCKETS: &[f64] = &[
0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 900.0, 1800.0, 3600.0,
];
/// Seconds buckets for the build-duration histogram. Our histograms are
/// seconds-shaped, so both are overridden off `common_metrics`' ms-shaped default.
const BUILD_DURATION_BUCKETS: &[f64] = &[
0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0,
];
/// `product` global label for per-product metric cost attribution.
const METRICS_PRODUCT: &str = "feature_flags";
#[derive(Envconfig)]
struct InfraConfig {
#[envconfig(
from = "READ_DATABASE_URL",
default = "postgres://posthog:posthog@localhost:5432/posthog"
)]
read_database_url: String,
/// Dedicated flags Redis tier. Required: refusing to fall back to the shared
/// `REDIS_URL` matches the warmer's `check_dedicated_cache_configured()` guard
/// so we never write flags-cache entries into the wrong Redis.
#[envconfig(from = "FLAGS_REDIS_URL", default = "")]
flags_redis_url: String,
#[envconfig(from = "OBJECT_STORAGE_BUCKET", default = "posthog")]
object_storage_bucket: String,
#[envconfig(from = "OBJECT_STORAGE_REGION", default = "us-east-1")]
object_storage_region: String,
#[envconfig(from = "OBJECT_STORAGE_ENDPOINT", default = "")]
object_storage_endpoint: String,
#[envconfig(from = "DATABASE_MAX_CONNECTIONS", default = "10")]
database_max_connections: u32,
#[envconfig(from = "FLAGS_REDIS_RESPONSE_TIMEOUT_MS", default = "1000")]
redis_response_timeout_ms: u64,
#[envconfig(from = "FLAGS_REDIS_CONNECTION_TIMEOUT_MS", default = "5000")]
redis_connection_timeout_ms: u64,
}
#[derive(Envconfig, Clone)]
struct BuilderConfig {
#[envconfig(from = "METRICS_PORT", default = "9090")]
metrics_port: u16,
/// Coalesce window: the longest we wait to fill a batch before building.
#[envconfig(from = "FLAGS_CACHE_COALESCE_WINDOW_MS", default = "500")]
coalesce_window_ms: u64,
#[envconfig(from = "FLAGS_CACHE_MAX_BATCH", default = "256")]
max_batch: usize,
#[envconfig(from = "FLAGS_CACHE_BUILD_MAX_ATTEMPTS", default = "3")]
build_max_attempts: u32,
/// Cache TTL on write. Matches Django's `FLAGS_CACHE_TTL` default (7 days).
#[envconfig(from = "FLAGS_CACHE_TTL", default = "604800")]
cache_ttl_seconds: u64,
#[envconfig(from = "KAFKA_DLQ_TOPIC", default = "flags_cache_invalidation_dlq")]
dlq_topic: String,
/// How long a team's last shadow mismatch stays eligible to confirm a repeat,
/// written as the TTL on the tracker's Redis key. Long enough that quiet
/// teams (two shadow builds days apart would miss a 1h window) still confirm
/// persistent drift; short enough that a disagreement Python has since
/// repaired stops counting as the team's previous observation.
#[envconfig(from = "FLAGS_CACHE_SHADOW_MISMATCH_TTL", default = "86400")]
shadow_mismatch_ttl_seconds: u64,
}
/// All offsets and timing for a single team's coalesced invalidations. Generic
/// over the offset type so the coalescing logic stays testable without
/// constructing `Offset` values (which have no public constructor).
struct TeamBatch<O = Offset> {
offsets: Vec<O>,
/// Oldest `emitted_at` across the coalesced messages — the worst-case
/// staleness the build resolves. Stamps the DLQ message if the build
/// ultimately fails, where worst case is the quantity triage wants.
oldest_emitted_at: DateTime<Utc>,
/// Oldest `emitted_at` across the coalesced *edits*, absent when the batch
/// holds none. Both the reported source and the latency the build is
/// credited with fall out of this one value, so neither can disagree with it.
oldest_edit_emitted_at: Option<DateTime<Utc>>,
}
impl<O> TeamBatch<O> {
/// Fold one coalesced message into the per-team map: append its offset and
/// keep both minimums. Both are a `min` over a set, so neither depends on
/// arrival order.
fn fold_into(
by_team: &mut HashMap<TeamId, TeamBatch<O>>,
team_id: TeamId,
emitted_at: DateTime<Utc>,
source: Source,
offset: O,
) {
let entry = by_team.entry(team_id).or_insert_with(|| TeamBatch {
offsets: Vec::new(),
oldest_emitted_at: emitted_at,
oldest_edit_emitted_at: None,
});
entry.oldest_emitted_at = entry.oldest_emitted_at.min(emitted_at);
if source == Source::Edit {
entry.oldest_edit_emitted_at = Some(
entry
.oldest_edit_emitted_at
.map_or(emitted_at, |oldest| oldest.min(emitted_at)),
);
}
entry.offsets.push(offset);
}
/// The path this build is reported as serving. One build serves every
/// coalesced message, and an edit is the one with a serve-latency
/// expectation, so a batch holding any edit is an edit however many sweep
/// messages it also absorbed.
fn reported_source(&self) -> Source {
if self.oldest_edit_emitted_at.is_some() {
Source::Edit
} else {
Source::Refresh
}
}
/// How long the reported path waited. An edit is credited with the oldest
/// edit's age, never the older refresh it coalesced with — otherwise the
/// sweep's minutes land in the edit histogram, which is the reading the
/// `source` label exists to keep separable. With no edit in the batch every
/// message is a refresh, so the batch minimum is already the refresh minimum.
fn attributed_emitted_at(&self) -> DateTime<Utc> {
self.oldest_edit_emitted_at
.unwrap_or(self.oldest_emitted_at)
}
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
init_tracing();
let infra = InfraConfig::init_from_env().expect("Invalid infrastructure configuration");
let builder_cfg = BuilderConfig::init_from_env().expect("Invalid builder configuration");
let kafka_cfg = KafkaConfig::init_from_env().expect("Invalid Kafka configuration");
ConsumerConfig::set_defaults(DEFAULT_CONSUMER_GROUP, DEFAULT_CONSUMER_TOPIC, false);
let consumer_cfg = ConsumerConfig::init_from_env().expect("Invalid consumer configuration");
if infra.flags_redis_url.is_empty() {
tracing::error!(
"FLAGS_REDIS_URL is not set. Refusing to start — that would risk writing \
flags-cache entries into the shared Redis tier. Set FLAGS_REDIS_URL."
);
std::process::exit(1);
}
tracing::info!(
consumer_group = consumer_cfg.kafka_consumer_group,
topic = consumer_cfg.kafka_consumer_topic,
dlq_topic = builder_cfg.dlq_topic,
coalesce_window_ms = builder_cfg.coalesce_window_ms,
max_batch = builder_cfg.max_batch,
build_max_attempts = builder_cfg.build_max_attempts,
cache_ttl_seconds = builder_cfg.cache_ttl_seconds,
"Starting {SERVICE_NAME}"
);
let mut manager = Manager::builder(SERVICE_NAME).build();
let metrics_handle = manager.register(
"metrics_server",
ComponentOptions::new().is_observability(true),
);
let main_handle = manager.register(
"main",
ComponentOptions::new()
.with_graceful_shutdown(Duration::from_secs(15))
.with_liveness_deadline(Duration::from_secs(60)),
);
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
let monitor = manager.monitor_background();
spawn_metrics_server(
metrics_handle,
readiness,
liveness,
builder_cfg.metrics_port,
);
let pg_pool = get_pool(&infra.read_database_url, infra.database_max_connections)
.expect("Failed to create database pool");
let pg_reader: PostgresReader = Arc::new(pg_pool);
let (writer, live_reader, redis_client) = build_cache_clients(&infra).await;
let writer = Arc::new(writer);
let live_reader = Arc::new(live_reader);
let consumer = SingleTopicConsumer::new(kafka_cfg.clone(), consumer_cfg)
.expect("Failed to create Kafka consumer");
let dlq_producer = create_kafka_producer(&kafka_cfg, main_handle.clone())
.await
.expect("Failed to create DLQ Kafka producer");
let loop_handle = main_handle.clone();
// Drive the loop off the lifecycle manager's own shutdown token so SIGTERM
// reaches the consumer directly. A private token would only be cancelled
// after `monitor.wait()` returns — i.e. after the graceful window already
// timed out — leaving the loop to be dropped mid-batch.
let loop_shutdown = loop_handle.shutdown_token();
tokio::spawn(async move {
let _guard = loop_handle.process_scope();
consume_loop(
consumer,
pg_reader,
writer,
live_reader,
redis_client,
dlq_producer,
builder_cfg,
loop_handle.clone(),
loop_shutdown,
)
.await;
});
monitor.wait().await?;
Ok(())
}
/// Register every counter series this binary can emit, at zero, so all of them
/// are present on `/metrics` from boot. Called from `spawn_metrics_server`,
/// immediately after the recorder install it depends on.
///
/// Counters are created lazily by `metrics::counter!`, so a series does not
/// exist until its first increment, and an absent series is indistinguishable
/// from a real zero: a silent DLQ producer reads exactly like a healthy one
/// that has had nothing to route. Two more consequences follow from that.
/// `increase()` needs a series to exist before an increment to count it, so the
/// first DLQ entry or the first parse error is invisible to a rate query. Alert
/// expressions also need live series to validate against, which has to be
/// possible outside an incident.
///
/// `posthog/tasks/hypercache_verification.py` pre-creates its label triples at
/// import for the same two reasons; this is the same guarantee on the Rust side.
///
/// Registration must not change what a counter reports, so every series is
/// incremented by zero. Histograms are left out: a histogram is only read as a
/// quantile or a rate over its buckets, neither of which misreads an absent
/// series as zero, and pre-creating them would publish a bucket ladder per
/// metric for no gain.
fn precreate_counters() {
for name in [
MESSAGES_RECEIVED,
BUILD_RETRIES,
PARSE_ERRORS,
KAFKA_RECV_ERRORS,
] {
metrics::counter!(name).increment(0);
}
// The success arm of BUILDS_TOTAL emits no `reason` (see `process_team`), so
// it is pre-created without one. Adding `reason="none"` there would change a
// live metric's label set to say nothing new. `source` changes the same label
// set deliberately: it splits a series that a sweep-driven build would
// otherwise share with the serve path, which is a distinction worth regrouping
// a panel for.
for source in Source::iter() {
metrics::counter!(BUILDS_TOTAL, "result" => RESULT_SUCCESS, "source" => source.as_label())
.increment(0);
}
metrics::counter!(DLQ_PRODUCED, "result" => RESULT_SUCCESS).increment(0);
metrics::counter!(DLQ_PRODUCED, "result" => RESULT_FAILURE).increment(0);
// The whole category set on both metrics. That publishes four series no
// emitter reaches: `BUILDS_TOTAL{reason=cache_parse}`, which only the
// shadow live-read produces, and `SHADOW_FAILURES` for `s3`, `serialize`
// and `other`, which only the real write produces. The alternative is a
// per-category map of which metric it reaches, and the compiler can force a
// new variant to fill that in but not to fill it in correctly.
for category in FailureCategory::iter() {
for source in Source::iter() {
metrics::counter!(BUILDS_TOTAL, "result" => RESULT_FAILURE, "reason" => category.as_label(), "source" => source.as_label())
.increment(0);
}
metrics::counter!(SHADOW_FAILURES, "category" => category.as_label()).increment(0);
}
for outcome in ShadowOutcomeLabel::iter() {
metrics::counter!(SHADOW_BUILDS, "outcome" => outcome.as_label()).increment(0);
}
for issue_type in ShadowIssueType::iter() {
metrics::counter!(SHADOW_MISMATCH, "issue_type" => issue_type.as_label()).increment(0);
metrics::counter!(SHADOW_MISMATCH_FIRST_SIGHT, "issue_type" => issue_type.as_label())
.increment(0);
}
for op in TrackerStoreOp::iter() {
metrics::counter!(SHADOW_TRACKER_STORE_ERRORS, "op" => op.as_label()).increment(0);
}
}
/// Build the cache writer (real builds) and a reader over the same Redis + config
/// (shadow builds), and hand back the Redis client they share. The reader is used
/// Redis-only via `get_typed_from_redis`, so its S3 client is never exercised;
/// sharing the writer's keeps one construction path and one set of connections.
/// The returned client is the shadow mismatch tracker's store, for the same
/// reason: one connection pool against `FLAGS_REDIS_URL`, not three.
async fn build_cache_clients(
infra: &InfraConfig,
) -> (
HyperCacheWriter,
HyperCacheReader,
Arc<dyn common_redis::Client + Send + Sync>,
) {
tracing::info!("Connecting to Redis");
let Some(redis_client) = create_redis_client(
&infra.flags_redis_url,
REDIS_CLIENT_TYPE,
CompressionConfig::default(),
infra.redis_response_timeout_ms,
infra.redis_connection_timeout_ms,
REDIS_CONNECT_RETRIES,
)
.await
else {
// create_redis_client logs the underlying error before returning None.
std::process::exit(1);
};
let redis_client: Arc<dyn common_redis::Client + Send + Sync> = redis_client;
let endpoint = Some(infra.object_storage_endpoint.as_str());
let s3_client = cache_writer::create_s3_client(&infra.object_storage_region, endpoint).await;
let config = cache_writer::make_cache_config(
&infra.object_storage_region,
&infra.object_storage_bucket,
endpoint,
);
let writer = HyperCacheWriter::new(redis_client.clone(), s3_client.clone(), config.clone());
let reader = HyperCacheReader::new_with_s3_client(redis_client.clone(), s3_client, config);
(writer, reader, redis_client)
}
/// The consumer hot loop. Returns when `shutdown` is cancelled (graceful drain).
#[allow(clippy::too_many_arguments)]
async fn consume_loop(
consumer: SingleTopicConsumer,
pg_reader: PostgresReader,
writer: Arc<HyperCacheWriter>,
live_reader: Arc<HyperCacheReader>,
redis_client: Arc<dyn common_redis::Client + Send + Sync>,
dlq_producer: FutureProducer<KafkaContext>,
cfg: BuilderConfig,
health: Handle,
shutdown: CancellationToken,
) {
let coalesce = Duration::from_millis(cfg.coalesce_window_ms);
let mismatch_tracker = MismatchTracker::new(
redis_client,
Duration::from_secs(cfg.shadow_mismatch_ttl_seconds),
);
loop {
// Reporting healthy each iteration covers the idle case: with no traffic
// the batch fetch returns empty after the coalesce window, so we still
// tick well inside the liveness deadline.
health.report_healthy();
let batch = tokio::select! {
_ = shutdown.cancelled() => break,
batch = consumer.json_recv_batch::<FlagsCacheInvalidation>(cfg.max_batch, coalesce) => batch,
};
if batch.is_empty() {
continue;
}
let coalesced = coalesce_batch(batch);
if coalesced.real.is_empty() && coalesced.shadow.is_empty() {
// Batch held only poison pills or receive errors. Poison offsets were
// auto-stored by json_recv, so commit to avoid reprocessing them.
commit_offsets(&consumer);
if coalesced.had_kafka_error {
// A receive error stores no offset, so there's nothing to make
// progress on until the broker recovers — back off rather than
// hot-loop on immediate errors.
tokio::time::sleep(KAFKA_RECV_ERROR_BACKOFF).await;
}
continue;
}
if !coalesced.real.is_empty() {
metrics::histogram!(COALESCED_TEAMS).record(coalesced.real.len() as f64);
}
let mut interrupted = false;
// Collect every processed offset and store the per-partition max once, at
// the end of the batch. `real`/`shadow` are HashMaps, so we build teams in
// arbitrary order; storing each team's offsets as we go could check-point a
// partition *backwards* (one partition carries many teams' interleaved
// messages), needlessly reprocessing on the next restart. See
// `store_max_offsets_per_partition`.
let mut batch_offsets: Vec<Offset> = Vec::new();
for (team_id, team_batch) in coalesced.real {
// Stop between teams once shutdown is signalled: a large batch (up to
// max_batch unique teams, each with retry backoff) could otherwise
// outrun the graceful-shutdown budget and be killed mid-build.
if shutdown.is_cancelled() {
interrupted = true;
break;
}
// Also tick per team: a large batch of unique teams (with retry
// backoff) could otherwise outlast the liveness deadline mid-batch.
health.report_healthy();
let offsets = process_team(
&pg_reader,
&writer,
&dlq_producer,
&cfg,
team_id,
team_batch,
)
.await;
batch_offsets.extend(offsets);
}
// Shadow teams run after every real build, so within a batch shadow
// work never delays a serve-path write. Across batches it can: the next
// fetch waits for this loop, so a shadow-heavy batch adds head-of-line
// delay to the real builds behind it. SHADOW_BUILD_DURATION_SECONDS measures
// that delay; the producer-side gate is the lever if it grows.
if !interrupted {
for (team_id, team_batch) in coalesced.shadow {
if shutdown.is_cancelled() {
interrupted = true;
break;
}
health.report_healthy();
let offsets = process_shadow_team(
&pg_reader,
&live_reader,
&mismatch_tracker,
team_id,
team_batch,
)
.await;
batch_offsets.extend(offsets);
}
}
if interrupted {
// Don't store or commit a partially processed batch. Kafka commits a
// single per-partition high-water mark, and one partition holds many
// teams' messages interleaved (each team is keyed to a partition, but a
// partition carries many teams). Advancing the commit point now would
// move it past the offsets of teams we haven't built yet — including
// any with a *lower* offset than a team we did build — dropping their
// invalidations for good: the cache keeps serving the stale entry, and
// the lazy request-path fill only rebuilds on a miss. Leaving
// `batch_offsets` unstored means the whole batch re-delivers on
// restart; builds are idempotent, so reprocessing finished teams is
// cheap.
break;
}
store_max_offsets_per_partition(batch_offsets);
commit_offsets(&consumer);
}
// On a mid-batch interrupt we break *before* committing, so the commit point
// is the last fully processed batch — not necessarily the last one fetched.
tracing::info!("Consumer loop draining; offsets committed up to last fully processed batch");
}
/// A fetched batch deduped by `team_id`, split by delivery mode. Real and shadow
/// invalidations never coalesce with each other: a shadow message must not be
/// absorbed into a real build (that would serve-write a team Python owns), and a
/// real message must not be downgraded into a compare-only pass.
struct CoalescedBatch {
real: HashMap<TeamId, TeamBatch>,
shadow: HashMap<TeamId, TeamBatch>,
had_kafka_error: bool,
}
/// Fold one message into the map matching its delivery mode. Split out from
/// `coalesce_batch` so the real/shadow routing is testable without constructing
/// `Offset` values.
fn fold_message<O>(
real: &mut HashMap<TeamId, TeamBatch<O>>,
shadow: &mut HashMap<TeamId, TeamBatch<O>>,
team_id: TeamId,
is_shadow: bool,
emitted_at: DateTime<Utc>,
source: Source,
offset: O,
) {
let target = if is_shadow { shadow } else { real };
TeamBatch::fold_into(target, team_id, emitted_at, source, offset);
}
/// Dedupe a fetched batch by `team_id` within each delivery mode, counting
/// received messages and errors.
/// Poison pills (parse failures) already had their offsets stored by `json_recv`;
/// Kafka receive errors stored nothing. Returns the per-team work plus whether a
/// Kafka receive error occurred, so the caller can back off instead of hot-looping
/// while the broker is unreachable.
fn coalesce_batch(batch: Vec<Result<(FlagsCacheInvalidation, Offset), RecvErr>>) -> CoalescedBatch {
let mut real: HashMap<TeamId, TeamBatch> = HashMap::new();
let mut shadow: HashMap<TeamId, TeamBatch> = HashMap::new();
let mut received: u64 = 0;
let mut had_kafka_error = false;
for result in batch {
match result {
Ok((msg, offset)) => {
received += 1;
fold_message(
&mut real,
&mut shadow,
msg.team_id,
msg.shadow,
msg.emitted_at,
msg.source,
offset,
);
}
// A receive error is a broker/transport problem, not a bad message:
// nothing was consumed and no offset was stored. Track it apart from
// poison pills so `parse_errors` stays meaningful.
Err(RecvErr::Kafka(e)) => {
had_kafka_error = true;
metrics::counter!(KAFKA_RECV_ERRORS).increment(1);
tracing::warn!(error = %e, "Kafka receive error");
}
Err(e) => {
metrics::counter!(PARSE_ERRORS).increment(1);
tracing::warn!(error = %e, "Skipping unparseable invalidation (offset auto-stored)");
}
}
}
metrics::counter!(MESSAGES_RECEIVED).increment(received);
CoalescedBatch {
real,
shadow,
had_kafka_error,
}
}
/// Build one team's cache, routing to the DLQ on terminal failure, and return the
/// team's offsets for the caller to store. Offsets are returned (and later stored)
/// regardless of build/DLQ outcome — a poison message must not wedge the partition
/// forever; the DLQ is the durable record for triage.
async fn process_team(
pg_reader: &PostgresReader,
writer: &HyperCacheWriter,
dlq_producer: &FutureProducer<KafkaContext>,
cfg: &BuilderConfig,
team_id: TeamId,
team_batch: TeamBatch,
) -> Vec<Offset> {
let source = team_batch.reported_source().as_label();
match build_with_retry(pg_reader, writer, team_id, cfg).await {
Ok(()) => {
metrics::counter!(BUILDS_TOTAL, "result" => RESULT_SUCCESS, "source" => source)
.increment(1);
let latency = (Utc::now() - team_batch.attributed_emitted_at())
.num_milliseconds()
.max(0) as f64
/ 1000.0;
metrics::histogram!(E2E_LATENCY_SECONDS, "source" => source).record(latency);
}
Err(failure) => {
metrics::counter!(BUILDS_TOTAL, "result" => RESULT_FAILURE, "reason" => failure.category.as_label(), "source" => source)
.increment(1);
tracing::error!(team_id, source, category = failure.category.as_label(), error = %failure.message, "Cache build failed after retries; routing to DLQ");
// The message is a trigger, not a payload, so reconstruct it for the
// DLQ from the team, its oldest coalesced timestamp and the path that
// asked for the build — a replayed sweep must not arrive as an edit.
let dlq_message = FlagsCacheInvalidation::new(
team_id,
team_batch.oldest_emitted_at,
team_batch.reported_source(),
);
dlq_produce(dlq_producer, &cfg.dlq_topic, &dlq_message, &failure).await;
}
}
// Return offsets regardless of build/DLQ outcome. Kafka commits a single
// per-partition high-water mark, not a set, so we can't selectively skip one
// failed message's offset while committing later ones from the same partition
// — the later commit subsumes it. A DLQ-produce failure (rare; same cluster
// as the source topic) therefore loses the triage record, but never flag data:
// the lazy request-path fill rebuilds on the next /flags request. That failure
// is surfaced loudly via the error log above and the DLQ_PRODUCED{result=failure}
// counter — alert on it rather than wedging the partition.
team_batch.offsets
}
/// Outcome of a single shadow compare. Each outcome carries exactly one
/// `SHADOW_BUILDS` label, via `label`.
enum ShadowOutcome {
Match,
Mismatch(ShadowObservation),
/// No live Redis entry to compare against — Python may simply not have
/// built the team yet. Counted, never alarmed on.
LiveEntryMissing,
/// The shadow build or the live-entry read failed. Dropped and counted on
/// the shadow-only failure counter — never the real-build failure metric
/// (which pages) and never the DLQ (which is the real path's triage queue).
Failed(BuildFailure),
}
/// The `SHADOW_BUILDS{outcome}` label values. Held apart from `ShadowOutcome`
/// because the two are not 1:1: `Mismatch` reports as `mismatch_suppressed` or
/// `mismatch_confirmed` depending on its payload. Iterating this enum is what
/// lets `precreate_counters` cover the label set without a hand-written list of
/// strings, and `ShadowOutcome::label` must map every outcome into it, so
/// neither side can gain a value the other misses.
#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter)]
enum ShadowOutcomeLabel {
Match,
LiveEntryMissing,
Error,
MismatchSuppressed,
MismatchConfirmed,
}
impl ShadowOutcomeLabel {
fn as_label(&self) -> &'static str {
match self {
Self::Match => "match",
Self::LiveEntryMissing => "live_entry_missing",
Self::Error => "error",
Self::MismatchSuppressed => "mismatch_suppressed",
Self::MismatchConfirmed => "mismatch_confirmed",
}
}
}
impl ShadowOutcome {
/// Every outcome maps to exactly one label, so the counter's unlabelled sum
/// is the processed count.
fn label(&self) -> ShadowOutcomeLabel {
match self {
Self::Match => ShadowOutcomeLabel::Match,
Self::LiveEntryMissing => ShadowOutcomeLabel::LiveEntryMissing,
Self::Failed(_) => ShadowOutcomeLabel::Error,
Self::Mismatch(observation) if observation.confirmed.is_empty() => {
ShadowOutcomeLabel::MismatchSuppressed
}
Self::Mismatch(_) => ShadowOutcomeLabel::MismatchConfirmed,
}
}
}
/// Run one team's shadow compare and emit its telemetry. Never writes to the
/// cache (no writer in reach), never DLQs, and always returns the offsets so a
/// failing shadow message can't wedge the partition.
async fn process_shadow_team(
pg_reader: &PostgresReader,
live_reader: &HyperCacheReader,
tracker: &MismatchTracker,
team_id: TeamId,
team_batch: TeamBatch,
) -> Vec<Offset> {
let start = Instant::now();
let outcome = shadow_compare(pg_reader, live_reader, tracker, team_id).await;
// Recorded for every outcome: the per-team wall time delays the next batch
// whether the compare matched, mismatched, or failed.
metrics::histogram!(SHADOW_BUILD_DURATION_SECONDS).record(start.elapsed().as_secs_f64());
metrics::counter!(SHADOW_BUILDS, "outcome" => outcome.label().as_label()).increment(1);
match outcome {
ShadowOutcome::Match | ShadowOutcome::LiveEntryMissing => {}
ShadowOutcome::Failed(failure) => {
metrics::counter!(SHADOW_FAILURES, "category" => failure.category.as_label())
.increment(1);
tracing::warn!(team_id, category = failure.category.as_label(), error = %failure.message, "Shadow build failed; dropping (not DLQ'd)");
}
ShadowOutcome::Mismatch(observation) => {
for diff in &observation.confirmed {
metrics::counter!(SHADOW_MISMATCH, "issue_type" => diff.issue_type.as_label())
.increment(1);
}
for diff in &observation.first_sight {
metrics::counter!(SHADOW_MISMATCH_FIRST_SIGHT, "issue_type" => diff.issue_type.as_label())
.increment(1);
}
// A first sighting logs at WARN and a confirmed mismatch at ERROR,
// because a single-shot disagreement is expected: a shadow build races
// Python's own rebuild. `mismatch_stage` splits the two in a log query
// without a text match on the message.
if !observation.confirmed.is_empty() {
tracing::error!(
team_id,
mismatch_stage = "confirmed",
diff = %summarize_diffs(&observation.confirmed, SHADOW_LOG_MAX_ENTRIES, SHADOW_LOG_MAX_BYTES),
"Shadow compare mismatch persisted across consecutive builds"
);
}
if !observation.first_sight.is_empty() {
tracing::warn!(
team_id,
mismatch_stage = "first_sight",
diff = %summarize_diffs(&observation.first_sight, SHADOW_LOG_MAX_ENTRIES, SHADOW_LOG_MAX_BYTES),
"Shadow compare mismatch seen for the first time"
);
}
}
}
team_batch.offsets
}
/// Build the team's payload exactly as a real invalidation would, then diff it
/// against the live Redis entry instead of persisting it. Redis-only read: the
/// point is what the serve path's cache tier holds right now, and an S3 cascade
/// would blur "Python hasn't built this team yet" into a comparison.
///
/// The read shares the hypercache reader's miss-reason counter
/// (`hypercache_redis_miss_reason{reason="not_found"}`), so shadow builds of
/// teams Python hasn't built yet count there too. The builder is its own scrape
/// job, so dashboards scoped to the flags service are unaffected — but
/// aggregations of that counter across jobs should filter the builder out.
async fn shadow_compare(
pg_reader: &PostgresReader,
live_reader: &HyperCacheReader,
tracker: &MismatchTracker,
team_id: TeamId,
) -> ShadowOutcome {
let built = match build_flags_cache(pg_reader.clone(), team_id).await {
Ok(built) => built,
Err(e) => return ShadowOutcome::Failed(BuildFailure::database(e)),
};
let live = match live_reader
.get_typed_from_redis::<ShadowLiveEntry>(&KeyType::int(team_id))
.await
{
Ok(Some(live)) => live,
// The `__missing__` sentinel and an absent key both mean "nothing to
// compare against", not drift.
Ok(None) | Err(HyperCacheError::CacheMiss) => return ShadowOutcome::LiveEntryMissing,
Err(e) => return ShadowOutcome::Failed(BuildFailure::from_live_read(e)),
};
let diffs = diff_live_entry(&built, &live);
let observation = tracker.observe(team_id, diffs, SystemTime::now()).await;
// Counted here and not next to the outcome metrics, because a clean build can
// fail to clear its pending state, and that build reports as a match.
for op in &observation.store_errors {
metrics::counter!(SHADOW_TRACKER_STORE_ERRORS, "op" => op.as_label()).increment(1);
}
if observation.is_match() {
ShadowOutcome::Match
} else {
ShadowOutcome::Mismatch(observation)
}
}
/// The tier that failed, carried on `BuildFailure` and emitted as the `reason`
/// label on `BUILDS_TOTAL` and the `category` label on `SHADOW_FAILURES`. An
/// enum rather than a bare `&'static str` so the label vocabulary is one closed
/// set: `precreate_counters` iterates it, so a new tier cannot reach a metric
/// without also being pre-created at zero.
#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter)]
enum FailureCategory {
Database,
CacheParse,
Redis,
S3,
Serialize,
Other,
}
impl FailureCategory {
fn as_label(&self) -> &'static str {
match self {
Self::Database => "database",
Self::CacheParse => "cache_parse",
Self::Redis => "redis",
Self::S3 => "s3",
Self::Serialize => "serialize",
Self::Other => "other",
}
}
}
/// A terminal build failure tagged with the tier that failed, so the error metric
/// and DLQ headers can attribute it — that tier (database / redis / s3 / serialize)
/// is the triage signal the DLQ exists to provide.
struct BuildFailure {
category: FailureCategory,
message: String,
}
impl BuildFailure {
/// A failure reading flag/cohort state from Postgres (the `build_flags_cache`
/// step). The whole step is DB-bound on this path, so it's attributed wholesale.
fn database(err: impl std::fmt::Display) -> Self {
Self {
category: FailureCategory::Database,
message: err.to_string(),
}
}
/// A failure reading the live entry during a shadow compare. `Json`/`Pickle`
/// mean the cached bytes didn't parse into the typed model (`cache_parse` —
/// distinct from the write path's `serialize`); everything else is the Redis
/// tier, including timeouts. `CacheMiss` never reaches here — the caller
/// maps it to `LiveEntryMissing` first.
fn from_live_read(err: HyperCacheError) -> Self {
let category = match err {
HyperCacheError::Json(_) | HyperCacheError::Pickle(_) => FailureCategory::CacheParse,
_ => FailureCategory::Redis,
};
Self {
category,
message: err.to_string(),
}
}
/// A failure persisting the built cache, attributed to the HyperCache tier that
/// failed. `Timeout`/`CacheMiss`/`Pickle` don't occur on the write path, so they
/// fall through to `other`.
fn from_persist(err: HyperCacheError) -> Self {
let category = match err {
HyperCacheError::Redis(_) => FailureCategory::Redis,
HyperCacheError::S3(_) => FailureCategory::S3,
HyperCacheError::Json(_) => FailureCategory::Serialize,
_ => FailureCategory::Other,
};
Self {
category,
message: err.to_string(),
}
}
}
async fn build_with_retry(
pg_reader: &PostgresReader,
writer: &HyperCacheWriter,
team_id: TeamId,
cfg: &BuilderConfig,
) -> Result<(), BuildFailure> {
let mut attempt: u32 = 0;
loop {
attempt += 1;
let start = Instant::now();
match build_once(pg_reader, writer, team_id, cfg.cache_ttl_seconds).await {
Ok(outcome) => {
let elapsed = start.elapsed();
metrics::histogram!(BUILD_DURATION_SECONDS).record(elapsed.as_secs_f64());
tracing::info!(
team_id,
attempt,
duration_ms = elapsed.as_millis() as u64,
size_bytes = outcome.size_bytes,
etag = %outcome.etag,
"Built flags cache"
);
return Ok(());
}
Err(failure) => {
if attempt >= cfg.build_max_attempts {
return Err(failure);
}
metrics::counter!(BUILD_RETRIES).increment(1);
let backoff = retry_backoff(attempt);
tracing::warn!(
team_id,
attempt,
category = failure.category.as_label(),
error = %failure.message,
backoff_ms = backoff.as_millis() as u64,
"Cache build attempt failed; retrying"
);
tokio::time::sleep(backoff).await;
}
}
}
}
/// Backoff before the `attempt`-th retry (1-indexed): `base · multiplier^(attempt-1)`,
/// i.e. 200ms, 800ms, … `saturating_pow`/`saturating_mul` keep an unusually high
/// `build_max_attempts` from overflowing rather than panicking in debug.
fn retry_backoff(attempt: u32) -> Duration {
Duration::from_millis(
RETRY_BASE_MS.saturating_mul(RETRY_BACKOFF_MULTIPLIER.saturating_pow(attempt - 1)),
)
}
async fn build_once(
pg_reader: &PostgresReader,
writer: &HyperCacheWriter,
team_id: TeamId,
ttl_seconds: u64,
) -> Result<PersistOutcome, BuildFailure> {
let cache = build_flags_cache(pg_reader.clone(), team_id)
.await
.map_err(BuildFailure::database)?;
persist_flags_cache(writer, team_id, &cache, ttl_seconds)
.await
.map_err(BuildFailure::from_persist)
}
/// Cap an error string to `DLQ_ERROR_HEADER_MAX` bytes for use as a Kafka header
/// value, truncating on a char boundary so the result stays valid UTF-8.
fn truncate_for_header(error: &str) -> String {
const ELLIPSIS: &str = "…";
const _: () = assert!(
DLQ_ERROR_HEADER_MAX > ELLIPSIS.len(),
"DLQ_ERROR_HEADER_MAX must be larger than the ellipsis marker"
);
if error.len() <= DLQ_ERROR_HEADER_MAX {
return error.to_string();
}
// Reserve room for the ellipsis so the result — marker included — stays within
// the byte cap. The ellipsis is 3 bytes, so slicing to the cap and appending it
// would otherwise overshoot.
let mut end = DLQ_ERROR_HEADER_MAX - ELLIPSIS.len();
while !error.is_char_boundary(end) {
end -= 1;
}
format!("{}{ELLIPSIS}", &error[..end])
}
/// Re-produce the failed invalidation to the DLQ verbatim (so it stays a valid
/// v1 message, replayable by this same consumer), with the failure reason in
/// headers. Keyed by `team_id` to preserve partition affinity.
async fn dlq_produce(
dlq_producer: &FutureProducer<KafkaContext>,
topic: &str,
message: &FlagsCacheInvalidation,
failure: &BuildFailure,
) {
let failed_at = Utc::now().to_rfc3339();
let team_key = message.team_id.to_string();
let error_header = truncate_for_header(&failure.message);
let category = failure.category.as_label();
let results = send_keyed_iter_to_kafka_with_headers(
dlq_producer,
topic,
|_: &FlagsCacheInvalidation| Some(team_key.clone()),
|_: &FlagsCacheInvalidation| {
Some(
OwnedHeaders::new()
.insert(Header {
key: "x-dlq-error",
value: Some(error_header.as_str()),
})
// The failed tier (database / redis / s3 / serialize / other),
// split out so triage can filter the DLQ by cause without parsing
// the free-form error message.
.insert(Header {
key: "x-dlq-error-category",
value: Some(category),
})
.insert(Header {
key: "x-dlq-failed-at",
value: Some(failed_at.as_str()),
}),
)
},
std::iter::once(message.clone()),
)
.await;
match results.into_iter().next() {
Some(Ok(())) => metrics::counter!(DLQ_PRODUCED, "result" => RESULT_SUCCESS).increment(1),
Some(Err(e)) => {
metrics::counter!(DLQ_PRODUCED, "result" => RESULT_FAILURE).increment(1);
tracing::error!(team_id = message.team_id, error = %e, "Failed to produce to DLQ");
}
None => {}
}
}
/// Store the highest offset per partition and drop the rest. A committed offset N
/// implies every offset ≤ N on that partition is committed, so keeping only the max
/// is sufficient — and necessary: teams are processed in arbitrary `HashMap` order,
/// and rdkafka's `store_offset` overwrites the stored offset for a `(topic,
/// partition)` rather than keeping the max, so storing each offset blindly could
/// regress a partition's commit point. That only ever causes idempotent
/// reprocessing on the next restart (never dropped invalidations), but keeping the
/// commit point monotonic avoids it. Max-per-partition is safe because
/// `json_recv_batch` stops at the first receive error, so a batch is a contiguous
/// run of messages per partition with no gaps below the max.
///
/// This makes the *built teams'* offsets monotonic among themselves. Poison pills
/// auto-store their (possibly higher) offset inside `json_recv` and aren't visible
/// here, so absolute monotonicity across all messages isn't guaranteed — that
/// residual is pre-existing and benign (idempotent re-skip on replay).
fn store_max_offsets_per_partition(offsets: Vec<Offset>) {
for offset in max_per_partition(offsets, Offset::partition, Offset::get_value) {
if let Err(e) = offset.store() {
tracing::warn!(error = %e, "Failed to store offset");
}
}
}
/// Reduce `items` to the single highest-`value` item per `partition`, dropping the
/// rest. Input order is irrelevant — the result holds one item per distinct
/// partition, each the max by `value`. Extracted from `store_max_offsets_per_partition`
/// so the selection logic is testable without constructing `Offset` values.
fn max_per_partition<T>(
items: Vec<T>,
partition: impl Fn(&T) -> i32,
value: impl Fn(&T) -> i64,
) -> Vec<T> {
let mut highest: HashMap<i32, T> = HashMap::new();
for item in items {
match highest.entry(partition(&item)) {
Entry::Occupied(mut e) if value(&item) > value(e.get()) => {
e.insert(item);
}
Entry::Occupied(_) => {}
Entry::Vacant(e) => {
e.insert(item);
}
}
}
highest.into_values().collect()
}
fn commit_offsets(consumer: &SingleTopicConsumer) {
if let Err(e) = consumer.commit() {
tracing::warn!(error = %e, "Failed to commit offsets");
}
}
fn init_tracing() {
tracing_subscriber::fmt()
.with_env_filter(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy(),
)
.json()
.init();
}
/// Install the Prometheus recorder and serve `/metrics` plus the health routes.
///
/// The recorder install stays inside the spawned task, and `precreate_counters`
/// runs directly after it in that same task. Two reasons it sits here rather
/// than in `main`. Straight-line order in one task is what guarantees the
/// registrations reach the installed recorder, because `metrics::counter!`
/// writes to whichever recorder is installed at the moment it runs. And
/// `install_recorder` builds a `quanta::Clock`, whose TSC calibration busy-spins
/// for up to 200ms, so running it here keeps that off the startup critical path
/// and overlaps it with the database, Redis, and Kafka client setup in `main`.
fn spawn_metrics_server(
handle: lifecycle::Handle,
readiness: lifecycle::ReadinessHandler,
liveness: lifecycle::LivenessHandler,
port: u16,
) {
tokio::spawn(async move {
let _guard = handle.process_scope();
let health_router = Router::new()
.route(
"/_readiness",
get(move || {
let r = readiness.clone();
async move { r.check().await }
}),
)
.route("/_liveness", get(move || async move { liveness.check() }));
// Reuse the crate's shared recorder/router setup (prometheus install +
// /metrics + product label + HTTP metrics middleware), overriding the
// seconds-shaped histograms off its ms-shaped default buckets.
let overrides = [
(
Matcher::Full(BUILD_DURATION_SECONDS.to_string()),
BUILD_DURATION_BUCKETS,
),
(
Matcher::Full(E2E_LATENCY_SECONDS.to_string()),
E2E_LATENCY_BUCKETS,
),
(
Matcher::Full(SHADOW_BUILD_DURATION_SECONDS.to_string()),
BUILD_DURATION_BUCKETS,
),
];
let router = setup_metrics_routes_for_product_with_overrides(
health_router,
METRICS_PRODUCT,
&overrides,
);
precreate_counters();
let bind = format!("0.0.0.0:{port}");
let listener = tokio::net::TcpListener::bind(&bind)
.await
.expect("Failed to bind metrics port");
tracing::info!("Metrics server listening on {}", bind);
axum::serve(listener, router)
.with_graceful_shutdown(handle.shutdown_signal())
.await
.expect("Metrics server error");
});
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use chrono::{DateTime, TimeZone, Utc};
use rstest::rstest;
use super::{
fold_message, max_per_partition, precreate_counters, retry_backoff, truncate_for_header,
BuildFailure, ShadowOutcome, ShadowOutcomeLabel, Source, TeamBatch, DLQ_ERROR_HEADER_MAX,
};
// (partition, offset) pairs; keyed and valued by the two fields.
fn reduce(items: Vec<(i32, i64)>) -> Vec<(i32, i64)> {
let mut out = max_per_partition(items, |&(p, _)| p, |&(_, o)| o);
out.sort_unstable();
out
}
fn ts(secs: i64) -> DateTime<Utc> {
Utc.timestamp_opt(secs, 0)
.single()
.expect("valid timestamp")
}
/// Fold a list of (team_id, emitted_at, source, offset) into per-team batches
/// via the same `TeamBatch::fold_into` the consumer uses, and hand back the
/// batches themselves so each test reads the fields it cares about. Offset
/// type is `u64` here — the production `Offset` has no public constructor,
/// which is why the helper is generic.
fn fold(items: Vec<(i32, DateTime<Utc>, Source, u64)>) -> HashMap<i32, TeamBatch<u64>> {
let mut by_team: HashMap<i32, TeamBatch<u64>> = HashMap::new();
for (team_id, emitted_at, source, offset) in items {
TeamBatch::fold_into(&mut by_team, team_id, emitted_at, source, offset);
}
by_team
}
/// Every message an edit, which is what the tests that only care about
/// offsets and timestamps want.
fn fold_edits(items: Vec<(i32, DateTime<Utc>, u64)>) -> HashMap<i32, TeamBatch<u64>> {
fold(
items
.into_iter()
.map(|(team_id, emitted_at, offset)| (team_id, emitted_at, Source::Edit, offset))
.collect(),
)
}
#[tokio::test]
async fn build_once_persists_the_team_without_its_unsupported_flags() {
use common_hypercache::writer::HyperCacheWriter;
use common_redis::MockRedisClient;
use feature_flags::flags::cache_writer::make_cache_config;
use feature_flags::utils::test_utils::{
dummy_s3_client, insert_v1_v2_and_unsupported_flags, published_flag_keys, TestContext,
};
use std::sync::Arc;
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await.unwrap();
insert_v1_v2_and_unsupported_flags(&context, team.id).await;
let redis = Arc::new(MockRedisClient::new());
let writer = HyperCacheWriter::new(
redis.clone(),
dummy_s3_client(),
make_cache_config("us-east-1", "test-bucket", None),
);
let outcome = super::build_once(&context.non_persons_reader, &writer, team.id, 60)
.await
.unwrap_or_else(|failure| panic!("{}", failure.message));
assert_eq!(published_flag_keys(&redis), ["v1-flag", "v2-flag"]);
assert!(!outcome.etag.is_empty());
}
#[test]
fn coalesce_keeps_oldest_emitted_at_regardless_of_arrival_order() {
// Same team, timestamps arriving newest-first: the oldest must still win,
// since that worst-case staleness is what stamps the DLQ message.
let got = fold_edits(vec![(7, ts(300), 0), (7, ts(100), 1), (7, ts(200), 2)]);
assert_eq!(got[&7].oldest_emitted_at, ts(100));
assert_eq!(got[&7].offsets, vec![0, 1, 2]);
}
#[rstest]
#[case::edit_then_older_refresh(vec![(7, ts(200), Source::Edit, 0), (7, ts(100), Source::Refresh, 1)])]
#[case::older_refresh_then_edit(vec![(7, ts(100), Source::Refresh, 0), (7, ts(200), Source::Edit, 1)])]
fn an_edit_records_its_own_age_not_the_refresh_it_coalesced_with(
#[case] items: Vec<(i32, DateTime<Utc>, Source, u64)>,
) {
// One build serves both messages, and it is reported as the edit: the
// edit is the one with a serve expectation. It must be credited with its
// own age, because reading the batch minimum instead would put the
// sweep's minutes into the edit path's sub-second p99 — the reading the
// `source` label exists to keep separable.
let batch = &fold(items)[&7];
assert_eq!(batch.reported_source(), Source::Edit);
assert_eq!(batch.attributed_emitted_at(), ts(200));
// The DLQ stamp still wants worst-case staleness across the whole batch.
assert_eq!(batch.oldest_emitted_at, ts(100));
}
#[test]
fn a_batch_of_only_refreshes_stays_a_refresh() {
let batch = &fold(vec![
(7, ts(200), Source::Refresh, 0),
(7, ts(100), Source::Refresh, 1),
])[&7];
assert_eq!(batch.reported_source(), Source::Refresh);
// With no edit in the batch the attributed age is the batch minimum.
assert_eq!(batch.attributed_emitted_at(), ts(100));
assert_eq!(batch.oldest_emitted_at, ts(100));
}
/// Route a list of (team_id, shadow, emitted_at, source, offset) through the
/// same `fold_message` the consumer uses and hand back both maps.
fn route(
items: Vec<(i32, bool, DateTime<Utc>, Source, u64)>,
) -> (HashMap<i32, TeamBatch<u64>>, HashMap<i32, TeamBatch<u64>>) {
let mut real: HashMap<i32, TeamBatch<u64>> = HashMap::new();
let mut shadow: HashMap<i32, TeamBatch<u64>> = HashMap::new();
for (team_id, is_shadow, emitted_at, source, offset) in items {
fold_message(
&mut real,
&mut shadow,
team_id,
is_shadow,
emitted_at,
source,
offset,
);
}
(real, shadow)
}
#[test]
fn shadow_messages_never_reach_the_real_build_map() {
// The real map is the only route to a cache write; a shadow message
// landing there would serve-write a team Python still owns.
let (real, shadow) = route(vec![
(7, true, ts(100), Source::Edit, 0),
(7, true, ts(200), Source::Edit, 1),
]);
assert!(real.is_empty());
assert_eq!(shadow[&7].offsets, vec![0, 1]);
}
#[test]
fn real_and_shadow_messages_for_one_team_do_not_coalesce() {
// A mixed batch must produce both a real build and a shadow compare —
// absorbing either into the other changes what gets written.
let (real, shadow) = route(vec![
(7, false, ts(100), Source::Edit, 0),
(7, true, ts(200), Source::Edit, 1),
]);
assert_eq!(real[&7].offsets, vec![0]);
assert_eq!(shadow[&7].offsets, vec![1]);
}
#[test]
fn absent_shadow_field_routes_to_the_real_build_map() {
// Pre-shadow producers keep exactly today's behavior: `shadow` is absent
// on the wire, deserializes to false, and the message builds for real.
let msg: super::FlagsCacheInvalidation = serde_json::from_str(
r#"{"version": 1, "team_id": 7, "operation": "invalidate", "emitted_at": "2026-04-23T10:37:00Z"}"#,
)
.expect("v1 message without shadow must parse");
assert!(!msg.shadow);
let (real, shadow) = route(vec![(
msg.team_id,
msg.shadow,
msg.emitted_at,
msg.source,
0,
)]);
assert_eq!(real[&7].offsets, vec![0]);
assert!(shadow.is_empty());
}
#[test]
fn a_wire_refresh_reaches_the_batch_as_a_refresh() {
// Ties the deserialized `source` to the batch the builder reports on. The
// coalescing tests build `Source` values in memory, so without this the
// wire value could stop reaching `fold_message` and nothing would fail.
let msg: super::FlagsCacheInvalidation = serde_json::from_str(
r#"{"version": 1, "team_id": 7, "operation": "invalidate", "emitted_at": "2026-04-23T10:37:00Z", "source": "refresh"}"#,
)
.expect("v1 refresh message must parse");
let (real, _) = route(vec![(
msg.team_id,
msg.shadow,
msg.emitted_at,
msg.source,
0,
)]);
assert_eq!(real[&7].reported_source(), Source::Refresh);
}
#[test]
fn coalesce_groups_offsets_per_team() {
let got = fold_edits(vec![(1, ts(50), 10), (2, ts(60), 20), (1, ts(40), 11)]);
assert_eq!(got.len(), 2);
assert_eq!(got[&1].oldest_emitted_at, ts(40));
assert_eq!(got[&1].offsets, vec![10, 11]);
assert_eq!(got[&2].oldest_emitted_at, ts(60));
assert_eq!(got[&2].offsets, vec![20]);
}
#[test]
fn retry_backoff_follows_documented_schedule() {
// 200ms · 4^(n-1): the first two retries are the 200ms / 800ms the module
// doc and PR describe.
assert_eq!(retry_backoff(1).as_millis(), 200);
assert_eq!(retry_backoff(2).as_millis(), 800);
assert_eq!(retry_backoff(3).as_millis(), 3200);
}
#[test]
fn retry_backoff_saturates_instead_of_overflowing() {
// A large `build_max_attempts` must not panic on the exponent — the
// multiplier saturates to u64::MAX, capping the sleep rather than wrapping.
assert_eq!(retry_backoff(u32::MAX).as_millis(), u64::MAX as u128);
}
#[test]
fn truncate_for_header_passes_short_errors_through() {
assert_eq!(truncate_for_header("boom"), "boom");
}
#[test]
fn build_failure_attributes_persist_errors_to_their_tier() {
use common_hypercache::HyperCacheError;
use common_redis::CustomRedisError;
use common_s3::S3Error;
// The category is the triage signal the DLQ exists for, so each HyperCache
// tier must map to its own label; unexpected variants degrade to "other"
// rather than being mislabelled.
let cases = [
(HyperCacheError::Redis(CustomRedisError::Timeout), "redis"),
(
HyperCacheError::S3(S3Error::OperationFailed("boom".into())),
"s3",
),
(
HyperCacheError::Json(serde_json::from_str::<i32>("x").unwrap_err()),
"serialize",
),
(HyperCacheError::CacheMiss, "other"),
];
for (err, expected) in cases {
assert_eq!(
BuildFailure::from_persist(err).category.as_label(),
expected
);
}
}
#[test]
fn build_failure_attributes_build_step_to_database() {
assert_eq!(
BuildFailure::database("pg unreachable").category.as_label(),
"database"
);
}
#[test]
fn build_failure_attributes_live_read_errors_to_their_tier() {
use common_hypercache::HyperCacheError;
use common_redis::CustomRedisError;
// Shadow live-read categories: parse failures mean the cached bytes
// don't fit the typed model (persistent, worth triaging apart), while
// Redis/timeout errors are the transport tier.
let cases = [
(
HyperCacheError::Json(serde_json::from_str::<i32>("x").unwrap_err()),
"cache_parse",
),
(HyperCacheError::Pickle("bad pickle".into()), "cache_parse"),
(HyperCacheError::Redis(CustomRedisError::Timeout), "redis"),
(HyperCacheError::Timeout("redis timeout".into()), "redis"),
];
for (err, expected) in cases {
assert_eq!(
BuildFailure::from_live_read(err).category.as_label(),
expected
);
}
}
/// Build a real `ShadowObservation` through the public diff + tracker API:
/// one observation is a first sighting (suppressed), two consecutive ones
/// confirm. The tracker's pending state lives in Redis, so the second
/// observation has to read what the first wrote. `MockRedisClient` records
/// writes but serves no reads, so the recorded write is replayed into the
/// client the second observation reads from.
///
/// `next_build` in `flags::cache_shadow`'s tests does the same replay for the
/// tracker's own tests. A change to what the mock records has to reach both.
async fn shadow_observation(
confirmed: bool,
) -> feature_flags::flags::cache_shadow::ShadowObservation {
use common_redis::{MockRedisClient, MockRedisValue};
use feature_flags::flags::cache_shadow::{
diff_live_entry, MismatchTracker, ShadowLiveEntry, MIN_CONFIRM_INTERVAL,
};
use feature_flags::flags::flag_models::{
EvaluationMetadata, FeatureFlag, HypercacheFlagsWrapper,
};
use std::sync::Arc;
use std::time::{Duration as StdDuration, SystemTime};
let flag = |has_experiment: bool| -> FeatureFlag {
serde_json::from_value(serde_json::json!({
"id": 1,
"team_id": 1,
"key": "flag-1",
"filters": {"groups": []},
"active": true,
"deleted": false,
"has_experiment": has_experiment,
}))
.expect("flag json must parse")
};
let built = HypercacheFlagsWrapper {
flags: vec![flag(false)],
evaluation_metadata: EvaluationMetadata::default(),
cohorts: Some(Vec::new()),
};
let live = ShadowLiveEntry {
flags: vec![flag(true)],
evaluation_metadata: Some(EvaluationMetadata::default()),
cohorts: Some(Vec::new()),
};
let ttl = StdDuration::from_secs(3600);
let first_at = SystemTime::UNIX_EPOCH + StdDuration::from_secs(1_000);
let redis = MockRedisClient::new();
let first = MismatchTracker::new(Arc::new(redis.clone()), ttl)
.observe(1, diff_live_entry(&built, &live), first_at)
.await;
if !confirmed {
return first;
}
let mut next = MockRedisClient::new();
for call in redis.get_calls() {
if let ("setex", MockRedisValue::StringWithTTL(value, _)) =
(call.op.as_str(), call.value)
{
next.get_ret(&call.key, Ok(value));
}
}
// The second observation has to sit at least MIN_CONFIRM_INTERVAL past the
// first, or the confirmation window is still closed and nothing confirms.
MismatchTracker::new(Arc::new(next), ttl)
.observe(
1,
diff_live_entry(&built, &live),
first_at + MIN_CONFIRM_INTERVAL,
)
.await
}
#[tokio::test]
async fn shadow_outcome_maps_to_exactly_the_documented_labels() {
// The outcome label is the shadow window's primary telemetry; a wrong
// label here misreads the ramp with no other test catching it.
let cases = [
(ShadowOutcome::Match, ShadowOutcomeLabel::Match),
(
ShadowOutcome::LiveEntryMissing,
ShadowOutcomeLabel::LiveEntryMissing,
),
(
ShadowOutcome::Failed(BuildFailure::database("pg unreachable")),
ShadowOutcomeLabel::Error,
),
(
ShadowOutcome::Mismatch(shadow_observation(false).await),
ShadowOutcomeLabel::MismatchSuppressed,
),
(
ShadowOutcome::Mismatch(shadow_observation(true).await),
ShadowOutcomeLabel::MismatchConfirmed,
),
];
for (outcome, expected) in cases {
assert_eq!(outcome.label(), expected);
}
}
#[test]
fn truncate_for_header_stays_within_cap_on_char_boundary() {
// Multibyte chars (3 bytes each) so the cap lands mid-character: the
// truncation must back up to a char boundary and stay valid UTF-8.
let long = "✓".repeat(DLQ_ERROR_HEADER_MAX); // 3 · 1024 bytes, well over the cap
let got = truncate_for_header(&long);
assert!(got.len() <= DLQ_ERROR_HEADER_MAX, "exceeded byte cap");
assert!(got.ends_with('…'), "missing truncation marker");
// `String` is UTF-8 by construction; reaching here without a slice panic is
// the real assertion.
}
/// Every series `precreate_counters` must register, as
/// `name{label=value,...}` with the labels sorted. The names and label
/// values are written out here rather than read back off the enums, so
/// adding a variant fails this test: a new label value is a new series on a
/// metric that dashboards and alerts group by, and it should not appear
/// unnoticed.
const EXPECTED_PRECREATED_SERIES: &[&str] = &[
"flags_cache_builder_build_retries_total{}",
"flags_cache_builder_builds_total{reason=cache_parse,result=failure,source=edit}",
"flags_cache_builder_builds_total{reason=cache_parse,result=failure,source=refresh}",
"flags_cache_builder_builds_total{reason=database,result=failure,source=edit}",
"flags_cache_builder_builds_total{reason=database,result=failure,source=refresh}",
"flags_cache_builder_builds_total{reason=other,result=failure,source=edit}",
"flags_cache_builder_builds_total{reason=other,result=failure,source=refresh}",
"flags_cache_builder_builds_total{reason=redis,result=failure,source=edit}",
"flags_cache_builder_builds_total{reason=redis,result=failure,source=refresh}",
"flags_cache_builder_builds_total{reason=s3,result=failure,source=edit}",
"flags_cache_builder_builds_total{reason=s3,result=failure,source=refresh}",
"flags_cache_builder_builds_total{reason=serialize,result=failure,source=edit}",
"flags_cache_builder_builds_total{reason=serialize,result=failure,source=refresh}",
"flags_cache_builder_builds_total{result=success,source=edit}",
"flags_cache_builder_builds_total{result=success,source=refresh}",
"flags_cache_builder_dlq_produced_total{result=failure}",
"flags_cache_builder_dlq_produced_total{result=success}",
"flags_cache_builder_kafka_recv_errors_total{}",
"flags_cache_builder_messages_received_total{}",
"flags_cache_builder_parse_errors_total{}",
"flags_cache_shadow_build_failures_total{category=cache_parse}",
"flags_cache_shadow_build_failures_total{category=database}",
"flags_cache_shadow_build_failures_total{category=other}",
"flags_cache_shadow_build_failures_total{category=redis}",
"flags_cache_shadow_build_failures_total{category=s3}",
"flags_cache_shadow_build_failures_total{category=serialize}",
"flags_cache_shadow_builds_total{outcome=error}",
"flags_cache_shadow_builds_total{outcome=live_entry_missing}",
"flags_cache_shadow_builds_total{outcome=match}",
"flags_cache_shadow_builds_total{outcome=mismatch_confirmed}",
"flags_cache_shadow_builds_total{outcome=mismatch_suppressed}",
"flags_cache_shadow_mismatch_first_sight_total{issue_type=cohort_field_mismatch}",
"flags_cache_shadow_mismatch_first_sight_total{issue_type=cohort_missing_in_cache}",
"flags_cache_shadow_mismatch_first_sight_total{issue_type=cohort_stale_in_cache}",
"flags_cache_shadow_mismatch_first_sight_total{issue_type=evaluation_metadata_mismatch}",
"flags_cache_shadow_mismatch_first_sight_total{issue_type=field_mismatch}",
"flags_cache_shadow_mismatch_first_sight_total{issue_type=missing_evaluation_metadata}",
"flags_cache_shadow_mismatch_first_sight_total{issue_type=missing_in_cache}",
"flags_cache_shadow_mismatch_first_sight_total{issue_type=stale_in_cache}",
"flags_cache_shadow_mismatch_total{issue_type=cohort_field_mismatch}",
"flags_cache_shadow_mismatch_total{issue_type=cohort_missing_in_cache}",
"flags_cache_shadow_mismatch_total{issue_type=cohort_stale_in_cache}",
"flags_cache_shadow_mismatch_total{issue_type=evaluation_metadata_mismatch}",
"flags_cache_shadow_mismatch_total{issue_type=field_mismatch}",
"flags_cache_shadow_mismatch_total{issue_type=missing_evaluation_metadata}",
"flags_cache_shadow_mismatch_total{issue_type=missing_in_cache}",
"flags_cache_shadow_mismatch_total{issue_type=stale_in_cache}",
"flags_cache_shadow_tracker_store_errors_total{op=clear}",
"flags_cache_shadow_tracker_store_errors_total{op=read}",
"flags_cache_shadow_tracker_store_errors_total{op=write}",
];
#[test]
fn precreate_counters_registers_every_series_at_zero() {
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use std::collections::BTreeSet;
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::with_local_recorder(&recorder, precreate_counters);
let got: Vec<(String, DebugValue)> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.map(|(ckey, _, _, value)| {
let key = ckey.key();
let mut labels: Vec<String> = key
.labels()
.map(|label| format!("{}={}", label.key(), label.value()))
.collect();
labels.sort();
(format!("{}{{{}}}", key.name(), labels.join(",")), value)
})
.collect();
// Compared as the two differences rather than as two full lists, so a
// failure names the series instead of printing the whole inventory.
let expected: BTreeSet<&str> = EXPECTED_PRECREATED_SERIES.iter().copied().collect();
let found: BTreeSet<&str> = got.iter().map(|(name, _)| name.as_str()).collect();
let missing: Vec<&str> = expected.difference(&found).copied().collect();
let unexpected: Vec<&str> = found.difference(&expected).copied().collect();
assert!(
missing.is_empty() && unexpected.is_empty(),
"not pre-created: {missing:#?}\npre-created but not expected: {unexpected:#?}"
);
// Registering a series must not report activity on it. An increment of
// anything but zero here would make every dashboard and alert read one
// event per pod from boot.
for (name, value) in &got {
assert_eq!(
*value,
DebugValue::Counter(0),
"{name} must register at zero"
);
}
}
#[test]
fn keeps_highest_offset_per_partition() {
// Interleaved, out-of-order input across two partitions.
let got = reduce(vec![(0, 5), (1, 2), (0, 3), (1, 10), (0, 8)]);
assert_eq!(got, vec![(0, 8), (1, 10)]);
}
#[test]
fn ascending_input_keeps_last() {
let got = reduce(vec![(0, 1), (0, 2), (0, 3)]);
assert_eq!(got, vec![(0, 3)]);
}
#[test]
fn descending_input_keeps_first() {
// The max must win regardless of arrival order — a later lower offset
// must not regress the partition's checkpoint.
let got = reduce(vec![(0, 3), (0, 2), (0, 1)]);
assert_eq!(got, vec![(0, 3)]);
}
#[test]
fn empty_input_yields_empty() {
assert!(reduce(vec![]).is_empty());
}
#[test]
fn single_offset_per_partition_passes_through() {
let got = reduce(vec![(0, 7), (1, 4), (2, 9)]);
assert_eq!(got, vec![(0, 7), (1, 4), (2, 9)]);
}
}