Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

68.7 KB
Raw Blame
use crate::api::errors::FlagError;
use crate::api::request_outcome::OutcomeMarks;
use crate::config::PersonsDbDeadlineBound;
use crate::flags::flag_matching::EvaluationType;
use crate::handler::phases::PhaseDurations;
use crate::metrics::consts::{
    FLAG_BODY_READ_TIME_MS, FLAG_CONCURRENCY_LIMIT_WAIT_TIME_MS, FLAG_DB_OPERATIONS_PER_REQUEST,
    FLAG_PHASE_DURATION_MS, FLAG_PRE_HANDLER_TIME_MS, FLAG_QUEUE_TIME_MS,
};
use crate::team::request_blocklist::BlockReason;
use crate::utils::bot_detection::{BotCategory, BotSource};
use std::cell::RefCell;
use std::future::Future;
use std::time::Instant;
use uuid::Uuid;

// Task-local storage for the canonical log line.
// This allows any code in the async task to access and modify the log
// without explicit parameter passing.
tokio::task_local! {
    static CANONICAL_LOG: RefCell<FlagsCanonicalLogLine>;
}

// Thread-local fallback for rayon worker threads.
// Rayon threads don't have access to tokio task-locals, so we use a
// std::thread_local to capture evaluation counters during parallel flag
// evaluation. The counters are then merged back into the request's
// canonical log on the tokio side after rayon returns.
std::thread_local! {
    static RAYON_CANONICAL_LOG: RefCell<Option<FlagsCanonicalLogLine>> = const { RefCell::new(None) };
}

/// Safely modify the canonical log if one exists in scope.
/// No-ops silently if called outside a canonical log scope.
///
/// Tries the tokio task-local first (normal async request path), then
/// falls back to the std::thread_local (rayon parallel-eval path).
///
/// # Example
/// ```ignore
/// with_canonical_log(|log| log.team_id = Some(123));
/// with_canonical_log(|log| log.eval.property_cache_hits += 1);
/// ```
pub fn with_canonical_log(f: impl FnOnce(&mut FlagsCanonicalLogLine)) {
    // Probe whether the tokio task-local is accessible without consuming `f`.
    // On rayon threads there is no tokio task context, so this returns Err.
    if CANONICAL_LOG.try_with(|_| ()).is_ok() {
        CANONICAL_LOG.with(|log| f(&mut log.borrow_mut()));
    } else {
        RAYON_CANONICAL_LOG.with(|cell| {
            if let Some(ref mut log) = *cell.borrow_mut() {
                f(log);
            }
        });
    }
}

#[must_use = "guard cleans up the thread-local on drop; dropping immediately is a no-op"]
pub(crate) struct RayonCanonicalLogGuard;

impl Drop for RayonCanonicalLogGuard {
    fn drop(&mut self) {
        drop(take_rayon_canonical_log());
    }
}

/// Install a fresh canonical log on the current thread for rayon evaluation.
/// Returns an RAII guard that clears the thread-local on drop, ensuring
/// cleanup even if the evaluation panics. Call [`take_rayon_canonical_log`]
/// before the guard is dropped to retrieve the accumulated counters.
pub(crate) fn install_rayon_canonical_log() -> RayonCanonicalLogGuard {
    RAYON_CANONICAL_LOG.with(|cell| {
        *cell.borrow_mut() = Some(FlagsCanonicalLogLine::default());
    });

    RayonCanonicalLogGuard
}

/// Take the accumulated canonical log from the current rayon thread.
/// Returns `None` if no log was installed.
pub(crate) fn take_rayon_canonical_log() -> Option<FlagsCanonicalLogLine> {
    RAYON_CANONICAL_LOG.with(|cell| cell.borrow_mut().take())
}

/// Run an async block with a canonical log in scope.
/// Returns both the result and the final log state.
///
/// The log is automatically available to all code within the async block
/// via `with_canonical_log()`.
///
/// # Example
/// ```ignore
/// let log = FlagsCanonicalLogLine::new(request_id, ip);
/// let (result, log) = run_with_canonical_log(log, async {
///     // Code here can use with_canonical_log() to modify the log
///     process_request().await
/// }).await;
/// log.emit();
/// ```
pub async fn run_with_canonical_log<F, T>(
    log: FlagsCanonicalLogLine,
    f: F,
) -> (T, FlagsCanonicalLogLine)
where
    F: Future<Output = T>,
{
    CANONICAL_LOG
        .scope(RefCell::new(log), async {
            let result = f.await;
            let log = CANONICAL_LOG.with(|l| l.take());
            (result, log)
        })
        .await
}

/// Truncate a string to a maximum number of characters (not bytes).
/// Handles multibyte UTF-8 characters correctly.
pub(crate) fn truncate_chars(s: &str, max_chars: usize) -> &str {
    match s.char_indices().nth(max_chars) {
        Some((byte_idx, _)) => &s[..byte_idx],
        None => s,
    }
}

/// Per-flag evaluation counters accumulated during `evaluate_single_flag` and its callees.
///
/// These counters are the fields written via `with_canonical_log` on rayon worker threads.
/// Extracting them into a sub-struct ensures that `merge` covers every field: the
/// exhaustive destructuring pattern in `merge` causes a compile error when a new field
/// is added without a corresponding merge rule.
#[derive(Debug, Clone, Default)]
pub struct EvalCounters {
    /// Number of flags that used device_id for bucketing (instead of distinct_id).
    pub flags_device_id_bucketing: usize,
    /// Number of cohort property filters evaluated across all flags.
    pub cohorts_evaluated: usize,
    /// Number of property lookups served from the evaluation state cache.
    pub property_cache_hits: usize,
    /// Number of property lookups that missed the evaluation state cache.
    pub property_cache_misses: usize,
    /// True if person properties were not found in evaluation state cache.
    pub person_properties_not_cached: bool,
    /// True if group properties were not found in evaluation state cache.
    pub group_properties_not_cached: bool,
}

impl EvalCounters {
    /// Merge another set of counters into this one.
    ///
    /// Numeric counters are summed; boolean flags are OR'd.
    ///
    /// The exhaustive destructuring pattern ensures that adding a new field to
    /// `EvalCounters` without handling it here is a compile error.
    pub fn merge(&mut self, other: &EvalCounters) {
        let EvalCounters {
            flags_device_id_bucketing,
            cohorts_evaluated,
            property_cache_hits,
            property_cache_misses,
            person_properties_not_cached,
            group_properties_not_cached,
        } = other;

        self.flags_device_id_bucketing += flags_device_id_bucketing;
        self.cohorts_evaluated += cohorts_evaluated;
        self.property_cache_hits += property_cache_hits;
        self.property_cache_misses += property_cache_misses;
        self.person_properties_not_cached |= person_properties_not_cached;
        self.group_properties_not_cached |= group_properties_not_cached;
    }
}

/// Accumulates data throughout a /flags request lifecycle for canonical logging.
///
/// A canonical log line is a single comprehensive log entry emitted at request
/// completion containing all key telemetry. This enables simple ClickHouse queries
/// for debugging without needing to join multiple log entries.
///
/// Access and modify via `with_canonical_log()` from anywhere in the async task.
#[derive(Debug, Clone)]
pub struct FlagsCanonicalLogLine {
    // Request identification
    pub request_id: Uuid,
    pub ip: String,
    pub start_time: Instant,

    // Request metadata (useful for SDK debugging)
    pub user_agent: Option<String>,
    /// Host of the `Origin` header, or of `Referer` without one. Shows support which site
    /// calls `/flags` with a team's token. Log only: it is unbounded, so never a metric label.
    pub origin_host: Option<String>,
    pub lib: Option<String>,
    pub lib_version: Option<String>,
    pub api_version: Option<String>,
    /// The response shape served, such as `FlagsV2` or `FlagsV4`.
    pub response_format: Option<&'static str>,

    // Populated during authentication
    pub team_id: Option<i32>,
    pub distinct_id: Option<String>,
    pub device_id: Option<String>,
    /// The anonymous distinct ID sent with the request for experience continuity.
    pub anon_distinct_id: Option<String>,

    // Populated during flag evaluation
    pub flags_evaluated: usize,
    pub flags_experience_continuity: usize,
    pub flags_disabled: bool,
    pub quota_limited: bool,
    /// Set when the request supplied a `$geoip_*` value disagreeing with the MaxMind lookup.
    /// Attributable counterpart to `flags_geoip_properties_differ_from_lookup_total`, which has
    /// no labels.
    pub geoip_properties_differ_from_lookup: bool,
    /// Flag keys that were overridden with custom definitions (for testing/historical evaluation)
    pub flags_overridden: Option<Vec<String>>,
    /// Source of the flags data: "Redis", "S3", or "Fallback" (PostgreSQL).
    pub flags_cache_source: Option<&'static str>,

    /// Per-flag evaluation counters accumulated on rayon threads.
    /// See [`EvalCounters::merge`] for the merge strategy.
    pub eval: EvalCounters,

    // Deep evaluation metrics (populated via task_local from flag_matching.rs)
    /// Total number of database property fetch operations (aggregate counter).
    pub db_property_fetches: usize,
    /// Number of person property queries made to the database.
    pub person_queries: usize,
    /// Number of group property queries made to the database.
    pub group_queries: usize,
    /// Number of static cohort membership queries made to the database.
    pub static_cohort_queries: usize,
    /// Number of realtime cohort membership queries made to the behavioral cohorts database.
    pub realtime_cohort_queries: usize,
    /// Count of realtime cohort IDs encountered during evaluation.
    /// Incremented even for anonymous users (no person UUID), where no DB query is made.
    /// Use `realtime_cohort_queries` to distinguish how many queries actually reached the DB.
    pub realtime_cohorts_evaluated: usize,
    /// Time spent on person property queries in milliseconds.
    pub person_query_time_ms: u64,
    /// Time spent on group property queries in milliseconds.
    pub group_query_time_ms: u64,
    /// Time spent on static cohort membership queries in milliseconds.
    pub cohort_query_time_ms: u64,
    /// Time spent on realtime cohort membership queries in milliseconds.
    pub realtime_cohort_query_time_ms: u64,
    /// Number of flags whose evaluation returned an error. Not in `EvalCounters` because it is
    /// incremented in `process_flag_result`, which runs on the tokio task after rayon returns.
    pub flags_errored: usize,
    /// Number of errors encountered during dependency graph construction.
    /// These errors (like missing dependencies or cycles) set errors_while_computing_flags=true
    /// in the response but don't increment flags_errored.
    pub dependency_graph_errors: usize,
    /// Status of hash key override lookup for experience continuity.
    /// Values:
    /// - None: no flags require experience continuity
    /// - "skipped": optimization applied (100% rollout, no variants needing lookup)
    /// - "error": query failed
    /// - "empty": query succeeded, no overrides found
    /// - "found": query succeeded, overrides returned
    pub hash_key_override_status: Option<&'static str>,
    /// The first persons DB call that the persons DB deadline stopped, such as
    /// "fetch_properties". The flags that needed persons data returned an error.
    pub persons_db_deadline_exceeded: Option<&'static str>,
    /// Which limit set the persons DB deadline. With `RequestTimeout`, a
    /// `persons_db_deadline_exceeded` on the same line can occur on a healthy persons DB.
    pub persons_db_deadline_bound: Option<PersonsDbDeadlineBound>,
    /// Shared with the request outcome middleware. Mark it from the tokio task. A rayon thread's
    /// canonical log gets its own `OutcomeMarks`, and `merge_rayon_delta` does not copy it back,
    /// so a mark made there never reaches the middleware.
    pub outcome_marks: OutcomeMarks,

    /// Which evaluation strategy was used for this request.
    /// Set to `Parallel` if any dependency level triggered parallel evaluation.
    /// `None` if flags were not evaluated (disabled, quota limited, empty).
    pub evaluation_type: Option<EvaluationType>,

    // Rate limiting
    pub rate_limited: bool,
    /// True when a rate limit warn threshold was exceeded but the request was still allowed.
    pub rate_limit_warned: bool,

    // Transit time
    /// Proxy-to-app queue time in milliseconds (server_now - X-Request-Start).
    /// None when the header is missing or the computed delta is negative.
    pub queue_time_ms: Option<i64>,

    /// Synchronous pre-handler work in milliseconds: covers UA parse, IP
    /// rate-limit, token extract, and token rate-limit. Subdivides
    /// `queue_time_ms` so we can attribute spikes to the right tier. None
    /// when the request was rejected before reaching the pre-handler stamp.
    pub pre_handler_duration_ms: Option<u64>,

    /// Time spent waiting for a permit on the tower
    /// `ConcurrencyLimitLayer` (Phase F instrumentation). None until that
    /// phase ships; while None, `emit_timing_metrics` skips emission.
    pub concurrency_limit_wait_ms: Option<u64>,

    /// Wall-clock duration of inbound request body buffering, in
    /// milliseconds with sub-ms precision. Captured by the
    /// `record_body_read` middleware shim, populated on the request
    /// extensions, and seeded onto the canonical log at handler entry.
    /// `None` only when the shim is absent on the route that reached the
    /// handler (the production wiring installs it on `/flags|/decide`,
    /// so this is the rollout-safety / test fallback path); the shim
    /// itself records a duration for every method, including empty
    /// bodies. `f64` (not `u64`) because `BODY_READ_BUCKETS_MS` has
    /// sub-ms boundaries — integer-ms truncation would hide warm
    /// in-memory POSTs in the bottom bucket.
    pub body_read_ms: Option<f64>,

    /// Per-phase wall-clock breakdown of `process_request_inner`.
    /// Populated by [`crate::handler::phases::PhaseGuard`] drops; emitted
    /// as a labeled histogram by [`Self::emit_phase_metrics`] once the
    /// `team_id` is resolved.
    pub phases: PhaseDurations,

    // Cache sources (populated during data fetching)
    /// Where team metadata was fetched from: "redis", "s3", "fallback", or None if not fetched
    pub team_cache_source: Option<&'static str>,

    // Outcome (populated at response time)
    pub http_status: u16,
    /// Error code from FlagError::error_code(). Uses &'static str to avoid allocation.
    pub error_code: Option<&'static str>,

    /// True when the request was short-circuited by the bot filter and
    /// skipped token rate-limit, auth, billing, and evaluation.
    pub is_bot: bool,
    /// Matched bot category, set iff `is_bot` is true. The enum bounds
    /// the label set; [`Self::emit`] stringifies at log time.
    pub bot_category: Option<BotCategory>,
    /// Which signal fired (UA or IP), set iff `is_bot` is true.
    pub bot_source: Option<BotSource>,

    /// Set when the team's request blocklist refused the request before billing and
    /// evaluation.
    pub request_blocked_by: Option<BlockReason>,
}

impl Default for FlagsCanonicalLogLine {
    fn default() -> Self {
        Self {
            request_id: Uuid::nil(),
            ip: String::new(),
            start_time: Instant::now(),
            user_agent: None,
            origin_host: None,
            lib: None,
            lib_version: None,
            api_version: None,
            response_format: None,
            team_id: None,
            distinct_id: None,
            device_id: None,
            anon_distinct_id: None,
            flags_evaluated: 0,
            flags_experience_continuity: 0,
            flags_disabled: false,
            quota_limited: false,
            geoip_properties_differ_from_lookup: false,
            flags_overridden: None,
            flags_cache_source: None,
            eval: EvalCounters::default(),
            db_property_fetches: 0,
            person_queries: 0,
            group_queries: 0,
            static_cohort_queries: 0,
            realtime_cohort_queries: 0,
            realtime_cohorts_evaluated: 0,
            person_query_time_ms: 0,
            group_query_time_ms: 0,
            cohort_query_time_ms: 0,
            realtime_cohort_query_time_ms: 0,
            flags_errored: 0,
            dependency_graph_errors: 0,
            hash_key_override_status: None,
            persons_db_deadline_exceeded: None,
            persons_db_deadline_bound: None,
            outcome_marks: OutcomeMarks::default(),
            evaluation_type: None,
            rate_limited: false,
            rate_limit_warned: false,
            queue_time_ms: None,
            pre_handler_duration_ms: None,
            concurrency_limit_wait_ms: None,
            body_read_ms: None,
            phases: PhaseDurations::default(),
            team_cache_source: None,
            http_status: 200,
            error_code: None,
            is_bot: false,
            bot_category: None,
            bot_source: None,
            request_blocked_by: None,
        }
    }
}

impl FlagsCanonicalLogLine {
    pub fn new(request_id: Uuid, ip: String) -> Self {
        Self {
            request_id,
            ip,
            ..Default::default()
        }
    }

    /// Emit the canonical log line. Call once at request completion.
    pub fn emit(&self) {
        let duration_ms = self.start_time.elapsed().as_millis() as u64;

        // Truncate user_agent to prevent log bloat from very long headers (some bots send KB+).
        // Note: distinct_id is already truncated at request parsing time (see MAX_DISTINCT_ID_LEN).
        let user_agent = self.user_agent.as_deref().map(|ua| truncate_chars(ua, 512));
        let origin_host = self
            .origin_host
            .as_deref()
            .map(|host| truncate_chars(host, 512));

        tracing::info!(
            request_id = %self.request_id,
            team_id = self.team_id,
            distinct_id = self.distinct_id.as_deref(),
            device_id = self.device_id.as_deref(),
            anon_distinct_id = self.anon_distinct_id.as_deref(),
            ip = %self.ip,
            user_agent = user_agent,
            origin_host = origin_host,
            lib = self.lib.as_deref(),
            lib_version = self.lib_version.as_deref(),
            api_version = self.api_version.as_deref(),
            response_format = self.response_format,
            duration_ms = duration_ms,
            http_status = self.http_status,
            flags_evaluated = self.flags_evaluated,
            flags_experience_continuity = self.flags_experience_continuity,
            flags_device_id_bucketing = self.eval.flags_device_id_bucketing,
            flags_disabled = self.flags_disabled,
            quota_limited = self.quota_limited,
            geoip_properties_differ_from_lookup = self.geoip_properties_differ_from_lookup,
            flags_overridden = ?self.flags_overridden,
            flags_cache_source = self.flags_cache_source,
            db_property_fetches = self.db_property_fetches,
            person_queries = self.person_queries,
            group_queries = self.group_queries,
            static_cohort_queries = self.static_cohort_queries,
            realtime_cohort_queries = self.realtime_cohort_queries,
            realtime_cohorts_evaluated = self.realtime_cohorts_evaluated,
            person_query_time_ms = self.person_query_time_ms,
            group_query_time_ms = self.group_query_time_ms,
            cohort_query_time_ms = self.cohort_query_time_ms,
            realtime_cohort_query_time_ms = self.realtime_cohort_query_time_ms,
            property_cache_hits = self.eval.property_cache_hits,
            property_cache_misses = self.eval.property_cache_misses,
            person_properties_not_cached = self.eval.person_properties_not_cached,
            group_properties_not_cached = self.eval.group_properties_not_cached,
            cohorts_evaluated = self.eval.cohorts_evaluated,
            flags_errored = self.flags_errored,
            dependency_graph_errors = self.dependency_graph_errors,
            hash_key_override_status = self.hash_key_override_status,
            persons_db_deadline_exceeded = self.persons_db_deadline_exceeded,
            persons_db_deadline_bound = self.persons_db_deadline_bound.map(|b| b.as_str()),
            evaluation_type = self.evaluation_type.map(|t| t.as_str()),
            rate_limited = self.rate_limited,
            rate_limit_warned = self.rate_limit_warned,
            queue_time_ms = self.queue_time_ms,
            pre_handler_duration_ms = self.pre_handler_duration_ms,
            concurrency_limit_wait_ms = self.concurrency_limit_wait_ms,
            body_read_ms = self.body_read_ms,
            team_cache_source = self.team_cache_source,
            error_code = self.error_code,
            is_bot = self.is_bot,
            bot_category = self.bot_category.map(|c| c.as_str()),
            bot_source = self.bot_source.map(|s| s.as_str()),
            request_blocked_by = self.request_blocked_by.map(|r| r.as_str()),
            "canonical_log_line"
        );
    }

    /// Emit DB operations metrics for observability.
    /// This emits a histogram for each operation type with the count of operations.
    /// Labels: team_id, operation_type
    pub fn emit_db_operations_metrics(&self) {
        let team_id = self
            .team_id
            .map(|id| id.to_string())
            .unwrap_or_else(|| "unknown".to_string());

        // Emit person query count
        if self.person_queries > 0 {
            common_metrics::histogram(
                FLAG_DB_OPERATIONS_PER_REQUEST,
                &[
                    ("team_id".to_string(), team_id.clone()),
                    ("operation_type".to_string(), "person_query".to_string()),
                ],
                self.person_queries as f64,
            );
        }

        // Emit group query count
        if self.group_queries > 0 {
            common_metrics::histogram(
                FLAG_DB_OPERATIONS_PER_REQUEST,
                &[
                    ("team_id".to_string(), team_id.clone()),
                    ("operation_type".to_string(), "group_query".to_string()),
                ],
                self.group_queries as f64,
            );
        }

        // Emit static cohort query count
        if self.static_cohort_queries > 0 {
            common_metrics::histogram(
                FLAG_DB_OPERATIONS_PER_REQUEST,
                &[
                    ("team_id".to_string(), team_id.clone()),
                    ("operation_type".to_string(), "cohort_query".to_string()),
                ],
                self.static_cohort_queries as f64,
            );
        }

        // Emit realtime cohort query count
        if self.realtime_cohort_queries > 0 {
            common_metrics::histogram(
                FLAG_DB_OPERATIONS_PER_REQUEST,
                &[
                    ("team_id".to_string(), team_id.clone()),
                    (
                        "operation_type".to_string(),
                        "realtime_cohort_query".to_string(),
                    ),
                ],
                self.realtime_cohort_queries as f64,
            );
        }
    }

    /// Emit queue/pre-handler/concurrency-wait histograms with `team_id` labels.
    ///
    /// Owns the `flags_queue_time_ms` emission (replaces a previously-unlabeled
    /// emission in `endpoint::flags`). Sibling metrics share the same label
    /// shape so a dashboard can subtract them safely:
    ///
    ///   queue_time = pre_handler + concurrency_wait + (envoy + axum + non-tower)
    ///
    /// `concurrency_limit_wait_ms` intentionally omits `team_id` — permit-wait
    /// is a property of pod-level load, not of any one team, and labeling
    /// would mislead readers of the dashboard.
    ///
    /// Call once per request, after `team_id` has been resolved (i.e. after
    /// `process_request` returns) and before `emit()`.
    pub fn emit_timing_metrics(&self) {
        let team_id_label = self
            .team_id
            .map(|id| id.to_string())
            .unwrap_or_else(|| "unknown".to_string());
        let team_id_labels = [("team_id".to_string(), team_id_label)];

        if let Some(delta) = self.queue_time_ms {
            // Filter out negative deltas defensively. The endpoint already
            // filters them, but the field is i64 to preserve the original
            // calculation; negative values would break the histogram.
            if delta >= 0 {
                common_metrics::histogram(FLAG_QUEUE_TIME_MS, &team_id_labels, delta as f64);
            }
        }

        if let Some(duration) = self.pre_handler_duration_ms {
            common_metrics::histogram(FLAG_PRE_HANDLER_TIME_MS, &team_id_labels, duration as f64);
        }

        if let Some(wait) = self.concurrency_limit_wait_ms {
            common_metrics::histogram(FLAG_CONCURRENCY_LIMIT_WAIT_TIME_MS, &[], wait as f64);
        }

        if let Some(body_read) = self.body_read_ms {
            // No `team_id` label — body buffering happens before
            // authentication, mirroring `concurrency_limit_wait_ms`.
            common_metrics::histogram(FLAG_BODY_READ_TIME_MS, &[], body_read);
        }
    }

    /// Emit `flags_phase_duration_ms` histograms for every phase that ran.
    ///
    /// Each entry is labeled `phase=<name>` and `team_id=<id>` (filtered
    /// through the existing team allowlist; non-allowlisted teams collapse
    /// to `unknown`). Phases that never ran on this request — e.g.
    /// `record_billing` for a non-billable request, `cookieless` for a
    /// quota-limited request — are silently skipped.
    ///
    /// Call once per request, after `team_id` has been resolved (i.e.
    /// after `process_request` returns). Idempotent only in the trivial
    /// sense: calling twice would double-count, so callers must not.
    pub fn emit_phase_metrics(&self) {
        let team_id_label_value = self
            .team_id
            .map(|id| id.to_string())
            .unwrap_or_else(|| "unknown".to_string());

        for (phase, elapsed) in self.phases.iter() {
            // The phase label varies per iteration; team_id is constant
            // for the request. We have to go through `common_metrics::
            // histogram` (rather than `metrics::histogram!` directly) to
            // pick up the team-id allowlist filter — without it cardinality
            // is `phase_count × every_team`, not `phase_count ×
            // allowlist_size`. Allocations are bounded: ~four short
            // `String`s per phase, ~28 per request total.
            let labels = [
                ("phase".to_string(), phase.name().to_string()),
                ("team_id".to_string(), team_id_label_value.clone()),
            ];
            common_metrics::histogram(
                FLAG_PHASE_DURATION_MS,
                &labels,
                elapsed.as_secs_f64() * 1000.0,
            );
        }
    }

    /// Merge evaluation counters accumulated on a rayon thread back into this log.
    ///
    /// Delegates to [`EvalCounters::merge`], which uses an exhaustive destructuring
    /// pattern so that adding a new counter field without a merge rule is a compile error.
    ///
    /// Only merges fields that are written during per-flag evaluation inside
    /// `evaluate_single_flag` and its callees. Request-level metadata (request_id,
    /// team_id, ip, etc.) lives outside `EvalCounters` and is intentionally not merged.
    pub fn merge_rayon_delta(&mut self, delta: &FlagsCanonicalLogLine) {
        self.eval.merge(&delta.eval);
    }

    /// Populate error fields from a FlagError without emitting.
    pub fn set_error(&mut self, error: &FlagError) {
        self.http_status = error.status_code();
        self.error_code = Some(error.error_code());
    }

    /// Populate error fields from a FlagError and emit the log line.
    pub fn emit_for_error(&mut self, error: &FlagError) {
        self.set_error(error);
        self.emit();
    }

    /// Emit dependent histograms then the canonical log. For short-circuit
    /// paths that bypass `run_with_canonical_log` (IP rate-limit blocked,
    /// bot-Enforced).
    pub fn emit_short_circuit(&self) {
        self.emit_db_operations_metrics();
        self.emit_timing_metrics();
        self.emit_phase_metrics();
        self.emit();
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::api::errors::{ClientFacingError, FlagError};
    use crate::utils::graph_utils::DependencyType;
    use metrics_util::debugging::{DebugValue, DebuggingRecorder};

    /// Shared metric-snapshot fixture for the `emit_*_metrics` test
    /// modules below. Both modules need the same captured-sample shape
    /// (name + sorted labels + raw value) and the same conversion from
    /// `DebuggingRecorder`'s nested-tuple snapshot.
    struct MetricSample {
        name: String,
        labels: Vec<(String, String)>,
        #[allow(dead_code)]
        value: DebugValue,
    }

    fn snapshot_metrics(recorder: &DebuggingRecorder) -> Vec<MetricSample> {
        recorder
            .snapshotter()
            .snapshot()
            .into_vec()
            .into_iter()
            .map(|(ckey, _, _, value)| {
                let key = ckey.key();
                let mut labels: Vec<(String, String)> = key
                    .labels()
                    .map(|l| (l.key().to_string(), l.value().to_string()))
                    .collect();
                labels.sort();
                MetricSample {
                    name: key.name().to_string(),
                    labels,
                    value,
                }
            })
            .collect()
    }

    #[test]
    fn test_new_creates_with_defaults() {
        let request_id = Uuid::new_v4();
        let log = FlagsCanonicalLogLine::new(request_id, "192.168.1.1".to_string());

        assert_eq!(log.request_id, request_id);
        assert_eq!(log.ip, "192.168.1.1");
        assert!(log.user_agent.is_none());
        assert!(log.lib.is_none());
        assert!(log.lib_version.is_none());
        assert!(log.api_version.is_none());
        assert!(log.response_format.is_none());
        assert!(log.team_id.is_none());
        assert!(log.distinct_id.is_none());
        assert!(log.device_id.is_none());
        assert!(log.anon_distinct_id.is_none());
        assert_eq!(log.flags_evaluated, 0);
        assert_eq!(log.flags_experience_continuity, 0);
        assert_eq!(log.eval.flags_device_id_bucketing, 0);
        assert!(!log.flags_disabled);
        assert!(!log.quota_limited);
        assert!(log.flags_cache_source.is_none());
        assert_eq!(log.db_property_fetches, 0);
        assert_eq!(log.person_queries, 0);
        assert_eq!(log.group_queries, 0);
        assert_eq!(log.static_cohort_queries, 0);
        assert_eq!(log.realtime_cohort_queries, 0);
        assert_eq!(log.realtime_cohorts_evaluated, 0);
        assert_eq!(log.realtime_cohort_query_time_ms, 0);
        assert_eq!(log.eval.property_cache_hits, 0);
        assert_eq!(log.eval.property_cache_misses, 0);
        assert!(!log.eval.person_properties_not_cached);
        assert!(!log.eval.group_properties_not_cached);
        assert_eq!(log.eval.cohorts_evaluated, 0);
        assert_eq!(log.flags_errored, 0);
        assert_eq!(log.dependency_graph_errors, 0);
        assert!(log.hash_key_override_status.is_none());
        assert!(!log.rate_limited);
        assert!(!log.rate_limit_warned);
        assert!(log.team_cache_source.is_none());
        assert_eq!(log.http_status, 200);
        assert!(log.error_code.is_none());
        assert!(log.queue_time_ms.is_none());
        assert!(log.pre_handler_duration_ms.is_none());
        assert!(log.concurrency_limit_wait_ms.is_none());
    }

    #[test]
    fn test_emit_does_not_panic() {
        let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
        log.emit();
    }

    #[test]
    fn test_emit_with_all_fields_populated() {
        let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
        log.user_agent = Some("posthog-python/1.0.0".to_string());
        log.origin_host = Some("app.example.com".to_string());
        log.request_blocked_by = Some(BlockReason::Host);
        log.lib = Some("posthog-python".to_string());
        log.lib_version = Some("1.0.0".to_string());
        log.api_version = Some("3".to_string());
        log.response_format = Some("FlagsV4");
        log.team_id = Some(123);
        log.distinct_id = Some("user_abc".to_string());
        log.device_id = Some("device_123".to_string());
        log.flags_evaluated = 10;
        log.flags_experience_continuity = 2;
        log.eval.flags_device_id_bucketing = 3;
        log.flags_disabled = false;
        log.quota_limited = true;
        log.flags_cache_source = Some("redis");
        log.db_property_fetches = 3;
        log.eval.property_cache_hits = 5;
        log.eval.property_cache_misses = 2;
        log.eval.cohorts_evaluated = 4;
        log.flags_errored = 1;
        log.hash_key_override_status = Some("found");
        log.rate_limited = false;
        log.team_cache_source = Some("redis");
        log.http_status = 200;
        log.queue_time_ms = Some(15);
        log.emit();
    }

    #[test]
    fn test_emit_with_error() {
        let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
        log.http_status = 429;
        log.rate_limited = true;
        log.error_code = Some("rate_limited");
        log.emit();
    }

    mod emit_timing_metrics_tests {
        use super::*;
        use rstest::rstest;

        fn set_queue_time(log: &mut FlagsCanonicalLogLine) {
            log.queue_time_ms = Some(150);
        }

        fn set_pre_handler(log: &mut FlagsCanonicalLogLine) {
            log.pre_handler_duration_ms = Some(3);
        }

        #[rstest]
        #[case::queue_time(set_queue_time, "flags_queue_time_ms", 42)]
        #[case::pre_handler(set_pre_handler, "flags_pre_handler_time_ms", 7)]
        fn test_emits_timing_metric_with_team_id_label(
            #[case] set_field: fn(&mut FlagsCanonicalLogLine),
            #[case] metric_name: &str,
            #[case] team_id: i32,
        ) {
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                log.team_id = Some(team_id);
                set_field(&mut log);
                log.emit_timing_metrics();
            });

            let metrics = snapshot_metrics(&recorder);
            let sample = metrics
                .iter()
                .find(|s| s.name == metric_name)
                .unwrap_or_else(|| panic!("{metric_name} not emitted"));
            let expected = team_id.to_string();
            assert!(
                sample
                    .labels
                    .iter()
                    .any(|(k, v)| k == "team_id" && v == &expected),
                "expected team_id={expected} label on {metric_name}, got {:?}",
                sample.labels
            );
        }

        #[test]
        fn test_concurrency_wait_emitted_without_team_id_label() {
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                log.team_id = Some(99);
                log.concurrency_limit_wait_ms = Some(12);
                log.emit_timing_metrics();
            });

            let metrics = snapshot_metrics(&recorder);
            let cw = metrics
                .iter()
                .find(|s| s.name == "flags_concurrency_limit_wait_ms")
                .expect("flags_concurrency_limit_wait_ms not emitted");
            // Must NOT carry team_id — permit-wait is pod-level, not per-team.
            assert!(
                !cw.labels.iter().any(|(k, _)| k == "team_id"),
                "concurrency_limit_wait_ms should not carry team_id, got {:?}",
                cw.labels
            );
        }

        #[test]
        fn test_unknown_team_id_when_team_unresolved() {
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                // team_id stays None — e.g. token extraction failed before auth.
                log.queue_time_ms = Some(20);
                log.emit_timing_metrics();
            });

            let metrics = snapshot_metrics(&recorder);
            let queue = metrics
                .iter()
                .find(|s| s.name == "flags_queue_time_ms")
                .expect("flags_queue_time_ms not emitted");
            assert!(queue
                .labels
                .iter()
                .any(|(k, v)| k == "team_id" && v == "unknown"));
        }

        #[test]
        fn test_no_emission_when_field_is_none() {
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                log.emit_timing_metrics();
            });

            let metrics = snapshot_metrics(&recorder);
            for name in [
                "flags_queue_time_ms",
                "flags_pre_handler_time_ms",
                "flags_concurrency_limit_wait_ms",
                "flags_body_read_ms",
            ] {
                assert!(
                    !metrics.iter().any(|s| s.name == name),
                    "did not expect {name} to be emitted with no data"
                );
            }
        }

        #[test]
        fn test_negative_queue_time_is_skipped() {
            // Defensive: the endpoint already filters negative deltas, but
            // belt-and-suspenders since `queue_time_ms` is i64.
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                log.queue_time_ms = Some(-5);
                log.emit_timing_metrics();
            });
            let metrics = snapshot_metrics(&recorder);
            assert!(!metrics.iter().any(|s| s.name == "flags_queue_time_ms"));
        }

        #[test]
        fn test_body_read_emitted_without_team_id_label() {
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                log.team_id = Some(123);
                log.body_read_ms = Some(7.0);
                log.emit_timing_metrics();
            });

            let metrics = snapshot_metrics(&recorder);
            let sample = metrics
                .iter()
                .find(|s| s.name == "flags_body_read_ms")
                .expect("flags_body_read_ms not emitted");
            // Pod-level: must NOT carry team_id, mirroring concurrency_limit_wait_ms.
            assert!(
                !sample.labels.iter().any(|(k, _)| k == "team_id"),
                "body_read_ms should not carry team_id, got {:?}",
                sample.labels
            );
        }
    }

    mod emit_phase_metrics_tests {
        use super::*;
        use crate::handler::phases::Phase;
        use std::time::Duration;

        #[test]
        fn test_emits_one_sample_per_recorded_phase() {
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                log.team_id = Some(42);
                log.phases.record(Phase::Auth, Duration::from_millis(3));
                log.phases
                    .record(Phase::Evaluate, Duration::from_millis(50));
                log.emit_phase_metrics();
            });

            let metrics = snapshot_metrics(&recorder);
            let phase_samples: Vec<_> = metrics
                .iter()
                .filter(|s| s.name == "flags_phase_duration_ms")
                .collect();
            assert_eq!(phase_samples.len(), 2);

            let phase_label_values: Vec<&str> = phase_samples
                .iter()
                .flat_map(|s| {
                    s.labels
                        .iter()
                        .filter(|(k, _)| k == "phase")
                        .map(|(_, v)| v.as_str())
                })
                .collect();
            assert!(phase_label_values.contains(&"auth"));
            assert!(phase_label_values.contains(&"evaluate"));
        }

        #[test]
        fn test_phase_carries_team_id_label() {
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                log.team_id = Some(777);
                log.phases
                    .record(Phase::Cookieless, Duration::from_millis(1));
                log.emit_phase_metrics();
            });

            let metrics = snapshot_metrics(&recorder);
            let sample = metrics
                .iter()
                .find(|s| s.name == "flags_phase_duration_ms")
                .expect("flags_phase_duration_ms not emitted");
            assert!(sample
                .labels
                .iter()
                .any(|(k, v)| k == "team_id" && v == "777"));
            assert!(sample
                .labels
                .iter()
                .any(|(k, v)| k == "phase" && v == "cookieless"));
        }

        #[test]
        fn test_phase_unknown_team_when_unresolved() {
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                // team_id stays None (e.g. auth failed before team resolution).
                log.phases.record(Phase::Auth, Duration::from_millis(2));
                log.emit_phase_metrics();
            });

            let metrics = snapshot_metrics(&recorder);
            let sample = metrics
                .iter()
                .find(|s| s.name == "flags_phase_duration_ms")
                .expect("flags_phase_duration_ms not emitted");
            assert!(sample
                .labels
                .iter()
                .any(|(k, v)| k == "team_id" && v == "unknown"));
        }

        #[test]
        fn test_no_emission_when_no_phases_recorded() {
            let recorder = DebuggingRecorder::new();
            metrics::with_local_recorder(&recorder, || {
                let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
                log.emit_phase_metrics();
            });
            let metrics = snapshot_metrics(&recorder);
            assert!(!metrics.iter().any(|s| s.name == "flags_phase_duration_ms"));
        }
    }

    #[test]
    fn test_with_canonical_log_no_ops_outside_scope() {
        // Should not panic when called outside a scope
        with_canonical_log(|log| log.team_id = Some(123));
    }

    #[tokio::test]
    async fn test_run_with_canonical_log_provides_access() {
        let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());

        let (result, final_log) = run_with_canonical_log(log, async {
            with_canonical_log(|l| l.team_id = Some(456));
            with_canonical_log(|l| l.flags_evaluated = 10);
            with_canonical_log(|l| l.eval.property_cache_hits += 1);
            with_canonical_log(|l| l.eval.property_cache_hits += 1);
            "done"
        })
        .await;

        assert_eq!(result, "done");
        assert_eq!(final_log.team_id, Some(456));
        assert_eq!(final_log.flags_evaluated, 10);
        assert_eq!(final_log.eval.property_cache_hits, 2);
    }

    #[tokio::test]
    async fn test_run_with_canonical_log_returns_modified_log() {
        let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());

        let (_, final_log) = run_with_canonical_log(log, async {
            with_canonical_log(|l| {
                l.db_property_fetches = 3;
                l.eval.cohorts_evaluated = 5;
                l.hash_key_override_status = Some("found");
            });
        })
        .await;

        assert_eq!(final_log.db_property_fetches, 3);
        assert_eq!(final_log.eval.cohorts_evaluated, 5);
        assert_eq!(final_log.hash_key_override_status, Some("found"));
    }

    #[tokio::test]
    async fn test_dependency_graph_errors_is_tracked() {
        let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());

        let graph_errors_count = 2;
        let (_, final_log) = run_with_canonical_log(log, async {
            with_canonical_log(|l| l.dependency_graph_errors = graph_errors_count);
        })
        .await;

        assert_eq!(
            final_log.dependency_graph_errors, graph_errors_count,
            "dependency_graph_errors should track the count of graph construction errors"
        );
    }

    #[test]
    fn test_clone_creates_independent_copy() {
        // Clone correctness is compiler-verified via #[derive(Clone)].
        // This test verifies clone creates an independent copy.
        let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
        log.team_id = Some(123);

        let mut cloned = log.clone();
        cloned.team_id = Some(456);

        // Ensure original is unmodified
        assert_eq!(log.team_id, Some(123));
        assert_eq!(cloned.team_id, Some(456));
    }

    mod task_local_isolation_tests {
        use super::*;

        #[tokio::test]
        async fn test_concurrent_requests_are_isolated() {
            // Spawn multiple concurrent tasks that each modify their own canonical log.
            // Verify that modifications in one task don't affect other tasks.
            let handles: Vec<_> = (0..10)
                .map(|i| {
                    tokio::spawn(async move {
                        let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), format!("10.0.0.{i}"));

                        let (_, final_log) = run_with_canonical_log(log, async {
                            // Each task sets its own unique team_id
                            with_canonical_log(|l| l.team_id = Some(i as i32));
                            with_canonical_log(|l| l.flags_evaluated = i);

                            // Small delay to allow interleaving
                            tokio::task::yield_now().await;

                            // Update more fields after yield
                            with_canonical_log(|l| l.eval.property_cache_hits = i * 2);
                        })
                        .await;

                        // Return the final values for verification
                        (
                            i,
                            final_log.team_id,
                            final_log.flags_evaluated,
                            final_log.eval.property_cache_hits,
                        )
                    })
                })
                .collect();

            // Verify each task got its own isolated log
            for handle in handles {
                let (i, team_id, flags_evaluated, cache_hits) = handle.await.unwrap();
                assert_eq!(team_id, Some(i as i32), "team_id should match task index");
                assert_eq!(
                    flags_evaluated, i,
                    "flags_evaluated should match task index"
                );
                assert_eq!(
                    cache_hits,
                    i * 2,
                    "property_cache_hits should match task index * 2"
                );
            }
        }

        #[tokio::test]
        async fn test_nested_scopes_are_independent() {
            // Test that we can't accidentally nest canonical log scopes
            // (each run_with_canonical_log creates a new scope)
            let outer_log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "outer".to_string());

            let (_, outer_final) = run_with_canonical_log(outer_log, async {
                with_canonical_log(|l| l.team_id = Some(1));

                // Start a nested scope with different values
                let inner_log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "inner".to_string());
                let (_, inner_final) = run_with_canonical_log(inner_log, async {
                    with_canonical_log(|l| l.team_id = Some(2));
                })
                .await;

                // Inner scope should have its own values
                assert_eq!(inner_final.team_id, Some(2));
                assert_eq!(inner_final.ip, "inner");

                // After inner scope ends, we should still be in outer scope
                with_canonical_log(|l| l.flags_evaluated = 100);
            })
            .await;

            // Outer scope should have its own values, unaffected by inner scope
            assert_eq!(outer_final.team_id, Some(1));
            assert_eq!(outer_final.flags_evaluated, 100);
            assert_eq!(outer_final.ip, "outer");
        }

        #[tokio::test]
        async fn test_with_canonical_log_outside_scope_is_safe() {
            // Call with_canonical_log outside any scope - should no-op without panic
            with_canonical_log(|l| l.team_id = Some(999));

            // Now run a proper scope and verify the outside call had no effect
            let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "test".to_string());
            let (_, final_log) = run_with_canonical_log(log, async {
                // Don't modify team_id
            })
            .await;

            assert!(final_log.team_id.is_none());
        }
    }

    mod set_error_tests {
        use super::*;
        use common_cookieless::CookielessManagerError;
        use rstest::rstest;

        #[rstest]
        #[case(FlagError::ClientFacing(ClientFacingError::BadRequest("test".into())), 400, "bad_request")]
        #[case(FlagError::ClientFacing(ClientFacingError::Unauthorized("test".into())), 401, "unauthorized")]
        #[case(
            FlagError::ClientFacing(ClientFacingError::RateLimited),
            429,
            "rate_limited"
        )]
        #[case(
            FlagError::ClientFacing(ClientFacingError::IpRateLimited),
            429,
            "ip_rate_limited"
        )]
        #[case(
            FlagError::ClientFacing(ClientFacingError::TokenRateLimited),
            429,
            "token_rate_limited"
        )]
        #[case(
            FlagError::ClientFacing(ClientFacingError::BillingLimit),
            402,
            "billing_limit"
        )]
        #[case(
            FlagError::ClientFacing(ClientFacingError::ServiceUnavailable),
            503,
            "service_unavailable"
        )]
        #[case(FlagError::internal(anyhow::anyhow!("test")), 500, "internal_error")]
        #[case(FlagError::RequestDecodingError("test".into()), 400, "request_decoding_error")]
        #[case(FlagError::MissingDistinctId, 400, "missing_distinct_id")]
        #[case(FlagError::NoTokenError, 401, "missing_token")]
        #[case(FlagError::TokenValidationError, 401, "invalid_token")]
        #[case(FlagError::PersonalApiKeyInvalid, 401, "personal_api_key_invalid")]
        #[case(FlagError::SecretApiTokenInvalid, 401, "secret_api_token_invalid")]
        #[case(FlagError::NoAuthenticationProvided, 401, "no_authentication")]
        #[case(FlagError::RowNotFound, 500, "row_not_found")]
        #[case(FlagError::flag_data_parsing("test"), 500, "flag_data_parsing_error")]
        #[case(FlagError::redis_unavailable(anyhow::anyhow!("connection refused")), 503, "redis_unavailable")]
        #[case(FlagError::DatabaseUnavailable, 503, "database_unavailable")]
        #[case(FlagError::TimeoutError(None), 503, "timeout")]
        #[case(FlagError::TimeoutError(Some("pool".into())), 503, "timeout")]
        #[case(
            FlagError::DependencyNotFound(DependencyType::Flag, 1),
            500,
            "dependency_not_found"
        )]
        #[case(
            FlagError::DependencyCycle(DependencyType::Cohort, 2),
            500,
            "dependency_cycle"
        )]
        #[case(
            FlagError::CohortFiltersParsingError,
            500,
            "cohort_filters_parsing_error"
        )]
        #[case(FlagError::person_not_found(), 503, "person_not_found")]
        #[case(FlagError::cache_miss(), 503, "cache_miss")]
        #[case(FlagError::data_parsing(anyhow::anyhow!("bad payload")), 500, "data_parsing_error")]
        #[case(
            FlagError::batch_evaluation_panicked(),
            500,
            "batch_evaluation_panicked"
        )]
        #[case(FlagError::HashKeyOverrideError, 500, "hash_key_override_error")]
        fn test_set_error_populates_fields(
            #[case] error: FlagError,
            #[case] expected_status: u16,
            #[case] expected_code: &'static str,
        ) {
            let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());

            log.set_error(&error);

            assert_eq!(
                log.http_status, expected_status,
                "http_status mismatch for {error:?}"
            );
            assert_eq!(
                log.error_code,
                Some(expected_code),
                "error_code mismatch for {error:?}"
            );
        }

        #[test]
        fn test_set_error_cookieless_bad_request() {
            let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
            let error: FlagError = CookielessManagerError::MissingProperty("test".into()).into();

            log.set_error(&error);

            assert_eq!(log.http_status, 400);
            assert_eq!(log.error_code, Some("cookieless_error"));
        }

        #[test]
        fn test_emit_for_error_sets_error_and_emits() {
            let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
            let error = FlagError::NoTokenError;

            log.emit_for_error(&error);

            // Verify error fields were set (emit_for_error calls set_error internally)
            assert_eq!(log.http_status, 401);
            assert_eq!(log.error_code, Some("missing_token"));
        }
    }

    mod hash_key_override_status_tests {
        use super::*;
        use rstest::rstest;

        #[rstest]
        #[case(None, "no experience continuity flags")]
        #[case(Some("skipped"), "optimization applied - 100% rollout")]
        #[case(Some("error"), "query failed")]
        #[case(Some("empty"), "query succeeded, no overrides")]
        #[case(Some("found"), "query succeeded, overrides returned")]
        fn test_hash_key_override_status_values(
            #[case] status: Option<&'static str>,
            #[case] _description: &str,
        ) {
            let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
            log.hash_key_override_status = status;
            // Verify the value can be set and emit doesn't panic
            assert_eq!(log.hash_key_override_status, status);
            log.emit();
        }

        #[tokio::test]
        async fn test_hash_key_override_status_skipped_in_scope() {
            let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());

            let (_, final_log) = run_with_canonical_log(log, async {
                with_canonical_log(|l| l.hash_key_override_status = Some("skipped"));
            })
            .await;

            assert_eq!(final_log.hash_key_override_status, Some("skipped"));
        }

        #[tokio::test]
        async fn test_hash_key_override_status_error_in_scope() {
            let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());

            let (_, final_log) = run_with_canonical_log(log, async {
                with_canonical_log(|l| l.hash_key_override_status = Some("error"));
            })
            .await;

            assert_eq!(final_log.hash_key_override_status, Some("error"));
        }

        #[tokio::test]
        async fn test_hash_key_override_status_empty_in_scope() {
            let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());

            let (_, final_log) = run_with_canonical_log(log, async {
                with_canonical_log(|l| l.hash_key_override_status = Some("empty"));
            })
            .await;

            assert_eq!(final_log.hash_key_override_status, Some("empty"));
        }
    }

    mod eval_counters_tests {
        use super::*;

        #[test]
        fn test_merge_sums_counters_and_ors_booleans() {
            let mut base = EvalCounters {
                flags_device_id_bucketing: 1,
                cohorts_evaluated: 2,
                property_cache_hits: 3,
                property_cache_misses: 4,
                person_properties_not_cached: false,
                group_properties_not_cached: true,
            };

            let other = EvalCounters {
                flags_device_id_bucketing: 10,
                cohorts_evaluated: 20,
                property_cache_hits: 30,
                property_cache_misses: 40,
                person_properties_not_cached: true,
                group_properties_not_cached: false,
            };

            base.merge(&other);

            assert_eq!(base.flags_device_id_bucketing, 11);
            assert_eq!(base.cohorts_evaluated, 22);
            assert_eq!(base.property_cache_hits, 33);
            assert_eq!(base.property_cache_misses, 44);
            assert!(base.person_properties_not_cached);
            assert!(base.group_properties_not_cached);
        }

        #[test]
        fn test_merge_default_is_identity() {
            let mut base = EvalCounters {
                flags_device_id_bucketing: 5,
                cohorts_evaluated: 3,
                property_cache_hits: 7,
                property_cache_misses: 2,
                person_properties_not_cached: true,
                group_properties_not_cached: false,
            };
            let snapshot = base.clone();

            base.merge(&EvalCounters::default());

            assert_eq!(
                base.flags_device_id_bucketing,
                snapshot.flags_device_id_bucketing
            );
            assert_eq!(base.cohorts_evaluated, snapshot.cohorts_evaluated);
            assert_eq!(base.property_cache_hits, snapshot.property_cache_hits);
            assert_eq!(base.property_cache_misses, snapshot.property_cache_misses);
            assert_eq!(
                base.person_properties_not_cached,
                snapshot.person_properties_not_cached
            );
            assert_eq!(
                base.group_properties_not_cached,
                snapshot.group_properties_not_cached
            );
        }
    }

    mod merge_rayon_delta_tests {
        use super::*;
        use rstest::rstest;

        #[rstest]
        // Single delta with all counter fields populated
        #[case(
            &[(3, 5, 10, 2, true, false)],
            3, 5, 10, 2, true, false
        )]
        // Two deltas accumulate: counters sum, booleans OR
        #[case(
            &[(1, 2, 3, 1, false, false), (2, 3, 7, 4, false, true)],
            3, 5, 10, 5, false, true
        )]
        // Empty delta is a no-op
        #[case(
            &[(0, 0, 0, 0, false, false)],
            0, 0, 0, 0, false, false
        )]
        // Boolean fields OR across deltas
        #[case(
            &[(0, 0, 0, 0, true, false), (0, 0, 0, 0, false, true)],
            0, 0, 0, 0, true, true
        )]
        fn test_merge_rayon_delta_from_zero(
            #[case] deltas: &[(usize, usize, usize, usize, bool, bool)],
            #[case] expected_device_id: usize,
            #[case] expected_cohorts: usize,
            #[case] expected_cache_hits: usize,
            #[case] expected_cache_misses: usize,
            #[case] expected_person_not_cached: bool,
            #[case] expected_group_not_cached: bool,
        ) {
            let mut log = FlagsCanonicalLogLine::default();

            for &(device_id, cohorts, hits, misses, person, group) in deltas {
                let delta = FlagsCanonicalLogLine {
                    eval: EvalCounters {
                        flags_device_id_bucketing: device_id,
                        cohorts_evaluated: cohorts,
                        property_cache_hits: hits,
                        property_cache_misses: misses,
                        person_properties_not_cached: person,
                        group_properties_not_cached: group,
                    },
                    ..Default::default()
                };
                log.merge_rayon_delta(&delta);
            }

            assert_eq!(log.eval.flags_device_id_bucketing, expected_device_id);
            assert_eq!(log.eval.cohorts_evaluated, expected_cohorts);
            assert_eq!(log.eval.property_cache_hits, expected_cache_hits);
            assert_eq!(log.eval.property_cache_misses, expected_cache_misses);
            assert_eq!(
                log.eval.person_properties_not_cached,
                expected_person_not_cached
            );
            assert_eq!(
                log.eval.group_properties_not_cached,
                expected_group_not_cached
            );
        }

        #[test]
        fn test_merge_into_preexisting_counters() {
            let mut log = FlagsCanonicalLogLine {
                eval: EvalCounters {
                    property_cache_hits: 10,
                    property_cache_misses: 5,
                    cohorts_evaluated: 3,
                    flags_device_id_bucketing: 2,
                    person_properties_not_cached: true,
                    group_properties_not_cached: false,
                },
                ..Default::default()
            };

            let delta = FlagsCanonicalLogLine {
                eval: EvalCounters {
                    property_cache_hits: 7,
                    property_cache_misses: 3,
                    cohorts_evaluated: 2,
                    flags_device_id_bucketing: 1,
                    person_properties_not_cached: false,
                    group_properties_not_cached: true,
                },
                ..Default::default()
            };

            log.merge_rayon_delta(&delta);

            assert_eq!(log.eval.property_cache_hits, 17);
            assert_eq!(log.eval.property_cache_misses, 8);
            assert_eq!(log.eval.cohorts_evaluated, 5);
            assert_eq!(log.eval.flags_device_id_bucketing, 3);
            assert!(log.eval.person_properties_not_cached);
            assert!(log.eval.group_properties_not_cached);
        }

        #[test]
        fn test_merge_does_not_touch_request_fields() {
            let mut log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
            log.team_id = Some(123);
            log.flags_evaluated = 50;

            let delta = FlagsCanonicalLogLine {
                team_id: Some(999),
                flags_evaluated: 42,
                eval: EvalCounters {
                    property_cache_hits: 5,
                    ..Default::default()
                },
                ..Default::default()
            };

            log.merge_rayon_delta(&delta);

            // Request-level fields are unchanged
            assert_eq!(log.team_id, Some(123));
            assert_eq!(log.flags_evaluated, 50);
            // Evaluation counter is merged
            assert_eq!(log.eval.property_cache_hits, 5);
        }
    }

    mod rayon_fallback_tests {
        use super::*;

        #[test]
        fn test_install_and_take_round_trip() {
            let _guard = install_rayon_canonical_log();

            // Modify via with_canonical_log (outside tokio scope, falls back to thread-local)
            with_canonical_log(|log| {
                log.eval.property_cache_hits = 7;
                log.eval.cohorts_evaluated = 3;
            });

            let delta = take_rayon_canonical_log();
            assert!(delta.is_some());
            let delta = delta.unwrap();
            assert_eq!(delta.eval.property_cache_hits, 7);
            assert_eq!(delta.eval.cohorts_evaluated, 3);
        }

        #[test]
        fn test_take_without_install_returns_none() {
            // Ensure clean state
            drop(take_rayon_canonical_log());
            assert!(take_rayon_canonical_log().is_none());
        }

        #[test]
        fn test_take_clears_the_thread_local() {
            let _guard = install_rayon_canonical_log();
            with_canonical_log(|log| log.eval.property_cache_hits = 1);

            let first = take_rayon_canonical_log();
            assert!(first.is_some());

            let second = take_rayon_canonical_log();
            assert!(second.is_none());
        }

        #[test]
        fn test_install_replaces_previous() {
            let _guard = install_rayon_canonical_log();
            with_canonical_log(|log| log.eval.property_cache_hits = 99);

            // Re-install should reset
            let _guard = install_rayon_canonical_log();
            let delta = take_rayon_canonical_log().unwrap();
            assert_eq!(delta.eval.property_cache_hits, 0);
        }

        #[test]
        fn test_guard_cleans_up_on_panic() {
            use std::panic::catch_unwind;

            // Ensure clean state
            drop(take_rayon_canonical_log());

            let result = catch_unwind(|| {
                let _guard = install_rayon_canonical_log();
                with_canonical_log(|log| log.eval.property_cache_hits = 42);
                panic!("simulated evaluation panic");
            });

            assert!(result.is_err(), "should have caught the panic");
            // The guard's Drop should have cleared the thread-local
            assert!(
                take_rayon_canonical_log().is_none(),
                "thread-local should be clean after panic"
            );
        }

        #[test]
        fn test_with_canonical_log_noop_without_either_scope() {
            // Ensure thread-local is clean
            drop(take_rayon_canonical_log());

            // Outside both tokio scope and rayon scope: should no-op without panic
            with_canonical_log(|log| log.team_id = Some(123));
        }

        #[tokio::test]
        async fn test_tokio_scope_takes_precedence_over_thread_local() {
            // Install a rayon thread-local
            let _guard = install_rayon_canonical_log();

            // Run inside a tokio canonical log scope
            let log = FlagsCanonicalLogLine::new(Uuid::new_v4(), "10.0.0.1".to_string());
            let (_, final_log) = run_with_canonical_log(log, async {
                with_canonical_log(|l| l.eval.property_cache_hits = 42);
            })
            .await;

            // The tokio scope should have received the update
            assert_eq!(final_log.eval.property_cache_hits, 42);

            // The rayon thread-local should be untouched
            let rayon_delta = take_rayon_canonical_log().unwrap();
            assert_eq!(rayon_delta.eval.property_cache_hits, 0);
        }

        #[test]
        fn test_rayon_threads_are_isolated() {
            use std::sync::{Arc, Barrier};

            let barrier = Arc::new(Barrier::new(2));
            let (tx1, rx1) = std::sync::mpsc::channel();
            let (tx2, rx2) = std::sync::mpsc::channel();

            let b1 = barrier.clone();
            rayon::spawn(move || {
                let _guard = install_rayon_canonical_log();
                with_canonical_log(|log| log.eval.property_cache_hits = 111);
                b1.wait(); // sync so both threads are alive simultaneously
                let delta = take_rayon_canonical_log().unwrap();
                tx1.send(delta.eval.property_cache_hits).unwrap();
            });

            let b2 = barrier.clone();
            rayon::spawn(move || {
                let _guard = install_rayon_canonical_log();
                with_canonical_log(|log| log.eval.property_cache_hits = 222);
                b2.wait();
                let delta = take_rayon_canonical_log().unwrap();
                tx2.send(delta.eval.property_cache_hits).unwrap();
            });

            assert_eq!(rx1.recv().unwrap(), 111);
            assert_eq!(rx2.recv().unwrap(), 222);
        }
    }

    mod truncate_chars_tests {
        use super::*;
        use rstest::rstest;

        #[rstest]
        #[case("", 10, "")]
        #[case("hello", 10, "hello")]
        #[case("hello", 5, "hello")]
        #[case("hello", 3, "hel")]
        #[case("hello", 0, "")]
        // Multibyte UTF-8: emojis are 4 bytes each but 1 character
        #[case("🎉🎊🎁", 3, "🎉🎊🎁")]
        #[case("🎉🎊🎁", 2, "🎉🎊")]
        #[case("🙂🙂🙂", 2, "🙂🙂")]
        #[case("hello🎉world", 6, "hello🎉")]
        // Mixed ASCII and multibyte
        #[case("café", 4, "café")]
        #[case("café", 3, "caf")]
        #[case("💩abc", 2, "💩a")]
        fn test_truncate_chars(
            #[case] input: &str,
            #[case] max_chars: usize,
            #[case] expected: &str,
        ) {
            assert_eq!(truncate_chars(input, max_chars), expected);
        }

        #[test]
        fn test_truncate_chars_long_input() {
            let long_input = "a".repeat(2000);
            let truncated = truncate_chars(&long_input, 512);
            assert_eq!(truncated.len(), 512);
            assert_eq!(truncated.chars().count(), 512);
        }
    }
}