Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

36.0 KB
Raw Blame
// Group type cache metrics
pub const GROUP_TYPE_CACHE_HIT_COUNTER: &str = "flags_group_type_cache_hit_total";
pub const GROUP_TYPE_CACHE_MISS_COUNTER: &str = "flags_group_type_cache_miss_total";
pub const GROUP_TYPE_CACHE_ENTRIES_GAUGE: &str = "flags_group_type_cache_entries";
pub const DB_GROUP_TYPE_READS_COUNTER: &str = "flags_db_group_type_reads_total";
pub const DB_GROUP_TYPE_ERRORS_COUNTER: &str = "flags_db_group_type_errors_total";

// Flag evaluation counters
pub const FLAG_EVALUATION_ERROR_COUNTER: &str = "flags_flag_evaluation_error_total";
pub const FLAG_HASH_KEY_WRITES_COUNTER: &str = "flags_flag_hash_key_writes_total";
pub const FLAG_HASH_KEY_RETRIES_COUNTER: &str = "flags_hash_key_retries_total";
pub const TEAM_CACHE_HIT_COUNTER: &str = "flags_team_cache_hit_total";
pub const DB_TEAM_READS_COUNTER: &str = "flags_db_team_reads_total";
pub const TOKEN_VALIDATION_ERRORS_COUNTER: &str = "flags_token_validation_errors_total";
pub const TEAM_NEGATIVE_CACHE_HIT_COUNTER: &str = "flags_team_negative_cache_hit_total";
pub const PG_TEAM_FALLBACK_SKIPPED_COUNTER: &str = "flags_pg_team_fallback_skipped_total";
pub const DB_COHORT_READS_COUNTER: &str = "flags_db_cohort_reads_total";
pub const DB_COHORT_ERRORS_COUNTER: &str = "flags_db_cohort_errors_total";
pub const COHORT_CACHE_HIT_COUNTER: &str = "flags_cohort_cache_hit_total";
pub const COHORT_CACHE_MISS_COUNTER: &str = "flags_cohort_cache_miss_total";
pub const COHORT_CACHE_SIZE_BYTES_GAUGE: &str = "flags_cohort_cache_size_bytes";
pub const COHORT_CACHE_ENTRIES_GAUGE: &str = "flags_cohort_cache_entries";
// Incremented once per unsupported cohort filter leaf (e.g. a `behavioral` filter)
// skipped during dependency extraction, instead of failing the whole cohort parse.
pub const COHORT_UNSUPPORTED_FILTER_COUNTER: &str = "flags_cohort_unsupported_filter_total";
// Incremented once per cohort whose filters fail dependency extraction or evaluation
// with CohortFiltersParsingError — most commonly a malformed leaf of a known type
// (CohortValuesItem::MalformedKnownType), but also excessive nesting depth or other
// structural errors. Cohort and team ids are in the companion debug log, not metric
// labels (cardinality).
pub const COHORT_MALFORMED_FILTER_COUNTER: &str = "flags_cohort_malformed_filter_total";
// Counts evaluated cohorts the two MembershipStampPolicy variants route differently.
// Labels: direction="would_lose" | "would_gain", active_policy = the policy this process
// runs, which says whether the reroute is still pending or already applied. Cohort and
// team ids are in the companion deduped warn log (cardinality).
pub const FLAG_COHORT_STAMP_POLICY_DIVERGENCE_COUNTER: &str =
    "flags_cohort_stamp_policy_divergence_total";
// Realtime cohort membership cache (CachedCohortMembershipProvider, keyed on
// (team_id, person_uuid)). hit = lookup fully served from cache; miss = a
// behavioral cohorts DB query was issued (no cache entry, or the entry was
// missing some of the requested cohort IDs).
pub const COHORT_MEMBERSHIP_CACHE_HIT_COUNTER: &str = "flags_cohort_membership_cache_hit_total";
pub const COHORT_MEMBERSHIP_CACHE_MISS_COUNTER: &str = "flags_cohort_membership_cache_miss_total";
pub const COHORT_MEMBERSHIP_CACHE_ENTRIES_GAUGE: &str = "flags_cohort_membership_cache_entries";
// Behavioral cohorts DB reads for realtime cohort membership (RealtimeCohortMembershipProvider)
pub const DB_COHORT_MEMBERSHIP_READS_COUNTER: &str = "flags_db_cohort_membership_reads_total";
pub const DB_COHORT_MEMBERSHIP_ERRORS_COUNTER: &str = "flags_db_cohort_membership_errors_total";
// In-memory flag definitions cache (deserialized + regex-compiled).
// Keyed on `(team_id, etag)` where `etag` is the version tag Django writes
// alongside the hypercache payload (`enable_etag=True`). Cache hits avoid the
// payload fetch + deserialization entirely.
pub const FLAG_DEFINITIONS_INMEM_CACHE_HIT_COUNTER: &str =
    "flags_definitions_inmem_cache_hit_total";
pub const FLAG_DEFINITIONS_INMEM_CACHE_MISS_COUNTER: &str =
    "flags_definitions_inmem_cache_miss_total";
pub const FLAG_DEFINITIONS_INMEM_CACHE_SIZE_BYTES_GAUGE: &str =
    "flags_definitions_inmem_cache_size_bytes";
pub const FLAG_DEFINITIONS_INMEM_CACHE_ENTRIES_GAUGE: &str =
    "flags_definitions_inmem_cache_entries";
// Counter for requests that bypassed the version-keyed fast path. Labels are
// mutually exclusive — a bypassing request increments exactly one:
//   reason="sentinel"         — Django wrote `__missing__` (empty team).
//   reason="etag_missing"     — etag key absent but the loader returned a
//                                non-empty wrapper (TTL drift / pre-etag).
//   reason="etag_redis_error" — the `get_etag` call itself failed.
pub const FLAG_DEFINITIONS_INMEM_CACHE_NO_VERSION_COUNTER: &str =
    "flags_definitions_inmem_cache_no_version_total";
// Cohort source for flag evaluation
// Labels: source="preloaded" (from flags hypercache) | source="cache_manager" (CohortCacheManager fallback)
pub const FLAG_COHORT_SOURCE_COUNTER: &str = "flags_cohort_source_total";
pub const PROPERTY_CACHE_HITS_COUNTER: &str = "flags_property_cache_hits_total";
pub const PROPERTY_CACHE_MISSES_COUNTER: &str = "flags_property_cache_misses_total";
pub const DB_PERSON_AND_GROUP_PROPERTIES_READS_COUNTER: &str =
    "flags_db_person_and_group_properties_reads_total";
pub const FLAG_REQUESTS_COUNTER: &str = "flags_requests_total";
pub const FLAG_REQUESTS_LATENCY: &str = "flags_requests_duration_ms";
// Incremented once per request that supplied a `$geoip_*` value disagreeing with the MaxMind
// lookup. Supplied values win over the lookup, so this counts the requests that evaluate
// differently than they would under lookup-wins precedence. Removable once the precedence
// change has shipped and settled. Per-team and per-SDK attribution lives in the canonical log
// line via Loki.
pub const GEOIP_PROPERTIES_DIFFER_FROM_LOOKUP_COUNTER: &str =
    "flags_geoip_properties_differ_from_lookup_total";

// Internal batch flag evaluation endpoint (static cohort generation). Dedicated
// `flags_batch_eval_*` names keep batch traffic separable from live `/flags` metrics.
// Not to be confused with the pre-existing `flags_batch_evaluation_*` family below
// (per-request sequential/parallel strategy metrics emitted from flag_matching.rs).
pub const FLAG_BATCH_EVAL_REQUESTS_COUNTER: &str = "flags_batch_eval_requests_total";
pub const FLAG_BATCH_EVAL_PERSONS_COUNTER: &str = "flags_batch_eval_persons_total";
pub const FLAG_BATCH_EVAL_PERSON_RETRIES_COUNTER: &str = "flags_batch_eval_person_retries_total";
pub const FLAG_BATCH_EVAL_TIME: &str = "flags_batch_eval_duration_ms";
pub const FLAG_QUEUE_TIME_MS: &str = "flags_queue_time_ms";
pub const FLAG_REQUEST_FAULTS_COUNTER: &str = "flags_request_faults_total";

// Pre-handler timing decomposition for `flags_queue_time_ms`.
// Together these subdivide the "Envoy stamp → handler entry" wall time into
// known synchronous pre-handler work + (residual) proxy/tower wait, so we
// can attribute spikes to the right tier instead of guessing.

// Total time spent in synchronous pre-handler work inside `endpoint::flags`
// (UA parse, IP rate-limit, token extract, token rate-limit). Labeled by
// `team_id` so noisy customers are attributable.
pub const FLAG_PRE_HANDLER_TIME_MS: &str = "flags_pre_handler_time_ms";

// Per-step rate-limit check timing. Labeled by `kind="ip"|"token"` to
// distinguish the two rate-limiter calls inside the endpoint.
pub const FLAG_RATE_LIMIT_CHECK_TIME_MS: &str = "flags_rate_limit_check_ms";

// Time spent inside `decoding::extract_token` (sync JSON DOM scan over the
// raw body). Pathological large bodies are the suspected outlier driver.
pub const FLAG_TOKEN_EXTRACT_TIME_MS: &str = "flags_token_extract_ms";

// Counter of FLAGS_LOG_BODIES_TEAMS refresh attempts. Labeled by
// `result="success"|"failure"`. Emit absence-of-success alerts on this
// (e.g., `rate(flags_body_log_refresh_total{result="success"}[10m]) == 0`)
// to detect a stuck cache while DB or parse errors keep firing.
pub const FLAG_BODY_LOG_REFRESH_TOTAL: &str = "flags_body_log_refresh_total";

// Permit-acquisition wait time on the tower `ConcurrencyLimitLayer`.
// Populated by Phase F; emitted only when populated. No `team_id` label
// because permit wait is a property of pod-level load, not of any one team.
pub const FLAG_CONCURRENCY_LIMIT_WAIT_TIME_MS: &str = "flags_concurrency_limit_wait_ms";

// Wall-clock time spent buffering the inbound POST body. Captured by a
// middleware shim placed between `record_concurrency_wait` and the handler
// so the duration excludes permit wait but includes any slow-client
// upload or large-compressed-payload buffering. No `team_id` label
// because this measurement happens before the request is authenticated.
pub const FLAG_BODY_READ_TIME_MS: &str = "flags_body_read_ms";

// Counter for body-buffering failures inside `record_body_read`
// (network error, peer disconnect, malformed framing). Pod-level: no
// `team_id` label, because this happens before authentication.
pub const FLAG_BODY_READ_FAILED_COUNTER: &str = "flags_body_read_failed_total";

// Counter for inbound bodies rejected because they exceeded
// `MAX_FLAGS_BODY_BYTES`. Split from the transport-failure counter so a
// dashboard can distinguish suspected upload abuse from genuine network
// errors. Pod-level: no `team_id` label.
pub const FLAG_BODY_READ_TOO_LARGE_COUNTER: &str = "flags_body_read_too_large_total";

// Counter for requests rejected because the gzip-*decompressed* body exceeded
// `MAX_FLAGS_DECOMPRESSED_BYTES`. Distinct from `FLAG_BODY_READ_TOO_LARGE_COUNTER`,
// which fires on the *compressed* size before decompression.
pub const FLAG_GZIP_OUTPUT_EXCEEDED_COUNTER: &str = "flags_gzip_output_exceeded_total";

// Per-phase wall-clock duration inside `process_request_inner`. Phases
// match handler-level state transitions:
// `auth → billing_check → cookieless → fetch_and_filter → evaluate →
// record_billing → config_response`. Together they reconstruct the
// majority of the post-pre-handler request budget so a spike in any one
// phase is attributable to the responsible await site.
//
// Labels: `phase` (one of the canonical phase names, see
// `handler::phases::Phase::name`), `team_id` (filtered through the
// existing team allowlist).
pub const FLAG_PHASE_DURATION_MS: &str = "flags_phase_duration_ms";

// Per-phase in-flight gauge. Incremented when a `PhaseGuard` enters a
// phase and decremented on drop. When `flags_tokio_alive_tasks` jumps,
// exactly one phase gauge climbs in lockstep — direct localization of
// the parking site without waiting for the histogram to rebucket.
//
// Pod-level by design: only a `phase` label, no `team_id`.
pub const FLAG_INFLIGHT_BY_PHASE: &str = "flags_inflight_by_phase";

// In-memory `FlagDefinitionsCache::get_or_load` total wall-clock time.
// Labels: `outcome` ∈ {`hit`, `miss_load_ok`, `miss_load_err`,
// `etag_missing`, `sentinel`, `fallback`}. A spike in `miss_load_*`
// means the loader (HyperCache fetch + Pickle/JSON decode + regex
// compile) is stalling; a spike in `hit` would indicate Moka itself is
// slow (very unlikely). Concurrent first-misses are not coalesced, so
// thunder-herd events on etag rollover show up as `miss_load_ok` p99
// climbing while the hit rate stays normal. `fallback` is split out
// from `miss_load_ok` so a PG-fallback storm is visible on its own.
pub const FLAG_DEFINITIONS_INMEM_LOAD_MS: &str = "flags_definitions_inmem_load_ms";

// Performance monitoring
pub const DB_CONNECTION_POOL_ACTIVE_COUNTER: &str = "flags_db_connection_pool_active_total";
pub const DB_CONNECTION_POOL_IDLE_COUNTER: &str = "flags_db_connection_pool_idle_total";
pub const DB_CONNECTION_POOL_MAX_COUNTER: &str = "flags_db_connection_pool_max_total";
pub const DB_CONNECTION_POOL_SIZE_GAUGE: &str = "flags_db_connection_pool_size";

// Billing aggregator metrics
// See `src/billing/aggregator.rs`. The accounting identity
//   flags_billing_records_total ≈ flags_billing_entries_flushed_total
//                                + flags_billing_unflushed_requests_total{cause="cap_drop"}
//                                + flags_billing_unflushed_requests_total{cause="flush_dropped_on_error"}
//                                + flags_billing_unflushed_requests_total{cause="shutdown_drop"}
//                                + flags_billing_pending_records (the residual still in `pending`)
// should hold per pod over any window. Every cause in
// `flags_billing_unflushed_requests_total` is a terminal loss — chunk
// failures during normal (`BailOnError`) ticks are not booked here because
// the records get requeued; those are visible separately on
// `flags_billing_flush_blocked_requests_total`.

// Counter, labeled by `request_type` ("decide" | "flag_definitions").
pub const FLAGS_BILLING_RECORDS: &str = "flags_billing_records_total";

// Counter: sum of `count` args across all successfully written HINCRBYs.
pub const FLAGS_BILLING_ENTRIES_FLUSHED: &str = "flags_billing_entries_flushed_total";

// Gauge of the live `pending` map size. Sustained growth is the leading
// indicator of a wedged flusher; alert when the gauge approaches
// `max_pending_entries`. Pair with `flags_billing_seconds_since_successful_flush`
// to distinguish a hung `execute_pipeline` (gauge growing, staleness rising)
// from steady-state high traffic (gauge oscillating, staleness near zero).
pub const FLAGS_BILLING_PENDING_ENTRIES: &str = "flags_billing_pending_entries";

// Gauge: sum of counts in `pending` — the per-pod count of records that
// would be lost if the process crashed at scrape time (SIGKILL past
// `terminationGracePeriodSeconds`, OOM-kill, node loss, panic). The
// crashed pod cannot emit at the moment of loss, so this gauge is the
// pre-crash signal: scrape it just before the pod is gone, and you have
// an upper bound on the loss. Use `sum(flags_billing_pending_records)`
// for fleet-wide loss exposure, and correlate per-pod spikes with
// `kube_pod_container_status_restarts_total` to attribute crash-loss to
// specific restarts. Differs from `flags_billing_pending_entries`
// (number of distinct keys), which can stay flat while record counts
// per key climb.
pub const FLAGS_BILLING_PENDING_RECORDS: &str = "flags_billing_pending_records";

// Gauge: seconds elapsed since the last successful flush (0 until the
// first successful flush completes).
pub const FLAGS_BILLING_SECONDS_SINCE_SUCCESSFUL_FLUSH: &str =
    "flags_billing_seconds_since_successful_flush";

pub const FLAGS_BILLING_FLUSH_DURATION_MS: &str = "flags_billing_flush_duration_ms";

// Counters: records the usage-ingestion mirror accepted / gave up on. A gap
// between them and `flags_billing_entries_flushed_total` is expected while the
// mirror is rolled out to a subset of teams.
// `flags_usage_records_failed_total` carries an `error_code` label naming why the
// records went nowhere: a gRPC code, `rejected` for what the service declined, or
// `queue_full`. Retries that later succeeded read on `flags_usage_retries_total`
// instead, so a drop here means the records really were lost.
pub const FLAGS_USAGE_RECORDS_SENT: &str = "flags_usage_records_sent_total";
pub const FLAGS_USAGE_RECORDS_FAILED: &str = "flags_usage_records_failed_total";
pub const FLAGS_USAGE_RETRIES: &str = "flags_usage_retries_total";

// Histogram of per-call `record()` latency in microseconds, with no labels
// to keep the hot-path emission allocation-free. The expected uncontended
// p50 is sub-microsecond (one atomic increment + a hash + a HashMap entry
// op). p99 climbing while p50 stays flat is the canonical signature of
// `pending` mutex contention — this metric is the sole signal for it, since
// `record()` would otherwise be invisible to monitoring.
pub const FLAGS_BILLING_RECORD_DURATION_US: &str = "flags_billing_record_duration_us";

// Counter: flusher-side Redis failures, labeled by `error_type`
// ("timeout" | "transport" | "not_found" | "parse" | "config") and
// `outcome` ("applied" | "ambiguous" | "not_applied"). `outcome`
// describes whether the failing call's records likely reached Redis:
//   - "applied":     Redis processed the chunk, but the response was
//                    malformed or otherwise unusable (e.g. ParseError).
//                    The aggregator credits the chunk to
//                    `entries_flushed_total` rather than retrying.
//   - "ambiguous":   Can't tell — timeout or generic transport failure
//                    that could fire before or after Redis applied.
//                    Today the aggregator still retries (`BailOnError`)
//                    or drops (`BestEffort`) like any other error;
//                    breaking this out gives a denominator for the
//                    "is the over-count from retrying ambiguous chunks
//                    actually large enough to care about?" question.
//   - "not_applied": Bytes never reached Redis (e.g. InvalidConfiguration).
//                    Safe to retry.
// Alert on `rate(flags_billing_flush_errors_total[1m]) > 0 for: 30s` to
// catch a wedged Redis link without a separate consecutive-failures gauge.
pub const FLAGS_BILLING_FLUSH_ERRORS: &str = "flags_billing_flush_errors_total";

// Counter: billable requests that did not reach Redis, labeled by `cause`.
// Every cause here is terminal — the request is lost. Use
// `sum(rate(flags_billing_unflushed_requests_total[5m]))` as the
// one-expression billing-leak rate. Cause values:
//   - "cap_drop": dropped at the hot path because `pending_entries` was at
//     `max_pending_entries` and the incoming key was new. Tripwire — any
//     non-zero rate is an incident signal.
//   - "flush_dropped_on_error": drained from `pending`, then lost because a
//     flush chunk errored with `FlushOutcome::Ambiguous` or `NotApplied`
//     AND the flush policy couldn't retry. In practice only fires on the
//     shutdown path (`BestEffort`); normal ticks (`BailOnError`) re-queue
//     failed entries back into `pending` and surface them on
//     `FLAGS_BILLING_FLUSH_BLOCKED_REQUESTS` instead. `Applied` errors
//     under either policy do NOT land here — Redis processed those
//     commands, so the chunk credits `entries_flushed_total`.
//   - "shutdown_drop": entries lost during shutdown because the final flush
//     timed out or panicked. SIGKILL past the grace window lands here.
pub const FLAGS_BILLING_UNFLUSHED_REQUESTS: &str = "flags_billing_unflushed_requests_total";

// Counter: affected-request count for chunks that hit a Redis error during a
// normal-tick (`BailOnError`) flush. Labeled by `error_type` — same
// classification as `FLAGS_BILLING_FLUSH_ERRORS`, so per-error-type rates line
// up across the two metrics. Emits for the failing chunk plus the unattempted
// remainder so the rate reflects all records blocked by the error, not just
// the chunk that tripped it. These records are requeued and retried on the
// next tick, so this is an incident-magnitude signal, not terminal loss —
// keeping it on a separate metric from `FLAGS_BILLING_UNFLUSHED_REQUESTS`
// keeps the terminal-loss identity (above) clean. Suppressed under
// `BestEffort` (shutdown) where the same counts terminally land in
// `flags_billing_unflushed_requests_total{cause="flush_dropped_on_error"}`.
pub const FLAGS_BILLING_FLUSH_BLOCKED_REQUESTS: &str = "flags_billing_flush_blocked_requests_total";

// Flag evaluation timing
pub const FLAG_EVALUATION_TIME: &str = "flags_evaluation_time";
pub const FLAG_HASH_KEY_PROCESSING_TIME: &str = "flags_hash_key_processing_time";
pub const FLAG_DB_PROPERTIES_FETCH_TIME: &str = "flags_properties_db_fetch_time";
pub const FLAG_GROUP_DB_FETCH_TIME: &str = "flags_groups_db_fetch_time"; // this is how long it takes to fetch the group type mappings from the DB
pub const FLAG_GROUP_CACHE_FETCH_TIME: &str = "flags_groups_cache_fetch_time"; // this is how long it takes to fetch the group type mappings from the cache
pub const FLAG_GET_MATCH_TIME: &str = "flags_get_match_time";
pub const FLAG_EVALUATE_ALL_CONDITIONS_TIME: &str = "flags_evaluate_all_conditions_time";
pub const FLAG_PERSON_QUERY_TIME: &str = "flags_person_query_time";
pub const FLAG_DEFINITION_QUERY_TIME: &str = "flags_definition_query_time";
pub const FLAG_PERSON_PROCESSING_TIME: &str = "flags_person_processing_time";
pub const FLAG_COHORT_QUERY_TIME: &str = "flags_cohort_query_time";
pub const FLAG_COHORT_PROCESSING_TIME: &str = "flags_cohort_processing_time";
pub const FLAG_REALTIME_COHORT_QUERY_TIME: &str = "flags_realtime_cohort_query_time";
pub const FLAG_REALTIME_COHORT_QUERY_ERROR_COUNTER: &str =
    "flags_realtime_cohort_query_error_total";
// Behavioral cohorts DB query latency alone (inside RealtimeCohortMembershipProvider),
// as opposed to FLAG_REALTIME_COHORT_QUERY_TIME above which wraps the whole provider
// call at the evaluation site and so mixes cache hits with DB round trips.
// Labels: outcome="success" | "error" | "timeout". Recorded with sub-ms precision.
pub const FLAG_REALTIME_COHORT_DB_QUERY_TIME: &str = "flags_realtime_cohort_db_query_time";
pub const FLAG_GROUP_QUERY_TIME: &str = "flags_group_query_time";
pub const FLAG_GROUP_PROCESSING_TIME: &str = "flags_group_processing_time";
pub const FLAG_DB_CONNECTION_TIME: &str = "flags_db_connection_time";

// Flag request kludges (to see how often we have to massage our request data to be able to parse it)
pub const FLAG_REQUEST_KLUDGE_COUNTER: &str = "flags_request_kludge_total";

// New diagnostic metrics for pool exhaustion investigation
pub const FLAG_POOL_UTILIZATION_GAUGE: &str = "flags_pool_utilization_ratio";
pub const FLAG_CONNECTION_HOLD_TIME: &str = "flags_connection_hold_time_ms";
pub const FLAG_EXPERIENCE_CONTINUITY_REQUESTS_COUNTER: &str =
    "flags_experience_continuity_requests_total";

// Experience continuity optimization metric
// Tracks requests where optimization could apply, with status label:
// - status="skipped": lookup was actually skipped (optimization enabled, no flags needed it)
// - status="eligible": lookup could be skipped but wasn't (optimization feature is disabled)
pub const FLAG_EXPERIENCE_CONTINUITY_OPTIMIZED: &str =
    "flags_experience_continuity_optimized_total";

// Hash key override query result metric
// Tracks the result of hash key override queries to understand cache optimization potential
// Labels: result="empty" (no overrides found) | result="has_overrides" (overrides exist)
pub const FLAG_HASH_KEY_QUERY_RESULT: &str = "flags_hash_key_query_result_total";
pub const FLAG_HASH_KEY_REPLICA_CHECK: &str = "flags_hash_key_override_replica_check_total";

// Flag definitions rate limiting
pub const FLAG_DEFINITIONS_RATE_LIMITED_COUNTER: &str = "flags_flag_definitions_rate_limited_total";
pub const FLAG_DEFINITIONS_RATE_LIMIT_BYPASSED_COUNTER: &str =
    "flags_flag_definitions_rate_limit_bypassed_total";
pub const FLAG_DEFINITIONS_REQUESTS_COUNTER: &str = "flags_flag_definitions_requests_total";

// Remote config rate limiting
pub const REMOTE_CONFIG_RATE_LIMITED_COUNTER: &str = "flags_remote_config_rate_limited_total";
pub const REMOTE_CONFIG_RATE_LIMIT_BYPASSED_COUNTER: &str =
    "flags_remote_config_rate_limit_bypassed_total";
pub const REMOTE_CONFIG_REQUESTS_COUNTER: &str = "flags_remote_config_requests_total";

// Remote config auth method
// Labels: method (project_secret_api_key, secret_api_key, personal_api_key). The secret-vs-personal
// split decides redact-vs-decrypt, so the mix is worth watching during the phase 2/3 cutover.
pub const REMOTE_CONFIG_AUTH_COUNTER: &str = "flags_remote_config_auth_total";

// Remote config ETag metrics
// Labels: result (hit = 304, miss = 200 with stale etag, none = request sent no
// If-None-Match; the 200 still carries an etag since it is computed per request). Unlike flag
// definitions there is no redis_error case: no cache backs this endpoint, so the etag is
// content-derived per request and can always be computed.
pub const REMOTE_CONFIG_ETAG_COUNTER: &str = "flags_remote_config_etag_total";

// Flag definitions cache metrics
// Labels: source (redis, s3, fallback)
pub const FLAG_DEFINITIONS_CACHE_HIT_COUNTER: &str = "flags_flag_definitions_cache_hit_total";
// Labels: reason (cache_miss, s3_error, redis_error, json_parse_error, timeout)
pub const FLAG_DEFINITIONS_CACHE_MISS_COUNTER: &str = "flags_flag_definitions_cache_miss_total";

// Flag definitions ETag metrics
// Labels: result (hit = 304, miss = 200 with stale etag, none = 200 without etag,
// redis_missing = Redis answered and held no etag key,
// redis_error = the etag read failed or the stored value did not decode)
pub const FLAG_DEFINITIONS_ETAG_COUNTER: &str = "flags_flag_definitions_etag_total";

// Billing decision for a /flags/definitions 304. Labels: outcome (billable, not_billable,
// unknown = the payload was unreadable, slow, or not served from Redis, so the poll went
// unbilled). Compare with the etag `hit` counter to see how many 304s went unbilled.
pub const FLAG_DEFINITIONS_NOT_MODIFIED_BILLING_COUNTER: &str =
    "flags_flag_definitions_not_modified_billing_total";

// Per-pod resolved cluster for the /flags/definitions reader: 1 on the dedicated flags Redis,
// 0 on the shared one. Every emission carries a `reason` label, so
// `{reason="no_dedicated_client"}` separates a pod that wanted the dedicated cluster and could
// not get one from a pod nobody has flipped yet.
pub const FLAG_DEFINITIONS_READS_DEDICATED_REDIS_GAUGE: &str =
    "flags_flag_definitions_reads_dedicated_redis";

// Flag definitions self-heal: a request enqueued a rebuild request for a Celery worker to
// drain. Labels: result (ok = enqueued, error = redis zadd failed), trigger (cache_miss =
// nothing served the request, s3_hit = Redis lost the entry and S3 answered).
pub const FLAG_DEFINITIONS_REBUILD_REQUESTED_COUNTER: &str =
    "flags_flag_definitions_rebuild_requested_total";

// Flag definitions auth method
// Labels: method (secret_api_key, personal_api_key) — Rust only supports these two; Python also tracks oauth, jwt, session, other
pub const FLAG_DEFINITIONS_AUTH_COUNTER: &str = "flags_flag_definitions_auth_total";

// Transitional (#66179): background comparison of legacy-token auth against the
// backfilled PSAK mirror rows. Labels: result = match | missing | team_mismatch |
// revoked (leak-revoked, must stay without a row) | error | timeout | skipped (cap).
pub const LEGACY_TOKEN_MIRROR_SHADOW_COUNTER: &str = "flags_legacy_token_mirror_shadow_total";

// Counter for /flags requests classified as bot traffic. Fires regardless
// of whether the request was actually short-circuited, so dashboards can
// plot the "would-have-rejected" rate during the log-only rollout phase
// alongside the actual enforce-mode reject rate.
// Labels: `bot_category` (google, ai, seo, uptime, social, headless,
// crawler, other), `bot_source` (user_agent, ip), and
// `mode` (`log_only` — classified but not blocked; `enforced` — classified
// and short-circuited). No `team_id` because classification fires before
// token extraction; per-team analysis lives in the canonical log line via
// Loki.
pub const FLAG_BOT_DETECTED_COUNTER: &str = "flags_bot_detected_total";

// Requests refused by the team's request blocklist. Labeled only by `reason`
// (`host` or `ip`). The blocked host and IP are unbounded, so they go to the
// canonical log line, never to a label.
pub const FLAG_REQUEST_BLOCKED_COUNTER: &str = "flags_request_blocked_total";

// Request-level timeout (tower TimeoutLayer killed the request before completion)
pub const FLAG_REQUEST_TIMEOUT_COUNTER: &str = "flags_request_timeout_total";

// One count per /flags request, labeled `outcome` and `needs_persons`.
pub const FLAG_REQUEST_OUTCOME_COUNTER: &str = "flags_request_outcome_total";

// Timeout tracking and classification
pub const FLAG_ACQUIRE_TIMEOUT_COUNTER: &str = "flags_acquire_timeout_total";

// Error classification
pub const FLAG_DATABASE_ERROR_COUNTER: &str = "flags_database_error_total";

// Dependency graph build metrics
pub const FLAG_DEPENDENCY_GRAPH_BUILD_COUNTER: &str = "flags_dependency_graph_build_total";
pub const FLAG_DEPENDENCY_GRAPH_BUILD_TIME: &str = "flags_dependency_graph_build_ms";
pub const FLAG_MISSING_REQUESTED_FLAG_KEY: &str = "missing_requested_flag_key";

// Requests short-circuited because the team is over its billing quota.
pub const FLAG_QUOTA_LIMITED_COUNTER: &str = "flags_quota_limited_total";

// Conditions skipped during evaluation because required context was absent.
// Labels: reason (missing_device_id, missing_group_type)
pub const FLAG_CONDITION_SKIPPED_COUNTER: &str = "flags_condition_skipped_total";

// V2 ingress outcomes use fixed labels and never include configuration values.
pub const FLAG_V2_PARSE_COUNTER: &str = "flags_v2_config_parse_total";

// Incremented once per flag left out of a team's payload because its JSON does not
// deserialize: the `filters` column on the PG read, or the whole flag object on the
// hypercache read. A property filter with no `"type"` key is one such blob, because
// PropertyFilter requires prop_type, and serde fails the whole outer struct. This
// counts every dropped flag, which is a superset of the flags the two builders
// disagree about: Python keeps an active or referenced flag that this drops, but an
// inactive, unreferenced flag is dropped by both builders. The `operation` label on
// the companion tombstone counter says which read path dropped it. Team id and flag
// key are in the companion warn log, not in metric labels (cardinality).
pub const FLAG_MALFORMED_FILTER_COUNTER: &str = "flags_flag_malformed_filter_total";
// Incremented once per team read that left out at least one flag for the reason
// above. FLAG_MALFORMED_FILTER_COUNTER divided by this gives the mean flags
// dropped per affected read. It does not measure how many teams are affected,
// because neither counter carries a team label and one team read many times
// inflates this denominator. Read the warn log for team identity and breadth.
pub const FLAG_MALFORMED_FILTER_READ_COUNTER: &str = "flags_flag_malformed_filter_reads_total";

// Tombstone metric for tracking "impossible" failures that should never happen in production
// Different failure types are tracked via the "failure_type" label
pub const TOMBSTONE_COUNTER: &str = "posthog_tombstone_total";

// DB operations per request metric
// Tracks the count of DB operations per request, labeled by team_id and operation_type.
// This surfaces teams generating excessive DB load regardless of individual query latency.
// Labels: team_id, operation_type (person_query, cohort_query, group_query)
pub const FLAG_DB_OPERATIONS_PER_REQUEST: &str = "flags_db_operations_per_request";

// Rayon dispatcher metrics
// These track semaphore backpressure on the parallel evaluation path.

// Time spent waiting for a semaphore permit before entering the Rayon pool (histogram, ms).
// Near-zero means the pool is not saturated; high values mean batches are queueing.
pub const RAYON_DISPATCHER_SEMAPHORE_WAIT_TIME: &str = "flags_rayon_dispatcher_semaphore_wait_ms";

// Number of semaphore permits available at dispatch time (gauge).
// Consistently 0 means the Rayon pool is fully saturated.
pub const RAYON_DISPATCHER_AVAILABLE_PERMITS: &str = "flags_rayon_dispatcher_available_permits";

// Time spent executing work on the Rayon pool, excluding semaphore wait (histogram, ms).
// Compare with RAYON_DISPATCHER_SEMAPHORE_WAIT_TIME to understand whether tail latency
// comes from semaphore contention or actual computation time.
pub const RAYON_DISPATCHER_EXECUTION_TIME: &str = "flags_rayon_dispatcher_execution_ms";

// Counter of semaphore acquisitions that had to wait (no permits available).
// The ratio contended/total indicates how often the Rayon pool is at capacity.
pub const RAYON_DISPATCHER_CONTENDED_ACQUIRES: &str =
    "flags_rayon_dispatcher_contended_acquires_total";

// Total semaphore acquisitions (counter). Used as denominator for contention ratio.
pub const RAYON_DISPATCHER_TOTAL_ACQUIRES: &str = "flags_rayon_dispatcher_acquires_total";

// Number of batch tasks currently executing on the Rayon pool (gauge).
// With N semaphore permits, this should stay in [0, N]. Consistently at N
// means the pool is saturated and the semaphore is the bottleneck.
pub const RAYON_DISPATCHER_INFLIGHT_TASKS: &str = "flags_rayon_dispatcher_inflight_tasks";

// Counter of semaphore acquisitions that timed out (request failed fast with 504).
// Non-zero means the configured timeout is being hit and requests are being
// redistributed to other pods via ingress retry.
pub const RAYON_DISPATCHER_SEMAPHORE_TIMEOUTS: &str =
    "flags_rayon_dispatcher_semaphore_timeouts_total";

// Flag batch evaluation metrics
// These track the performance difference between sequential and parallel evaluation strategies.
// Used for A/B testing and tuning the PARALLEL_EVAL_THRESHOLD.

// Time spent evaluating a batch of flags (histogram)
// Labels: evaluation_type ("sequential" or "parallel")
pub const FLAG_BATCH_EVALUATION_TIME: &str = "flags_batch_evaluation_time_ms";

// Counter for evaluation batches by type
// Labels: evaluation_type ("sequential" or "parallel")
pub const FLAG_BATCH_EVALUATION_COUNTER: &str = "flags_batch_evaluation_total";

// Histogram of flag counts per batch evaluation
// Labels: evaluation_type ("sequential" or "parallel")
pub const FLAG_BATCH_SIZE: &str = "flags_batch_size";

// Tokio runtime metrics
// These track worker thread utilization to inform thread pool sizing decisions.
// Sampled periodically by TokioRuntimeMonitor (default: every 15s).

// Fraction of wall-clock time that workers spent executing tasks (gauge, 0.0–1.0).
// Computed as: sum(worker_busy_duration_delta) / (elapsed * num_workers).
// < 0.3 means workers are idle 70%+ of the time — room to reduce worker count.
// > 0.8 means approaching saturation.
pub const TOKIO_RUNTIME_BUSY_RATIO: &str = "flags_tokio_busy_ratio";

// Number of tasks currently alive (spawned but not yet completed) on the runtime (gauge).
pub const TOKIO_RUNTIME_ALIVE_TASKS: &str = "flags_tokio_alive_tasks";

// Tasks pending in the runtime's global injection queue (gauge).
// Sustained values > 0 indicate workers cannot drain tasks fast enough.
pub const TOKIO_RUNTIME_GLOBAL_QUEUE_DEPTH: &str = "flags_tokio_global_queue_depth";

// Number of configured Tokio worker threads (gauge, constant after startup).
pub const TOKIO_RUNTIME_NUM_WORKERS: &str = "flags_tokio_num_workers";

// Per-worker local queue depth (gauge). Labels: worker="0", worker="1", etc.
// High values indicate a specific worker is overloaded.
pub const TOKIO_WORKER_LOCAL_QUEUE_DEPTH: &str = "flags_tokio_worker_local_queue_depth";

// Per-worker poll count delta over the sampling interval (gauge). Labels: worker.
// Shows throughput per worker — large imbalances suggest uneven task distribution.
pub const TOKIO_WORKER_POLL_DELTA: &str = "flags_tokio_worker_poll_delta";

// Per-worker park count delta over the sampling interval (gauge). Labels: worker.
// A park event means the worker went idle. High park rates indicate light workloads.
pub const TOKIO_WORKER_PARK_DELTA: &str = "flags_tokio_worker_park_delta";

// Per-worker busy duration over the sampling interval in seconds (gauge). Labels: worker.
// Used to detect load imbalance across workers.
pub const TOKIO_WORKER_BUSY_DURATION_DELTA: &str = "flags_tokio_worker_busy_duration_delta_secs";

// Number of threads in the blocking thread pool (gauge).
pub const TOKIO_BLOCKING_THREADS: &str = "flags_tokio_blocking_threads";

// Idle threads in the blocking thread pool (gauge).
pub const TOKIO_IDLE_BLOCKING_THREADS: &str = "flags_tokio_idle_blocking_threads";

// Tasks waiting for a blocking thread (gauge).
// High values indicate spawn_blocking contention.
pub const TOKIO_BLOCKING_QUEUE_DEPTH: &str = "flags_tokio_blocking_queue_depth";

// Mean poll duration per worker in microseconds (gauge). Labels: worker.
// Long poll times can indicate blocking or CPU-heavy futures on the Tokio runtime.
pub const TOKIO_WORKER_MEAN_POLL_TIME_US: &str = "flags_tokio_worker_mean_poll_time_us";

// Times a worker's local queue overflowed to the global queue (gauge, delta). Labels: worker.
// Non-zero indicates a worker couldn't keep up and had to shed work.
pub const TOKIO_WORKER_OVERFLOW_DELTA: &str = "flags_tokio_worker_overflow_delta";

// Number of tasks stolen from other workers (gauge, delta). Labels: worker.
// Active stealing indicates work imbalance being corrected by the scheduler.
pub const TOKIO_WORKER_STEAL_DELTA: &str = "flags_tokio_worker_steal_delta";