Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
32.6 KB
Raw Blame
import client, { Counter, Gauge, Histogram, Registry } from 'prom-client'

/**
 * Prometheus metrics for workflow approvals feature.
 *
 * Metric naming follows Prometheus conventions:
 *   - `*_total` for monotonic counters
 *   - `*_seconds` for latency histograms
 *   - labels kept low-cardinality (status/decision/result/bucket_type/status_code)
 *
 * The default registry is re-used so node process metrics (event loop lag,
 * heap, gc pauses) are exposed on /metrics alongside app metrics.
 */
export const registry: Registry = client.register

// Collect default Node process metrics once. Guard against double-registration
// when tests import this module more than once via module-graph re-entry.
const DEFAULT_METRICS_COLLECTED = Symbol.for('clerum.controlApi.defaultMetricsCollected')
type GlobalWithFlag = typeof globalThis & { [DEFAULT_METRICS_COLLECTED]?: boolean }
const globalWithFlag = globalThis as GlobalWithFlag
if (!globalWithFlag[DEFAULT_METRICS_COLLECTED]) {
  client.collectDefaultMetrics({ register: registry })
  globalWithFlag[DEFAULT_METRICS_COLLECTED] = true
}

/**
 * Helper: get-or-create a metric so hot-reload / repeated imports in tests
 * don't throw "A metric with the name X has already been registered".
 */
function getOrCreateCounter<Labels extends string>(opts: {
  name: string
  help: string
  labelNames: Labels[]
}): Counter<Labels> {
  const existing = registry.getSingleMetric(opts.name)
  if (existing) return existing as Counter<Labels>
  return new Counter<Labels>({ ...opts, registers: [registry] })
}

function getOrCreateHistogram<Labels extends string>(opts: {
  name: string
  help: string
  labelNames: Labels[]
  buckets?: number[]
}): Histogram<Labels> {
  const existing = registry.getSingleMetric(opts.name)
  if (existing) return existing as Histogram<Labels>
  return new Histogram<Labels>({ ...opts, registers: [registry] })
}

function getOrCreateGauge<Labels extends string>(opts: {
  name: string
  help: string
  labelNames: Labels[]
}): Gauge<Labels> {
  const existing = registry.getSingleMetric(opts.name)
  if (existing) return existing as Gauge<Labels>
  return new Gauge<Labels>({ ...opts, registers: [registry] })
}

// ─── Approval lifecycle counters ──────────────────────────────────────────
export const approvalsCreatedTotal = getOrCreateCounter({
  name: 'user_approval_requests_created_total',
  help: 'Count of workflow approval requests created, labelled by resulting status.',
  labelNames: ['status'] as const as Array<'status'>,
})

export const approvalsDecidedTotal = getOrCreateCounter({
  name: 'user_approval_requests_decided_total',
  help: 'Count of workflow approval requests decided (approve|deny).',
  labelNames: ['decision'] as const as Array<'decision'>,
})

export const approvalsExpiredTotal = getOrCreateCounter({
  name: 'user_approval_requests_expired_total',
  help: 'Count of workflow approval requests that transitioned to expired.',
  labelNames: [] as string[],
})

export const approvalsCancelledTotal = getOrCreateCounter({
  name: 'user_approval_requests_cancelled_total',
  help: 'Count of workflow approval requests that transitioned to cancelled.',
  labelNames: [] as string[],
})

export const approvalsDurationSeconds = getOrCreateHistogram({
  name: 'user_approval_requests_duration_seconds',
  help: 'Duration from approval creation to terminal decision (approve/deny).',
  labelNames: ['decision'] as const as Array<'decision'>,
  buckets: [1, 5, 15, 30, 60, 300, 900, 3600, 21600, 86400, 604800],
})

// ─── Notification stream/outbox metrics ──────────────────────────────────
export const notificationEventsEnqueuedTotal = getOrCreateCounter({
  name: 'notification_events_enqueued_total',
  help: 'Count of notification events enqueued in the durable outbox.',
  labelNames: ['event_type'] as const as Array<'event_type'>,
})

export const notificationOutboxEnqueueFailuresTotal = getOrCreateCounter({
  name: 'notification_outbox_enqueue_failures_total',
  help: 'Count of notification outbox enqueue failures.',
  labelNames: ['event_type'] as const as Array<'event_type'>,
})

// Figure D multi-bot: a user-bound approval delivery was skipped because no bot
// credential could be resolved for its CommunicationChannel. Distinct from a
// provider send failure — surfaces a suppression vector (e.g. a missing/deleted
// channel Secret) that would otherwise be invisible. Alert on rate > 0.
export const workflowApprovalDeliverySkippedNoBotTotal = getOrCreateCounter({
  name: 'workflow_approval_delivery_skipped_no_bot_total',
  help: 'Count of user-bound approval deliveries skipped because no bot credential resolved for the channel.',
  labelNames: ['medium'] as const as Array<'medium'>,
})

export const notificationStreamConnectionsActive = getOrCreateGauge({
  name: 'notification_stream_connections_active',
  help: 'Active Desktop notification stream connections.',
  labelNames: [] as string[],
})

export const notificationStreamEventsSentTotal = getOrCreateCounter({
  name: 'notification_stream_events_sent_total',
  help: 'Count of events written to Desktop notification streams.',
  labelNames: ['event_type'] as const as Array<'event_type'>,
})

export const notificationStreamEventsFilteredTotal = getOrCreateCounter({
  name: 'notification_stream_events_filtered_total',
  help: 'Count of notification stream events filtered before delivery.',
  labelNames: ['reason'] as const as Array<'reason'>,
})

export const notificationStreamDisconnectsTotal = getOrCreateCounter({
  name: 'notification_stream_disconnects_total',
  help: 'Count of Desktop notification stream disconnects.',
  labelNames: ['reason'] as const as Array<'reason'>,
})

export const notificationStreamSnapshotSize = getOrCreateHistogram({
  name: 'notification_stream_snapshot_size',
  help: 'Number of active approval notifications in each stream snapshot.',
  labelNames: [] as string[],
  buckets: [0, 1, 2, 5, 10, 20, 50],
})

// ─── Durable entity-change feed/stream metrics ──────────────────────────
export const entityChangeDispatchBatchesTotal = getOrCreateCounter({
  name: 'entity_change_dispatch_batches_total',
  help: 'Entity-change outbox dispatcher ticks by result.',
  labelNames: ['result'] as const as Array<'result'>,
})

export const entityChangeStreamConnectionsActive = getOrCreateGauge({
  name: 'entity_change_stream_connections_active',
  help: 'Active authenticated entity-change stream connections.',
  labelNames: ['principal_kind'] as const as Array<'principal_kind'>,
})

export const entityChangeStreamFramesSentTotal = getOrCreateCounter({
  name: 'entity_change_stream_frames_sent_total',
  help: 'Entity-change stream frames successfully handed to HTTP responses.',
  labelNames: ['principal_kind', 'frame_type'] as const as Array<'principal_kind' | 'frame_type'>,
})

export const entityChangeStreamDisconnectsTotal = getOrCreateCounter({
  name: 'entity_change_stream_disconnects_total',
  help: 'Entity-change stream closures by bounded reason.',
  labelNames: ['principal_kind', 'reason'] as const as Array<'principal_kind' | 'reason'>,
})

export const entityChangeStreamResyncRequiredTotal = getOrCreateCounter({
  name: 'entity_change_stream_resync_required_total',
  help: 'Entity-change streams that required a full authoritative resync.',
  labelNames: ['principal_kind'] as const as Array<'principal_kind'>,
})

// ─── Auth token counters ──────────────────────────────────────────────────
export const mcpHostJwtIssueTotal = getOrCreateCounter({
  name: 'workflow_auth_issue_total',
  help: 'Count of workflow access/refresh token issuance events.',
  labelNames: ['kind'] as const as Array<'kind'>, // access | refresh
})

export const oauthBrokerJwtIssueTotal = getOrCreateCounter({
  name: 'oauth_broker_jwt_issue_total',
  help: 'Count of recipe OAuth broker token issuance events labelled by result.',
  labelNames: ['result'] as const as Array<'result'>, // issued | recipe_not_found | not_background_access
})

// Refresh endpoint outcomes (also the label of the duration histogram below).
// The 401 body stays opaque; the label is emitted server-side in a `finally`.
// Label values (binding outcomes share their names with the re-issue counter):
//   success                  — token rotated, fresh token set minted
//   failed                   — missing bearer; signature/claims/expiry rejected;
//                              replay of a jti already consumed at verify time;
//                              unhandled error
//   host_unavailable         — Host lookup failed transiently (503, jti not consumed)
//   host_mismatch            — HCC chain's Host is gone or was recreated (new uid)
//   lineage_without_audience — hosts-namespace standalone chain without HCC audience
//   race_lost                — jti consumed concurrently between verify and consume
//                              (not the re-issue `revoked`, which also covers replay)
// Rate-limited requests never reach the handler and are not counted here.
export const mcpHostJwtRefreshTotal = getOrCreateCounter({
  name: 'workflow_auth_refresh_total',
  help: 'Count of workflow refresh attempts labelled by result.',
  labelNames: ['result'] as const as Array<'result'>,
})

export const mcpHostJwtRefreshDurationSeconds = getOrCreateHistogram({
  name: 'workflow_auth_refresh_duration_seconds',
  help: 'Duration of the workflow refresh endpoint.',
  labelNames: ['result'] as const as Array<'result'>,
  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
})

// Task #20: Re-issue recovery endpoint counter. Labels stay low-cardinality
// and do NOT leak which specific validation failed in the response body —
// the label is emitted server-side in a `finally` block regardless of
// the opaque 401 surfaced to the caller.
// Label values:
//   ok             — validation passed, fresh token pair minted
//   invalid_sig    — JWT signature / iss / aud / scope / format rejected
//   invalid_claims — JWT verified but failed refresh-token claim validation
//   expired_beyond_reissue_grace — signed refresh token is too old to reissue
//   revoked        — jti already present in workflow_revoked_refresh_jtis at verify
//                    time, or consumed concurrently between verify and consume
//   mismatch       — recipe_name body did not match JWT sub's recipe component
//   bad_request    — missing/malformed body or bearer token
//   host_unavailable — Host lookup failed transiently (503, jti not consumed)
//   host_mismatch  — HCC chain's Host is gone or was recreated (new uid)
//   lineage_without_audience — hosts-namespace standalone chain without HCC audience
// Rate-limited requests never reach the handler and are not counted here.
export const mcpHostJwtReissueTotal = getOrCreateCounter({
  name: 'workflow_auth_reissue_requests_total',
  help: 'Count of workflow auth re-issue attempts labelled by result.',
  labelNames: ['result'] as const as Array<'result'>,
})

// OAuth mcp-server broker refused a caller on its Host binding. reason:
// lineage | unauthenticated | host_not_bound | server_not_found |
// context_ref_mismatch | no_grant | unavailable. route: user_token |
// grants_exists. The mcp-server and Host go to the log, never a label.
export const mcpOauthBrokerDeniedTotal = getOrCreateCounter({
  name: 'mcp_oauth_broker_denied_total',
  help: 'Count of OAuth broker requests denied by the Host binding, by reason and route.',
  labelNames: ['reason', 'route'] as const as Array<'reason' | 'route'>,
})

// MCP server uninstall: a cleanup step failed. Every stage except
// oauth_grants_post_delete runs before the CR delete and leaves the CR in place for a
// retry (the request answers "repair required"); a sustained rate means uninstalls
// that nobody repaired. oauth_grants_post_delete is the only residual after a 200 —
// rows it leaves are inert to the uid-fenced readers. Stages: contexts |
// dynamic_client | oauth_grants | secrets | oauth_client_secret | mcp_server |
// oauth_grants_post_delete.
export const mcpServerUninstallTeardownFailuresTotal = getOrCreateCounter({
  name: 'mcp_server_uninstall_teardown_failures_total',
  help: 'Count of MCP server uninstall cleanup steps that failed, by stage.',
  labelNames: ['stage'] as const as Array<'stage'>,
})

// ─── HTTP counters / histograms (scoped to workflow-approvals endpoints) ──
export const mcpHostHttpTotal = getOrCreateCounter({
  name: 'mcp_host_http_total',
  help: 'Count of HTTP requests to workflow-approvals endpoints.',
  labelNames: ['route', 'method', 'status_code'] as const as Array<
    'route' | 'method' | 'status_code'
  >,
})

export const mcpHostHttpDurationSeconds = getOrCreateHistogram({
  name: 'mcp_host_http_duration_seconds',
  help: 'HTTP latency histogram for workflow-approvals endpoints.',
  labelNames: ['route', 'method', 'status_code'] as const as Array<
    'route' | 'method' | 'status_code'
  >,
  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
})

// ─── Expiry cron metrics ──────────────────────────────────────────────────
export const approvalsExpiryRunsTotal = getOrCreateCounter({
  name: 'user_approval_requests_expiry_runs_total',
  help: 'Count of expiry cron sweeps.',
  labelNames: ['result'] as const as Array<'result'>, // ok | error
})

export const approvalsExpiredByCronTotal = getOrCreateCounter({
  name: 'user_approval_requests_expired_by_cron_total',
  help: 'Total rows marked expired by the expiry cron.',
  labelNames: [] as string[],
})

export const approvalsExpiryDurationSeconds = getOrCreateHistogram({
  name: 'user_approval_requests_expiry_duration_seconds',
  help: 'Duration of each expiry cron sweep.',
  labelNames: [] as string[],
  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],
})

// ─── Archival cron metrics ────────────────────────────────────────────────
export const approvalsArchiveRunsTotal = getOrCreateCounter({
  name: 'user_approval_requests_archive_runs_total',
  help: 'Count of approval archive cron sweeps.',
  labelNames: ['result'] as const as Array<'result'>, // ok | error
})

export const approvalsArchivedTotal = getOrCreateCounter({
  name: 'user_approval_requests_archived_total',
  help: 'Total rows archived (moved from workflow_approval_requests to archive).',
  labelNames: [] as string[],
})

export const approvalsArchiveDurationSeconds = getOrCreateHistogram({
  name: 'user_approval_requests_archive_duration_seconds',
  help: 'Duration of each archive cron sweep.',
  labelNames: [] as string[],
  buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
})

// ─── Rate limiter metrics ─────────────────────────────────────────────────
export const rateLimitHitsTotal = getOrCreateCounter({
  name: 'rate_limit_hits_total',
  help: 'Rate-limiter hits, labelled by bucket_type and result (allowed|denied|unavailable|fallback_allowed|fallback_denied).',
  labelNames: ['bucket_type', 'result'] as const as Array<'bucket_type' | 'result'>,
})

// Every limiter query that failed, counted even when its warn line is
// throttled. No label: the bucket key is unbounded (see below).
export const rateLimitBackendErrorsTotal = getOrCreateCounter({
  name: 'rate_limit_backend_errors_total',
  help: 'Rate-limiter queries that failed, so the request was not counted.',
  labelNames: [] as string[],
})

export const gfsUploadAdmissionRequestsTotal = getOrCreateCounter({
  name: 'gfs_upload_admission_requests_total',
  help: 'Upload-v2 edge admission decisions by bounded limit family and result.',
  labelNames: ['limit', 'result'] as const as Array<'limit' | 'result'>,
})

export const gfsUploadAdmissionBytesTotal = getOrCreateCounter({
  name: 'gfs_upload_admission_bytes_total',
  help: 'Declared upload-v2 part bytes admitted or rejected before GFSC forwarding.',
  labelNames: ['result'] as const as Array<'result'>,
})

export const gfsUploadAdmissionActiveRequests = getOrCreateGauge({
  name: 'gfs_upload_admission_active_requests',
  help: 'Current upload-v2 requests holding replica-safe PostgreSQL admission slots.',
  labelNames: [] as string[],
})

// ─── External GFS rate-boundary metrics ────────────────────────────────────
//
// The rate-limit bucket key is deliberately *not* a Prometheus label: even a
// one-way digest is unbounded cardinality and would turn the metrics endpoint
// into a durable identity side channel. The middleware emits the digest in its
// structured event instead, while these metrics retain the bounded route and
// operation dimensions needed for alerting.
export const externalGfsRateLimitRequestsTotal = getOrCreateCounter({
  name: 'external_gfs_rate_limit_requests_total',
  help: 'External GFS rate-boundary decisions by operation class, route, phase, and outcome.',
  labelNames: [
    'operation_class',
    'route',
    'outcome',
    'phase',
    'authority_resolution_avoided',
  ] as const as Array<
    'operation_class' | 'route' | 'outcome' | 'phase' | 'authority_resolution_avoided'
  >,
})

export const externalGfsRateLimitDurationSeconds = getOrCreateHistogram({
  name: 'external_gfs_rate_limit_duration_seconds',
  help: 'External GFS rate-boundary decision latency by bounded operation dimensions.',
  labelNames: [
    'operation_class',
    'route',
    'outcome',
    'phase',
    'authority_resolution_avoided',
  ] as const as Array<
    'operation_class' | 'route' | 'outcome' | 'phase' | 'authority_resolution_avoided'
  >,
  buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
})

// ─── Workflow-run archival cron metrics (DB-first, replaces CRD reaper) ──
export const workflowRunsArchiveRunsTotal = getOrCreateCounter({
  name: 'workflow_runs_archive_runs_total',
  help: 'Count of workflow-run archive cron sweeps.',
  labelNames: ['result'] as const as Array<'result'>, // ok | error | skipped_lock
})

export const workflowRunsArchivedTotal = getOrCreateCounter({
  name: 'workflow_runs_archived_total',
  help: 'Total terminal workflow runs archived (moved from workflow_runs to workflow_runs_audit).',
  labelNames: [] as string[],
})

export const workflowRunsChildDeleteTotal = getOrCreateCounter({
  name: 'workflow_runs_child_delete_total',
  help: 'Count of child WorkflowRecipe deletes triggered by archive cron, labelled by result.',
  labelNames: ['result'] as const as Array<'result'>, // ok | not_found | error
})

export const workflowRunsArchiveDurationSeconds = getOrCreateHistogram({
  name: 'workflow_runs_archive_duration_seconds',
  help: 'Duration of each workflow-run archive cron sweep.',
  labelNames: [] as string[],
  buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
})

// ─── Fase 5 — schedule-worker metrics (DB-backed scheduler) ──────────────
// Replaces K8s CronJob scheduler. Each sweep acquires a cross-replica advisory
// lock on hashtext('wrc-schedule-worker-v1'); matured rows in workflow_schedules
// translate into workflow_runs inserts via workflowRunService.createRun().
export const workflowScheduleWorkerRunsTotal = getOrCreateCounter({
  name: 'wrc_schedule_worker_runs_total',
  help: 'Count of schedule-worker sweeps, labelled by result.',
  labelNames: ['result'] as const as Array<'result'>, // ok | error | skipped_lock
})

export const workflowScheduleWorkerFiresTotal = getOrCreateCounter({
  name: 'wrc_schedule_worker_fires_total',
  help: 'Count of individual schedule fires translated into workflow_runs rows.',
  labelNames: ['result'] as const as Array<'result'>, // ok | error | actor_not_allowed | concurrency_forbidden
})

export const workflowScheduleWorkerConcurrencyForbiddenSustainedTotal = getOrCreateCounter({
  name: 'wrc_schedule_worker_concurrency_forbidden_sustained_total',
  help: 'Schedules that reached the sustained consecutive-skip threshold under concurrencyPolicy Forbid (once per streak).',
  labelNames: [] as string[],
})

export const workflowScheduleWorkerDurationSeconds = getOrCreateHistogram({
  name: 'wrc_schedule_worker_duration_seconds',
  help: 'Duration of each schedule-worker sweep.',
  labelNames: [] as string[],
  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
})

// ─── Plugin Workload SDK (plan §5.3) ────────────────────────────────────

export const pluginWorkloadSdkAuthDecisionsTotal = getOrCreateCounter({
  name: 'clerum_plugin_workload_sdk_auth_decisions_total',
  help: 'Authorization decisions per recipe and method family (authorized | structured error code).',
  labelNames: ['recipe', 'method', 'decision'] as const as Array<'recipe' | 'method' | 'decision'>,
})

export const pluginWorkloadSdkNotificationAuthDurationSeconds = getOrCreateHistogram({
  name: 'clerum_plugin_workload_sdk_notification_auth_duration_seconds',
  help: 'clientNotifications authorization latency (SLO p99 < 500ms).',
  labelNames: ['recipe'] as const as Array<'recipe'>,
  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5],
})

export const pluginWorkloadSdkMaintenanceRunsTotal = getOrCreateCounter({
  name: 'clerum_plugin_workload_sdk_maintenance_runs_total',
  help: 'Plugin Workload SDK maintenance sweeps (stale-invocations + idempotency pruning).',
  labelNames: ['result'] as const as Array<'result'>, // ok | error
})

// ─── OAuth proactive-refresh cron metrics (mini-spec L) ──────────────────
export const oauthProactiveRefreshRunsTotal = getOrCreateCounter({
  name: 'oauth_proactive_refresh_runs_total',
  help: 'Count of OAuth proactive-refresh cron sweeps.',
  labelNames: ['result'] as const as Array<'result'>, // ok | error
})

export const oauthProactiveRefreshGrantsTotal = getOrCreateCounter({
  name: 'oauth_proactive_refresh_grants_total',
  help: 'Count of remote grants processed by the proactive-refresh cron, labelled by outcome.',
  // ok | transient | client_invalid | no_grant | skipped | error
  labelNames: ['outcome'] as const as Array<'outcome'>,
})

export const oauthDcrSecretStatusTotal = getOrCreateCounter({
  name: 'oauth_dcr_secret_status_total',
  help: 'Count of DCR confidential clients observed near/at client_secret expiry.',
  labelNames: ['state'] as const as Array<'state'>, // expiring | expired
})

export const oauthProactiveRefreshDurationSeconds = getOrCreateHistogram({
  name: 'oauth_proactive_refresh_duration_seconds',
  help: 'Duration of each OAuth proactive-refresh cron sweep.',
  labelNames: [] as string[],
  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
})

// ─── Governed tracing foundation ──────────────────────────────────────────
// Keep these dimensions bounded. Run, event, request, session, human, agent,
// team, recipe, Host, workload, and correlation identifiers are trace data,
// never Prometheus labels.
export const governedTraceAcceptedTotal = getOrCreateCounter({
  name: 'governed_trace_accepted_total',
  help: 'Count of governed trace events accepted for persistence.',
  labelNames: ['family', 'source', 'type'] as const as Array<'family' | 'source' | 'type'>,
})

export const governedTraceReplayedTotal = getOrCreateCounter({
  name: 'governed_trace_replayed_total',
  help: 'Count of governed trace events accepted as idempotent replays.',
  labelNames: ['family', 'source', 'type'] as const as Array<'family' | 'source' | 'type'>,
})

export const governedTraceRejectedTotal = getOrCreateCounter({
  name: 'governed_trace_rejected_total',
  help: 'Count of governed trace events rejected before persistence.',
  labelNames: ['family', 'source', 'type'] as const as Array<'family' | 'source' | 'type'>,
})

export const governedTraceConflictingTotal = getOrCreateCounter({
  name: 'governed_trace_conflicting_total',
  help: 'Count of governed trace events rejected because an idempotency key conflicted.',
  labelNames: ['family', 'source', 'type'] as const as Array<'family' | 'source' | 'type'>,
})

/**
 * A Host whose administrative-intent annotation names a generation the object
 * has already passed, so no outcome for it can ever be attributed (#329). The
 * authoritative signal for drift being OBSERVED: HCC's own reporter counter
 * labels this `result="rejected"`, the same label it uses for malformed input,
 * and cannot tell the two apart.
 *
 * Read it as an arrival rate, not as a population. Once HCC classifies the
 * refusal as terminal it stops resubmitting, so a Host that stays drifted stops
 * incrementing this counter — a standing drift is invisible here by design, and
 * `increase(...[1h])` going to zero means the loop was cut, not that the Host
 * was repaired. Nothing in this service currently gauges how many Hosts are
 * drifted right now.
 *
 * `namespace` is the only label. Host names are unbounded, so putting one in a
 * label would make the series count grow with the cluster.
 */
export const governedTraceAdministrativeIntentDriftTotal = getOrCreateCounter({
  name: 'governed_trace_administrative_intent_drift_total',
  help: 'Count of administrative tracing events refused because the intent annotation names a superseded generation.',
  labelNames: ['namespace'] as const as Array<'namespace'>,
})

/**
 * The write path failed to correct a predicted generation it knows is wrong
 * (#329 layer 3). Every increment means an administrative intent annotation was
 * left pinned to a value the object never reached, which the resolver will
 * later refuse — so this counter leads the drift counter above.
 *
 * It exists because the repair failures are otherwise invisible: the reconcile
 * is documented never to throw, so each one is a single `warn` line with no
 * metric. A revoked `patch` verb or an admission webhook that rejects
 * annotation writes would degrade the administrative audit trail silently,
 * with the drift counter naming the symptom and nothing naming the cause.
 *
 * `reason` is a closed set of four fixed strings, matching the `event` field of
 * the log line emitted beside it. No Host identity: names are unbounded, and
 * the namespace is already carried by the drift counter.
 */
export const governedTraceAdministrativeIntentReconcileFailedTotal = getOrCreateCounter({
  name: 'governed_trace_administrative_intent_reconcile_failed_total',
  help: 'Count of administrative intent generation corrections that could not be applied, labelled by reason.',
  labelNames: ['reason'] as const as Array<'reason'>,
})

export const governedTraceIngestDurationSeconds = getOrCreateHistogram({
  name: 'governed_trace_ingest_duration_seconds',
  help: 'Duration of governed trace ingestion and stream registration.',
  labelNames: ['family', 'source'] as const as Array<'family' | 'source'>,
  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
})

export const governedTraceBatchSize = getOrCreateHistogram({
  name: 'governed_trace_batch_size',
  help: 'Number of events submitted in each governed tracing ingestion batch.',
  labelNames: ['family', 'source'] as const as Array<'family' | 'source'>,
  buckets: [1, 2, 5, 10, 25, 50, 100],
})

export const governedTraceQueryCount = getOrCreateHistogram({
  name: 'governed_trace_query_count',
  help: 'Database query round trips used by each governed family append transaction.',
  labelNames: ['family', 'source'] as const as Array<'family' | 'source'>,
  buckets: [1, 2, 3, 4, 5, 8, 13],
})

export const governedTraceReadDurationSeconds = getOrCreateHistogram({
  name: 'governed_trace_read_duration_seconds',
  help: 'Duration of governed trace read operations.',
  labelNames: ['family'] as const as Array<'family'>,
  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
})

export const governedTracePoolAcquisitionDurationSeconds = getOrCreateHistogram({
  name: 'governed_trace_pool_acquisition_duration_seconds',
  help: 'Time spent acquiring a governed tracing database pool connection.',
  labelNames: ['pool'] as const as Array<'pool'>,
  buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2],
})

export const governedTracePoolRejectionsTotal = getOrCreateCounter({
  name: 'governed_trace_pool_rejections_total',
  help: 'Count of governed tracing work rejected because its pool could not serve it.',
  labelNames: ['pool'] as const as Array<'pool'>,
})

export const governedTracePoolConnections = getOrCreateGauge({
  name: 'governed_trace_pool_connections',
  help: 'Current active, idle, and waiting connection counts for governed tracing pools.',
  labelNames: ['pool', 'state'] as const as Array<'pool' | 'state'>,
})

export const governedTracePoolStatementTimeoutsTotal = getOrCreateCounter({
  name: 'governed_trace_pool_statement_timeouts_total',
  help: 'Governed tracing statements cancelled by the configured pool timeout.',
  labelNames: ['pool'] as const as Array<'pool'>,
})

export const GOVERNED_TRACE_OPERATIONAL_ERROR_REASONS = [
  'unsupported_content_type',
  'invalid_json',
  'body_too_large',
  'batch_too_large',
  'capacity_exhausted',
  'event_rejected',
  'idempotency_conflict',
  'submission_failed',
  'pool_rejected',
  'statement_timeout',
  'attribution_binding_unavailable',
  'attribution_binding_conflict',
  'prompt_history_disabled',
  'prompt_history_key_unavailable',
  'prompt_history_rejected',
] as const

export type GovernedTraceOperationalErrorReason =
  (typeof GOVERNED_TRACE_OPERATIONAL_ERROR_REASONS)[number]

export const GOVERNED_TRACE_OPERATIONAL_SCOPES = [
  'agent_run',
  'administrative',
  'infrastructure_telemetry',
  'read',
  'pool',
] as const

export type GovernedTraceOperationalScope = (typeof GOVERNED_TRACE_OPERATIONAL_SCOPES)[number]

export const governedTraceAdmissionRequestsTotal = getOrCreateCounter({
  name: 'governed_trace_admission_requests_total',
  help: 'Count of governed tracing requests admitted or rejected at the request boundary.',
  labelNames: ['family', 'result', 'reason'] as const as Array<'family' | 'result' | 'reason'>,
})

export const governedTraceRequestBodyBytes = getOrCreateHistogram({
  name: 'governed_trace_request_body_bytes',
  help: 'Raw byte size of governed tracing JSON request bodies accepted by the parser.',
  labelNames: ['family'] as const as Array<'family'>,
  buckets: [1_024, 4_096, 16_384, 65_536, 131_072, 262_144, 393_216, 524_288],
})

export const governedTraceInFlightRequests = getOrCreateGauge({
  name: 'governed_trace_in_flight_requests',
  help: 'Current governed tracing requests inside the control-api admission limiter.',
  labelNames: [] as string[],
})

export const governedTraceOperationalErrorsTotal = getOrCreateCounter({
  name: 'governed_trace_operational_errors_total',
  help: 'Count of bounded governed tracing operational error occurrences.',
  labelNames: ['scope', 'reason'] as const as Array<'scope' | 'reason'>,
})

export const governedTraceLastErrorTimestampSeconds = getOrCreateGauge({
  name: 'governed_trace_last_error_timestamp_seconds',
  help: 'Unix timestamp of the most recent bounded governed tracing operational error.',
  labelNames: ['scope', 'reason'] as const as Array<'scope' | 'reason'>,
})

export function recordGovernedTraceOperationalError(
  scope: GovernedTraceOperationalScope,
  reason: GovernedTraceOperationalErrorReason,
  timestampSeconds = Date.now() / 1_000
): void {
  governedTraceOperationalErrorsTotal.inc({ scope, reason })
  governedTraceLastErrorTimestampSeconds.set({ scope, reason }, timestampSeconds)
}
// ─── LLM allowlist ConfigMap materialization (spec §3-R3.4 / V7) ─────────
// Increments when the `clerum-llm-allowed-models` ConfigMap write fails after
// its short retry — LLM Models CRUD / Codex grant mutations (phase=mutation)
// or the non-fatal boot reconcile (phase=boot). A rate > 0 means Postgres and
// the delivered ConfigMap have drifted.
export const llmAllowlistConfigMapWriteFailuresTotal = getOrCreateCounter({
  name: 'clerum_llm_allowlist_configmap_write_failures_total',
  help: 'Count of failed clerum-llm-allowed-models ConfigMap writes, labelled by phase.',
  labelNames: ['phase'] as const as Array<'phase'>, // mutation | boot | sync
})

export type PasswordAdmissionDenialReason =
  | 'source_rate'
  | 'identifier_attempts'
  | 'identifier_cooldown'
  | 'global_pace'
  | 'verification_busy'
  | 'authority_failure'

export const passwordAdmissionDenialsTotal = getOrCreateCounter({
  name: 'password_admission_denials_total',
  help: 'Password admission denials by bounded internal reason; never labelled by identity.',
  labelNames: ['reason'] as const as Array<'reason'>,
})