Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

44.8 KB
Raw Blame
import { requestContext } from "@fastify/request-context";
import opentelemetry, {
  type Attributes,
  type Counter,
  type Histogram,
  type Meter,
  type MetricOptions
} from "@opentelemetry/api";
import type { Knex } from "knex";

import { classifyError } from "@app/lib/errors/classify";
import { RequestContextKey } from "@app/lib/request-context/request-context-keys";
import { getAgentPoolStats } from "@app/lib/validator/agent-pool";

import { getConfig } from "../config/env";

// IMPORTANT: this module is imported (transitively, via instrumentation.ts) BEFORE setupTelemetry()
// installs the global MeterProvider. A meter or instrument acquired at module-load time therefore binds
// to the OpenTelemetry API's no-op provider permanently and silently records nothing. To avoid that we
// resolve the real meter lazily (memoized, on first .add()/.record() — i.e. at request time, after init).
// Observable gauges can't defer to first use (the SDK pulls them), so they're registered at boot in
// registerInfrastructureMetrics(), which runs from main.ts after setupTelemetry().
const meterCache = new Map<string, Meter>();
const resolveMeter = (meterName: string): Meter => {
  let meter = meterCache.get(meterName);
  if (!meter) {
    meter = opentelemetry.metrics.getMeter(meterName);
    meterCache.set(meterName, meter);
  }
  return meter;
};

export const resolveCoreMeter = (): Meter => resolveMeter("InfisicalCore");

type LazyMeter = {
  createCounter: (name: string, options?: MetricOptions) => Counter;
  createHistogram: (name: string, options?: MetricOptions) => Histogram;
};

// Recording is fire-and-forget: a measurement is an observation of the work, never a step in it, so
// nothing here may throw into a call site. The SDK can raise on a malformed instrument name or a
// broken exporter, and getConfig() is empty until initEnvConfig() runs, which is reachable from
// anything recording during boot. Swallowing is silent on purpose — the logger is itself initialised
// from config, so reporting the failure here risks throwing a second time from the handler.
const safely = (record: () => void) => {
  try {
    record();
  } catch {
    // metrics must never break the path they observe
  }
};

// Returns instrument wrappers whose underlying instrument is created on first use (after init), so it
// binds to the real MeterProvider. Call sites keep using .add()/.record() exactly as before.
const lazyMeter = (meterName: string): LazyMeter => ({
  createCounter: (name, options) => {
    let instrument: Counter | undefined;
    return {
      add: (value: number, attributes?: Attributes) => {
        safely(() => {
          if (!instrument) instrument = resolveMeter(meterName).createCounter(name, options);
          instrument.add(value, attributes);
        });
      }
    } as Counter;
  },
  createHistogram: (name, options) => {
    let instrument: Histogram | undefined;
    return {
      record: (value: number, attributes?: Attributes) => {
        safely(() => {
          if (!instrument) instrument = resolveMeter(meterName).createHistogram(name, options);
          instrument.record(value, attributes);
        });
      }
    } as Histogram;
  }
});

// Exported so a call site can skip work it would only do to produce a measurement (an extra query,
// a serialization, a size computation). Recording is already gated internally, so this is never
// needed to make a record*Metric call safe.
export const isTelemetryEnabled = () => Boolean(getConfig()?.OTEL_TELEMETRY_COLLECTION_ENABLED);

export const shouldRecordHighCardinalityMetrics = () =>
  isTelemetryEnabled() && !getConfig()?.OTEL_DROP_HIGH_CARDINALITY_METERS;

export const highCardinalityMeter = (meterName: string): LazyMeter => {
  const meter = lazyMeter(meterName);

  return {
    createCounter: (name, options) => {
      const counter = meter.createCounter(name, options);
      return {
        add: (value: number, attributes?: Attributes) => {
          if (!shouldRecordHighCardinalityMetrics()) return;
          counter.add(value, attributes);
        }
      } as Counter;
    },
    createHistogram: (name, options) => {
      const histogram = meter.createHistogram(name, options);
      return {
        record: (value: number, attributes?: Attributes) => {
          if (!shouldRecordHighCardinalityMetrics()) return;
          histogram.record(value, attributes);
        }
      } as Histogram;
    }
  };
};

// High-cardinality, per-actor meter, kept on by default for self-hosted (where per-actor visibility is
// useful and cardinality is bounded by a single org); dropped in multi-tenant/cloud via
// OTEL_DROP_HIGH_CARDINALITY_METERS. The pre-existing metrics below ship with per-actor labels documented in the docs.
const infisicalMeter = highCardinalityMeter("Infisical");

// The MeterProvider applies a strict attribute allowlist (View in
// instrumentation.ts) to anything emitted here, dropping high-cardinality labels at the SDK level.
const infisicalCoreMeter = lazyMeter("InfisicalCore");

export enum AuthAttemptAuthMethod {
  EMAIL = "email",
  SAML = "saml",
  OIDC = "oidc",
  GOOGLE = "google",
  GITHUB = "github",
  GITLAB = "gitlab",
  LDAP = "ldap",
  TOKEN_AUTH = "token-auth",
  UNIVERSAL_AUTH = "universal-auth",
  KUBERNETES_AUTH = "kubernetes-auth",
  GCP_AUTH = "gcp-auth",
  ALICLOUD_AUTH = "alicloud-auth",
  AWS_AUTH = "aws-auth",
  AZURE_AUTH = "azure-auth",
  TLS_CERT_AUTH = "tls-cert-auth",
  OCI_AUTH = "oci-auth",
  OIDC_AUTH = "oidc-auth",
  JWT_AUTH = "jwt-auth",
  LDAP_AUTH = "ldap-auth",
  SPIFFE_AUTH = "spiffe-auth"
}

export enum AuthAttemptAuthResult {
  SUCCESS = "success",
  FAILURE = "failure"
}

// -- Legacy instruments (documented public contract; do not change labels) --------------------------

export const authAttemptCounter = infisicalMeter.createCounter("infisical.auth.attempt.count", {
  description: "Authentication attempts (both successful and failed)",
  unit: "{attempt}"
});

export const secretReadCounter = infisicalMeter.createCounter("infisical.secret.read.count", {
  description: "Number of secret read operations",
  unit: "{operation}"
});

export const recordSecretReadMetric = (params: { environment: string; secretPath: string; name?: string }) => {
  if (shouldRecordHighCardinalityMetrics()) {
    const attributes: Record<string, string> = {
      "infisical.environment": params.environment,
      "infisical.secret.path": params.secretPath,
      ...(params.name ? { "infisical.secret.name": params.name } : {})
    };

    const orgId = requestContext.get(RequestContextKey.OrgId);
    if (orgId) {
      attributes["infisical.organization.id"] = orgId;
    }

    const orgName = requestContext.get(RequestContextKey.OrgName);
    if (orgName) {
      attributes["infisical.organization.name"] = orgName;
    }

    const projectDetails = requestContext.get(RequestContextKey.ProjectDetails);
    if (projectDetails?.id) {
      attributes["infisical.project.id"] = projectDetails.id;
    }
    if (projectDetails?.name) {
      attributes["infisical.project.name"] = projectDetails.name;
    }

    const userAuthInfo = requestContext.get(RequestContextKey.UserAuthInfo);
    if (userAuthInfo?.userId) {
      attributes["infisical.user.id"] = userAuthInfo.userId;
    }
    if (userAuthInfo?.email) {
      attributes["infisical.user.email"] = userAuthInfo.email;
    }

    const identityAuthInfo = requestContext.get(RequestContextKey.IdentityAuthInfo);
    if (identityAuthInfo?.identityId) {
      attributes["infisical.identity.id"] = identityAuthInfo.identityId;
    }
    if (identityAuthInfo?.identityName) {
      attributes["infisical.identity.name"] = identityAuthInfo.identityName;
    }

    const userAgent = requestContext.get(RequestContextKey.UserAgent);
    if (userAgent) {
      attributes["user_agent.original"] = userAgent;
    }

    const ip = requestContext.get(RequestContextKey.Ip);
    if (ip) {
      attributes["client.address"] = ip;
    }

    secretReadCounter.add(1, attributes);
  }
};

export enum KmipOperationType {
  CREATE = "create",
  GET = "get",
  GET_ATTRIBUTES = "get_attributes",
  ACTIVATE = "activate",
  REVOKE = "revoke",
  DESTROY = "destroy",
  LOCATE = "locate",
  REGISTER = "register"
}

export const kmipOperationCounter = infisicalMeter.createCounter("infisical.kmip.operation.count", {
  description: "Number of KMIP operations performed",
  unit: "{operation}"
});

export const recordKmipOperationMetric = (params: {
  operationType: KmipOperationType;
  orgId: string;
  projectId: string;
  clientId: string;
  objectId?: string;
  objectName?: string;
}) => {
  if (shouldRecordHighCardinalityMetrics()) {
    const attributes: Record<string, string> = {
      "infisical.kmip.operation.type": params.operationType,
      "infisical.organization.id": params.orgId,
      "infisical.project.id": params.projectId,
      "infisical.kmip.client.id": params.clientId
    };

    if (params.objectId) {
      attributes["infisical.kmip.object.id"] = params.objectId;
    }

    if (params.objectName) {
      attributes["infisical.kmip.object.name"] = params.objectName;
    }

    const identityAuthInfo = requestContext.get(RequestContextKey.IdentityAuthInfo);
    if (identityAuthInfo?.identityId) {
      attributes["infisical.identity.id"] = identityAuthInfo.identityId;
    }
    if (identityAuthInfo?.identityName) {
      attributes["infisical.identity.name"] = identityAuthInfo.identityName;
    }

    const userAgent = requestContext.get(RequestContextKey.UserAgent);
    if (userAgent) {
      attributes["user_agent.original"] = userAgent;
    }

    const ip = requestContext.get(RequestContextKey.Ip);
    if (ip) {
      attributes["client.address"] = ip;
    }

    kmipOperationCounter.add(1, attributes);
  }
};

// -- New low-cardinality instruments (InfisicalCore meter) ------------------------------------------

// Queue worker lifecycle metrics. Wired in queue-service.ts via worker.on('completed' | 'failed' | 'stalled').
export const queueJobCounter = infisicalCoreMeter.createCounter("infisical.queue.job.count", {
  description: "Queue jobs processed by outcome (completed or failed)",
  unit: "{job}"
});

export const queueJobDurationHistogram = infisicalCoreMeter.createHistogram("infisical.queue.job.duration", {
  description:
    "Queue job processing duration (worker pickup to completion). Skipped on framework-level failures where processedOn is undefined.",
  unit: "s"
});

export const queueJobWaitHistogram = infisicalCoreMeter.createHistogram("infisical.queue.job.wait", {
  description:
    "Queue job wait time (queue contention only). Subtracts job.opts.delay so intentional scheduling doesn't inflate percentiles.",
  unit: "s"
});

export const queueJobFailureCounter = infisicalCoreMeter.createCounter("infisical.queue.job.failure.count", {
  description:
    "Queue job failures classified by error type. attempts.exhausted=true means all retries are spent (real failure).",
  unit: "{failure}"
});

export const queueStalledCounter = infisicalCoreMeter.createCounter("infisical.queue.stalled.count", {
  description:
    "Stalled queue jobs (lock expired without completing). Strongest signal of a stuck worker / OOM / network partition.",
  unit: "{job}"
});

// In-process refresh metrics. Wired in lib/cron/local-refresh.ts around every run. Each pod runs its own
// copy, so a failure here means that pod is serving stale in-memory state, not that a fleet job was lost.
export enum LocalRefreshOutcome {
  COMPLETED = "completed",
  FAILED = "failed",
  SKIPPED = "skipped"
}

export const localRefreshRunCounter = infisicalCoreMeter.createCounter("infisical.local_refresh.run.count", {
  description:
    "In-process refresh runs by job and outcome. skipped means the previous run was still in progress when the next tick fired.",
  unit: "{run}"
});

export const localRefreshRunDurationHistogram = infisicalCoreMeter.createHistogram(
  "infisical.local_refresh.run.duration",
  {
    description: "In-process refresh run duration by job and outcome. Not recorded for skipped runs.",
    unit: "s"
  }
);

// Read by the infisical.local_refresh.consecutive_failures gauge in registerInfrastructureMetrics().
const localRefreshConsecutiveFailures = new Map<string, number>();

export const recordLocalRefreshRunMetric = (params: {
  name: string;
  outcome: LocalRefreshOutcome;
  durationMs?: number;
  error?: unknown;
  consecutiveFailures?: number;
}) => {
  safely(() => {
    if (params.consecutiveFailures !== undefined) {
      localRefreshConsecutiveFailures.set(params.name, params.consecutiveFailures);
    }
    if (!isTelemetryEnabled()) return;
    const attributes: Record<string, string> = { "job.name": params.name, outcome: params.outcome };
    if (params.durationMs !== undefined) {
      localRefreshRunDurationHistogram.record(params.durationMs / 1000, attributes);
    }
    if (params.outcome === LocalRefreshOutcome.FAILED) {
      attributes["error.type"] = classifyError(params.error);
    }
    localRefreshRunCounter.add(1, attributes);
  });
};

// Audit log lifecycle metrics. Wired in audit-log-queue.ts: enqueued when an event is appended to
// the Redis ingest stream, dropped when the request-path push fails, persist duration around the
// batch insert in the unified consumer.
export const auditLogEnqueuedCounter = infisicalCoreMeter.createCounter("infisical.audit_log.enqueued.count", {
  description: "Audit log events appended to the ingest stream for persistence, by event type and actor type.",
  unit: "{event}"
});

export const auditLogPersistDurationHistogram = infisicalCoreMeter.createHistogram(
  "infisical.audit_log.persist.duration",
  {
    description: "Latency of the consumer's batch insert to durable storage (postgres or clickhouse), by outcome.",
    unit: "s"
  }
);

export const auditLogDroppedCounter = infisicalCoreMeter.createCounter("infisical.audit_log.dropped.count", {
  description:
    "Audit log events dropped on the request path because the ingest-stream push failed (at-most-once). Operators should alert on this.",
  unit: "{event}"
});

// Audit log stream metrics. Wired in audit-log-stream-outbox-service.ts drainStream() per provider send.
export const auditLogStreamDeliveryDurationHistogram = infisicalCoreMeter.createHistogram(
  "infisical.audit_log_stream.delivery.duration",
  {
    description: "Per-provider audit log stream delivery latency and attempt count (use _count for delivery volume).",
    unit: "s"
  }
);

// Wired in audit-log-stream-outbox-service.ts. Incremented when stream events are dropped after
// exhausting all delivery retries (there is no DLQ — the events are gone). Operators should alert on this.
export const auditLogStreamDeliveryExhaustedCounter = infisicalCoreMeter.createCounter(
  "infisical.audit_log_stream.delivery.exhausted.count",
  {
    description:
      "Audit log stream events dropped after exhausting all delivery retries, by stream and org. Operators should alert on this.",
    unit: "{event}"
  }
);

// Permission cache metrics. Wired in lib/cache/with-cache.ts withCacheFingerprint().
export const permissionCacheLookupCounter = infisicalCoreMeter.createCounter(
  "infisical.permission_cache.lookup.count",
  {
    description: "Permission cache lookups by branch: short-marker hit, fingerprint match, full refetch.",
    unit: "{lookup}"
  }
);

export const permissionCacheFingerprintDurationHistogram = infisicalCoreMeter.createHistogram(
  "infisical.permission_cache.fingerprint.duration",
  {
    description: "Time spent computing the permission fingerprint (lightweight DB read on marker expiry).",
    unit: "s"
  }
);

// Secret service-layer cache metrics. Wired in secret-service.ts at cache read/write boundaries.
export enum SecretCacheAccessResult {
  NOT_MODIFIED = "not_modified",
  HIT = "hit",
  MISS = "miss"
}

// What the server observed when an If-None-Match request did not 304 (cause is inferred downstream).
export enum SecretEtagMissReason {
  FIELD_ABSENT = "field_absent", // no stored ETag for this (actor, fingerprint, params) key
  VALUE_DIFFERS = "value_differs" // a stored ETag exists but differs from the client's If-None-Match
}

export const secretCacheAccessCounter = infisicalCoreMeter.createCounter("infisical.secret.cache.access.count", {
  description: "Secret cache accesses, labeled by result, whether If-None-Match was sent, and why it missed the 304.",
  unit: "{access}"
});

export const secretCacheEntryBytesHistogram = infisicalCoreMeter.createHistogram("infisical.secret.cache.entry.bytes", {
  description: "Encrypted secret cache entry size computed at write time",
  unit: "By"
});

export const secretCacheOversizeSkipCounter = infisicalCoreMeter.createCounter(
  "infisical.secret.cache.oversize_skip.count",
  {
    description: "Secret cache writes skipped because the entry exceeded the max byte cap",
    unit: "{skip}"
  }
);

export const recordSecretCacheAccessMetric = (
  result: SecretCacheAccessResult,
  opts?: { hasIfNoneMatch?: boolean; etagMissReason?: SecretEtagMissReason }
) => {
  if (!isTelemetryEnabled()) return;
  const attributes: Record<string, string> = { "cache.result": result };
  if (opts?.hasIfNoneMatch !== undefined) attributes["cache.if_none_match"] = opts.hasIfNoneMatch ? "true" : "false";
  if (opts?.etagMissReason) attributes["cache.etag_miss_reason"] = opts.etagMissReason;
  secretCacheAccessCounter.add(1, attributes);
};

export const recordSecretCacheWriteMetric = (params: { bytes: number; stored: boolean }) => {
  if (!isTelemetryEnabled()) return;
  secretCacheEntryBytesHistogram.record(params.bytes);
  if (!params.stored) {
    secretCacheOversizeSkipCounter.add(1);
  }
};

// -- safeRequest HTTPS agent pool (InfisicalCore meter) ----------------------------------------
// The pool is bounded, so a 201st distinct TLS signature costs the least recently used entry its
// connection. Non-zero for a sustained window means the cap is too low and should be raised.
export const safeRequestAgentEvictionCounter = infisicalCoreMeter.createCounter(
  "infisical.safe_request.agent_eviction.count",
  {
    description:
      "Agents evicted from the safeRequest connection pool because it was at capacity. Non-zero for a sustained window means raise AGENT_CACHE_MAX.",
    unit: "{eviction}"
  }
);

export const recordSafeRequestAgentEvictionMetric = () => {
  if (!isTelemetryEnabled()) return;
  safeRequestAgentEvictionCounter.add(1);
};

export const coreHttpErrorCounter = infisicalCoreMeter.createCounter("infisical.core.http.error.count", {
  description: "API errors with bounded error classification. Labels limited to InfisicalCore View allowlist.",
  unit: "{error}"
});

// Denominator for coreHttpErrorCounter so error rate per route survives OTEL_DROP_HIGH_CARDINALITY_METERS.
// normalizeHttpMethod mirrors @opentelemetry/instrumentation-http KNOWN_METHODS (_OTHER for the rest) for joinable labels.
const KNOWN_HTTP_METHODS = new Set([
  "GET",
  "HEAD",
  "POST",
  "PUT",
  "DELETE",
  "CONNECT",
  "OPTIONS",
  "TRACE",
  "PATCH",
  "QUERY"
]);

export const normalizeHttpMethod = (method?: string): string => {
  if (!method) return "GET";
  const upper = method.toUpperCase();
  return KNOWN_HTTP_METHODS.has(upper) ? upper : "_OTHER";
};

export const coreHttpRequestCounter = infisicalCoreMeter.createCounter("infisical.core.http.request.count", {
  description: "API requests with bounded labels. Labels limited to InfisicalCore View allowlist.",
  unit: "{request}"
});

// -- Signup abuse (InfisicalCore meter) -------------------------------------------------------------

export enum EmailDispatchPurpose {
  SIGNUP = "signup",
  ACCOUNT_RECOVERY = "account-recovery"
}

export enum EmailDispatchMailboxProvider {
  GOOGLE = "google",
  OTHER = "other"
}

export enum EmailDispatchAddressForm {
  CANONICAL = "canonical",
  ALIASED = "aliased"
}

export enum EmailDispatchOutcome {
  SENT = "sent",
  EXISTING_ACCOUNT = "existing-account",
  NO_RECIPIENT = "no-recipient",
  MAILBOX_CAPPED = "mailbox-capped",
  CAPTCHA_REJECTED = "captcha-rejected"
}

export enum EmailDispatchDimension {
  SOURCE = "source",
  MAILBOX = "mailbox"
}

export const emailDispatchRequestCounter = infisicalCoreMeter.createCounter("infisical.email_dispatch.request.count", {
  description:
    "Requests to the unauthenticated endpoints that mail a caller-chosen address, by purpose, mailbox provider, address form, and outcome.",
  unit: "{request}"
});

export const emailDispatchDistinctCounter = infisicalCoreMeter.createCounter(
  "infisical.email_dispatch.distinct.count",
  {
    description:
      "First sighting of a source host or target mailbox within the current abuse window. Compare against the request count to separate a broad campaign from a burst against a few targets.",
    unit: "{entity}"
  }
);

// Rate limit metric. Wired in error-handler.ts on RateLimitError.
export const rateLimitExceededCounter = infisicalCoreMeter.createCounter("infisical.rate_limit.exceeded.count", {
  description: "HTTP 429 responses (rate limit exceeded).",
  unit: "{request}"
});

// -- Authentication latency (InfisicalCore meter) ---------------------------------------------------
export const authAttemptDurationHistogram = infisicalCoreMeter.createHistogram("infisical.auth.attempt.duration", {
  description:
    "Authentication attempt latency by method and result. External verifications (SAML/OIDC/Kubernetes/cloud) include the IdP/provider network round trip.",
  unit: "s"
});

export const recordAuthAttemptMetric = (params: {
  startTime: number;
  method: AuthAttemptAuthMethod;
  result: AuthAttemptAuthResult;
  error?: unknown;
  orgId?: string;
}) => {
  if (!isTelemetryEnabled()) return;
  const attributes: Record<string, string> = {
    "infisical.auth.method": params.method,
    "infisical.auth.result": params.result
  };
  if (params.error !== undefined) attributes["error.type"] = classifyError(params.error);
  authAttemptDurationHistogram.record((performance.now() - params.startTime) / 1000, attributes);
};

// Machine identity access token renewals. Distinct from the auth.attempt series (initial login).
export const tokenRenewalCounter = infisicalCoreMeter.createCounter("infisical.auth.token.renewal.count", {
  description: "Machine identity access token renewal attempts by outcome.",
  unit: "{renewal}"
});

export const recordTokenRenewalMetric = (params: {
  outcome: "success" | "failure";
  authMethod?: string;
  error?: unknown;
}) => {
  if (!isTelemetryEnabled()) return;
  const attributes: Record<string, string> = { outcome: params.outcome };
  if (params.authMethod) attributes["infisical.auth.method"] = params.authMethod;
  if (params.error !== undefined) attributes["error.type"] = classifyError(params.error);
  tokenRenewalCounter.add(1, attributes);
};

// -- SCIM provisioning (InfisicalCore meter) --------------------------------------------------------
export enum ScimOperation {
  CreateUser = "create_user",
  UpdateUser = "update_user",
  ReplaceUser = "replace_user",
  DeleteUser = "delete_user",
  CreateGroup = "create_group",
  UpdateGroup = "update_group",
  ReplaceGroup = "replace_group",
  DeleteGroup = "delete_group"
}

export const scimOperationDurationHistogram = infisicalCoreMeter.createHistogram("infisical.scim.operation.duration", {
  description: "SCIM provisioning operation latency by operation type and outcome (use _count for operation volume).",
  unit: "s"
});

export const recordScimOperationMetric = (params: {
  startTime: number;
  operation: ScimOperation;
  outcome: "success" | "failure";
  orgId?: string;
  error?: unknown;
}) => {
  if (!isTelemetryEnabled()) return;
  const attributes: Record<string, string> = {
    "scim.operation": params.operation,
    outcome: params.outcome
  };
  if (params.error !== undefined) attributes["error.type"] = classifyError(params.error);
  scimOperationDurationHistogram.record((performance.now() - params.startTime) / 1000, attributes);
};

// -- SSO config lifecycle (InfisicalCore meter) -----------------------------------------------------
export enum SsoProvider {
  Saml = "saml",
  Oidc = "oidc",
  Ldap = "ldap"
}

export enum SsoConfigAction {
  Create = "create",
  Update = "update"
}

export const ssoConfigChangeCounter = infisicalCoreMeter.createCounter("infisical.sso.config.change.count", {
  description: "SSO configuration changes (create/update) by provider. Detects unexpected SSO reconfiguration.",
  unit: "{change}"
});

export const recordSsoConfigChangeMetric = (params: {
  provider: SsoProvider;
  action: SsoConfigAction;
  orgId?: string;
}) => {
  if (!isTelemetryEnabled()) return;
  const attributes: Record<string, string> = {
    "sso.provider": params.provider,
    "sso.action": params.action
  };
  ssoConfigChangeCounter.add(1, attributes);
};

// -- Secret operation metrics (InfisicalCore meter) ------------------------------------------------
export const secretOperationDurationHistogram = infisicalCoreMeter.createHistogram(
  "infisical.secret.operation.duration",
  {
    description: "Secret operation latency by operation type, outcome, and environment.",
    unit: "ms"
  }
);

export const secretWriteCounter = infisicalCoreMeter.createCounter("infisical.secret.write.count", {
  description: "Secret write operations (create/update/delete).",
  unit: "{operation}"
});

export const recordSecretOperationDuration = (params: {
  startTime: number;
  operation: "read" | "write";
  outcome: "success" | "failure";
}) => {
  if (!isTelemetryEnabled()) return;
  secretOperationDurationHistogram.record(performance.now() - params.startTime, {
    operation: params.operation,
    outcome: params.outcome
  });
};

export const recordSecretWriteMetric = (params: { operation: "create" | "update" | "delete" }) => {
  if (!isTelemetryEnabled()) return;
  secretWriteCounter.add(1, {
    operation: params.operation
  });
};

// -- Secret sync outcome (InfisicalCore meter) ----------------------------------------------------
export const secretSyncOutcomeCounter = infisicalCoreMeter.createCounter("infisical.secret_sync.outcome.count", {
  description:
    "Secret sync attempts by destination, operation, and outcome. A failure with attempts.exhausted=false is an intermediate retry; a finished sync is a success or a failure with attempts.exhausted=true.",
  unit: "{attempt}"
});

export const recordSecretSyncOutcomeMetric = (params: {
  destination: string;
  operation: "sync" | "import" | "remove";
  outcome: "success" | "failure";
  attemptsExhausted: boolean;
}) => {
  if (!isTelemetryEnabled()) return;
  secretSyncOutcomeCounter.add(1, {
    destination: params.destination,
    operation: params.operation,
    outcome: params.outcome,
    "attempts.exhausted": String(params.attemptsExhausted)
  });
};

export const secretSyncStatusChangeCounter = infisicalCoreMeter.createCounter(
  "infisical.secret_sync.status_change.count",
  {
    description:
      "Secret syncs whose finished result flipped between healthy and failed, by destination. Counts distinct configs breaking without a per-config label.",
    unit: "{change}"
  }
);

export const recordSecretSyncStatusChangeMetric = (params: {
  destination: string;
  newStatus: "failed" | "healthy";
}) => {
  if (!isTelemetryEnabled()) return;
  secretSyncStatusChangeCounter.add(1, {
    destination: params.destination,
    new_status: params.newStatus
  });
};

// -- Secret rotation outcome (InfisicalCore meter) --------------------------------------------------
export const secretRotationOutcomeCounter = infisicalCoreMeter.createCounter(
  "infisical.secret_rotation.outcome.count",
  {
    description:
      "Secret rotation attempts by type and outcome. A failure with attempts.exhausted=false is an intermediate retry; a finished rotation is a success or a failure with attempts.exhausted=true.",
    unit: "{attempt}"
  }
);

export const recordSecretRotationOutcomeMetric = (params: {
  type: string;
  outcome: "success" | "failure";
  attemptsExhausted: boolean;
}) => {
  if (!isTelemetryEnabled()) return;
  secretRotationOutcomeCounter.add(1, {
    type: params.type,
    outcome: params.outcome,
    "attempts.exhausted": String(params.attemptsExhausted)
  });
};

export const secretRotationStatusChangeCounter = infisicalCoreMeter.createCounter(
  "infisical.secret_rotation.status_change.count",
  {
    description:
      "Secret rotations whose finished result flipped between healthy and failed, by type. Counts distinct configs breaking without a per-config label.",
    unit: "{change}"
  }
);

export const recordSecretRotationStatusChangeMetric = (params: { type: string; newStatus: "failed" | "healthy" }) => {
  if (!isTelemetryEnabled()) return;
  secretRotationStatusChangeCounter.add(1, {
    type: params.type,
    new_status: params.newStatus
  });
};

// -- Dynamic secret orphaned lease (InfisicalCore meter) ---------------------------------------------
export const dynamicSecretOrphanedLeaseCounter = infisicalCoreMeter.createCounter(
  "infisical.dynamic_secret.orphaned_lease.count",
  {
    description:
      "Dynamic secret leases left unrevoked after every automatic revocation attempt failed, by provider. Each is a live credential that needs manual cleanup.",
    unit: "{failure}"
  }
);

export const recordDynamicSecretOrphanedLeaseMetric = (params: { provider: string; count?: number }) => {
  if (!isTelemetryEnabled()) return;
  dynamicSecretOrphanedLeaseCounter.add(params.count ?? 1, {
    provider: params.provider
  });
};

export const dynamicSecretRevocationOutcomeCounter = infisicalCoreMeter.createCounter(
  "infisical.dynamic_secret.revocation.outcome.count",
  {
    description:
      "Dynamic secret lease revocations by provider, trigger, and outcome. A scheduled revocation (lease expiry) records exactly one outcome per lease; retry records every attempt, with attempts.exhausted=true on the last.",
    unit: "{revocation}"
  }
);

export const recordDynamicSecretRevocationOutcomeMetric = (params: {
  provider: string;
  trigger: "scheduled" | "retry" | "manual" | "pruning";
  outcome: "success" | "failure";
  attemptsExhausted: boolean;
}) => {
  if (!isTelemetryEnabled()) return;
  dynamicSecretRevocationOutcomeCounter.add(1, {
    provider: params.provider,
    trigger: params.trigger,
    outcome: params.outcome,
    "attempts.exhausted": String(params.attemptsExhausted)
  });
};

export const dynamicSecretStatusChangeCounter = infisicalCoreMeter.createCounter(
  "infisical.dynamic_secret.status_change.count",
  {
    description:
      "Dynamic secret configs whose lease revocations flipped between working and failing, by provider: failed when a scheduled revocation fails for good, healthy when any revocation succeeds again. Counts distinct configs breaking without a per-config label.",
    unit: "{change}"
  }
);

export const recordDynamicSecretStatusChangeMetric = (params: {
  provider: string;
  newStatus: "failed" | "healthy";
}) => {
  if (!isTelemetryEnabled()) return;
  dynamicSecretStatusChangeCounter.add(1, {
    provider: params.provider,
    new_status: params.newStatus
  });
};

export const dynamicSecretUnrecordedCredentialCounter = infisicalCoreMeter.createCounter(
  "infisical.dynamic_secret.unrecorded_credential.count",
  {
    description:
      "Dynamic secret credentials created upstream whose lease failed to persist and whose cleanup revocation also failed, by provider. Each is a live credential with no lease to revoke it.",
    unit: "{credential}"
  }
);

export const recordDynamicSecretUnrecordedCredentialMetric = (params: { provider: string }) => {
  if (!isTelemetryEnabled()) return;
  dynamicSecretUnrecordedCredentialCounter.add(1, {
    provider: params.provider
  });
};

export enum AlertDispatchOutcome {
  // Every channel the run touched delivered.
  DeliverySuccess = "delivery_success",
  // Some channels delivered and at least one did not, so a notification was dropped on the failed
  // channels only. Dedup is per (channel, target), so the next run retries just those channels.
  DeliveryPartial = "delivery_partial",
  // No channel delivered: the whole run dropped.
  DeliveryFailed = "delivery_failed",
  // The alert row is gone by the time the job runs: deleted, or its project soft-deleted.
  AlertNotFound = "alert_not_found",
  // The alert still exists but was disabled between being enqueued and the job running.
  AlertDisabled = "alert_disabled",
  // No provider registered for the alert's resource type (misconfiguration).
  NoProvider = "no_provider",
  // Nothing to alert on: nothing matched the condition, or the targets an event named are gone.
  NoDueTargets = "no_due_targets",
  // The alert has no enabled channels, so the run is skipped before scanning for targets.
  NoChannels = "no_channels",
  // Every channel in the run was directed (email) with no resolvable recipient — the recipients left
  // the org, or the recipient group emptied out. Customer config drift, not a delivery fault, so it
  // is kept out of delivery_failed and must not alarm.
  NoRecipients = "no_recipients",
  // Targets matched, but every one had already been alerted inside the dedup window.
  AllDeduped = "all_deduped",
  // Event path only: no alert configured for the event. Normal in steady state.
  NoMatchingAlert = "no_matching_alert"
}

export const alertDispatchOutcomeCounter = infisicalCoreMeter.createCounter("infisical.alert.dispatch.outcome.count", {
  description:
    "Alert dispatch jobs by alert resource type and outcome. Alarm on delivery_failed + delivery_partial: both mean a channel could not be reached, so a notification was dropped. Watch the delivery_* share of total separately: a persistently low ratio means the cron is enqueueing mostly no-op jobs and should pre-filter instead.",
  unit: "{job}"
});

export const recordAlertDispatchOutcomeMetric = (params: { resourceType: string; outcome: AlertDispatchOutcome }) => {
  if (!isTelemetryEnabled()) return;
  alertDispatchOutcomeCounter.add(1, {
    type: params.resourceType,
    outcome: params.outcome
  });
};

// -- Object storage (InfisicalCore meter) -------------------------------------------------------------
// Upload links the shared object storage layer gave out, or refused at the daily cap. Instance totals only: a per-org
// split would need org ids as labels.

export enum ObjectStorageUploadOutcome {
  Admitted = "admitted",
  Capped = "capped"
}

export const objectStorageUploadCounter = infisicalCoreMeter.createCounter("infisical.object_storage.upload.count", {
  description:
    "Upload links the object storage layer gave out or refused at the daily cap, by storage type and outcome.",
  unit: "{upload}"
});

export const objectStorageUploadBytesCounter = infisicalCoreMeter.createCounter(
  "infisical.object_storage.upload.bytes",
  {
    description: "Bytes the object storage layer's upload links allow, by storage type and outcome.",
    unit: "By"
  }
);

export const recordObjectStorageUploadMetric = (params: {
  storageType: string;
  outcome: ObjectStorageUploadOutcome;
  bytes: number;
}) =>
  safely(() => {
    if (!isTelemetryEnabled()) return;
    const attributes = { type: params.storageType, outcome: params.outcome };
    objectStorageUploadCounter.add(1, attributes);
    objectStorageUploadBytesCounter.add(params.bytes, attributes);
  });

// -- Event outbox (InfisicalCore meter) --------------------------------------------------------------
// Outbox health, not what consumers do with events. The main canary is the oldest-pending-age gauge
// in event-outbox-queue.ts, fed by the relay tick.

export const eventOutboxLagHistogram = infisicalCoreMeter.createHistogram("infisical.event_outbox.lag", {
  description:
    "Seconds from an event occurring to the outbox reaching a terminal result for it. Tracks the relay interval plus delivery time; a p99 far above the interval means delivery, not discovery, is the bottleneck.",
  unit: "s"
});

export const recordEventOutboxLagMetric = (params: { consumer: string; status: string; seconds: number }) =>
  safely(() => {
    if (!isTelemetryEnabled()) return;
    eventOutboxLagHistogram.record(params.seconds, {
      "event_outbox.consumer": params.consumer,
      "event_outbox.status": params.status
    });
  });

export const eventOutboxExhaustedCounter = infisicalCoreMeter.createCounter("infisical.event_outbox.exhausted.count", {
  description:
    "Outbox events abandoned after exhausting their attempts. Every one of these is a notification the customer configured and did not receive, so alarm on any sustained rate.",
  unit: "{event}"
});

export const recordEventOutboxExhaustedMetric = (params: { consumer: string; count: number }) =>
  safely(() => {
    if (!isTelemetryEnabled()) return;
    eventOutboxExhaustedCounter.add(params.count, { "event_outbox.consumer": params.consumer });
  });

export enum EventOutboxRejectReason {
  InvalidEvent = "invalid_event",
  DataTooLarge = "data_too_large",
  DataRejected = "data_rejected"
}

export const eventOutboxRejectedCounter = infisicalCoreMeter.createCounter("infisical.event_outbox.rejected.count", {
  description:
    "Events the outbox refused because the emit site built a bad one, by event type and reason. The emit throws, so the business write the event belongs to fails with it. Alarm on any occurrence: it is a bug at the emit site.",
  unit: "{event}"
});

export const recordEventOutboxRejectedMetric = (params: {
  eventType: string;
  reason: EventOutboxRejectReason;
  consumer?: string;
}) =>
  safely(() => {
    if (!isTelemetryEnabled()) return;
    eventOutboxRejectedCounter.add(1, {
      "event_outbox.event_type": params.eventType,
      "event_outbox.reject_reason": params.reason,
      ...(params.consumer ? { "event_outbox.consumer": params.consumer } : {})
    });
  });

export enum ProductAnalyticsDropReason {
  Retention = "retention",
  Unparseable = "unparseable"
}

export const productAnalyticsPublishedCounter = infisicalCoreMeter.createCounter(
  "infisical.product_analytics.published.count",
  {
    description: "Buffered product analytics events drained from Redis and published to PostHog, by event type.",
    unit: "{event}"
  }
);

export const productAnalyticsDroppedCounter = infisicalCoreMeter.createCounter(
  "infisical.product_analytics.dropped.count",
  {
    description:
      "Buffered product analytics events dropped before reaching PostHog, by event type and reason. Occasional retention drops are tolerable; a sustained rate means the drain is not keeping up and the limits need tweaking.",
    unit: "{event}"
  }
);

export const productAnalyticsBacklogHistogram = infisicalCoreMeter.createHistogram(
  "infisical.product_analytics.shard.backlog",
  {
    description:
      "Entries left in a shard after its drain, by event type. Zero on a healthy run: a backlog that persists across runs is what precedes retention drops and, near the 100k MAXLEN, silent write-path eviction.",
    unit: "{entry}"
  }
);

export const recordProductAnalyticsPublishedMetric = (params: { eventType: string; count: number }) =>
  safely(() => {
    if (!isTelemetryEnabled() || params.count === 0) return;
    productAnalyticsPublishedCounter.add(params.count, { "product_analytics.event_type": params.eventType });
  });

export const recordProductAnalyticsDroppedMetric = (params: {
  eventType: string;
  reason: ProductAnalyticsDropReason;
  count: number;
}) =>
  safely(() => {
    if (!isTelemetryEnabled() || params.count === 0) return;
    productAnalyticsDroppedCounter.add(params.count, {
      "product_analytics.event_type": params.eventType,
      "product_analytics.drop_reason": params.reason
    });
  });

export const recordProductAnalyticsBacklogMetric = (params: { eventType: string; backlog: number }) =>
  safely(() => {
    if (!isTelemetryEnabled()) return;
    productAnalyticsBacklogHistogram.record(params.backlog, {
      "product_analytics.event_type": params.eventType
    });
  });

// -- Boot-time observable gauges (InfisicalCore meter) ----------------------------------------------
// Registered once at boot from main.ts with the primary Knex instance. Runs AFTER setupTelemetry() has
// installed the real MeterProvider, so we resolve the real meter directly here (observable gauges can't
// be deferred to first use like counters/histograms — the SDK pulls them on each export).
export const registerInfrastructureMetrics = (db: Knex) => {
  const meter = resolveCoreMeter();

  // Build info: constant-value gauge that emits 1 with build identification labels on every export.
  const buildInfoGauge = meter.createObservableGauge("infisical.build.info", {
    description: "Always 1. Labels carry build identification (version, git sha, node version)."
  });
  buildInfoGauge.addCallback((result) => {
    if (!isTelemetryEnabled()) return;
    result.observe(1, {
      "service.version": process.env.INFISICAL_PLATFORM_VERSION || "unknown",
      "git.commit.sha": process.env.DD_GIT_COMMIT_SHA || "unknown",
      "node.version": process.version
    });
  });

  // Connection pool: reads in-memory tarn pool counters, so it's cheap to observe on every export.
  const dbPoolGauge = meter.createObservableGauge("infisical.db.pool.connections", {
    description: "Knex/tarn connection pool counts by state (used, free, pending).",
    unit: "{connection}"
  });

  dbPoolGauge.addCallback((result) => {
    if (!isTelemetryEnabled()) return;
    const pool = (
      db.client as
        | {
            pool?: {
              numUsed?: () => number;
              numFree?: () => number;
              numPendingAcquires?: () => number;
            };
          }
        | undefined
    )?.pool;
    if (!pool) return;
    result.observe(pool.numUsed?.() ?? 0, { "db.pool.state": "used" });
    result.observe(pool.numFree?.() ?? 0, { "db.pool.state": "free" });
    result.observe(pool.numPendingAcquires?.() ?? 0, { "db.pool.state": "pending" });
  });

  // safeRequest agent pool: an in-memory Map size, so it's cheap to observe on every export.
  // Read alongside infisical.safe_request.agent_eviction.count — size pinned at max with a
  // non-zero eviction rate is the signal that the cap is too low.
  const agentPoolGauge = meter.createObservableGauge("infisical.safe_request.agent_pool.size", {
    description: "Agents currently held in the safeRequest connection pool.",
    unit: "{agent}"
  });

  agentPoolGauge.addCallback((result) => {
    if (!isTelemetryEnabled()) return;
    const { size, max } = getAgentPoolStats();
    result.observe(size, { "pool.max": String(max) });
  });

  // Resets to 0 on the next successful run, so a sustained non-zero value means the refresh is stuck
  // failing rather than flaking. Alert on this instead of on the failure rate.
  const localRefreshFailuresGauge = meter.createObservableGauge("infisical.local_refresh.consecutive_failures", {
    description: "Consecutive failed runs of each in-process refresh on this pod. Resets to 0 on success.",
    unit: "{run}"
  });

  localRefreshFailuresGauge.addCallback((result) => {
    if (!isTelemetryEnabled()) return;
    localRefreshConsecutiveFailures.forEach((count, name) => {
      result.observe(count, { "job.name": name });
    });
  });
};

// -- Legacy root-key usage (InfisicalCore meter) -----------------------------------------------------
// The pre-KMS tier pins the instance root encryption key, so it can never be rotated while anything
// still uses it. This counter is the evidence for when that tier can be deleted.
export const legacyRootKeyUsageCounter = infisicalCoreMeter.createCounter("infisical.legacy_root_key.usage", {
  description:
    "Reads and writes that still use the instance root encryption key directly instead of the KMS envelope, by surface."
});

export type LegacyRootKeySurface =
  | "project_bot"
  | "user_private_key"
  | "blind_index"
  | "external_migration"
  | "org_bot"
  | "project_ghost_user";

export const recordLegacyRootKeyUsageMetric = (params: {
  operation: "encrypt" | "decrypt";
  surface: LegacyRootKeySurface;
}) => {
  safely(() => {
    if (!isTelemetryEnabled()) return;
    legacyRootKeyUsageCounter.add(1, {
      "legacy_key.operation": params.operation,
      "legacy_key.surface": params.surface
    });
  });
};

export type TOcspStatusLabel = "successful" | "malformed_request" | "internal_error" | "try_later" | "unauthorized";

export type TOcspCertStatusLabel = "good" | "revoked" | "unknown" | "none";

export const ocspResponseCounter = infisicalCoreMeter.createCounter("infisical.ocsp.response.count", {
  description:
    "OCSP responses by response status, certificate status and cache outcome. cert_status is 'none' when the response carries no certificate status, which is every status other than successful. A rising try_later means the signing budget is saturated.",
  unit: "{response}"
});

export const ocspSigningDurationHistogram = infisicalCoreMeter.createHistogram("infisical.ocsp.signing.duration", {
  description: "Time spent resolving status and signing a fresh OCSP response, excluding cache hits.",
  unit: "ms"
});

export const recordOcspResponseMetric = (params: {
  status: TOcspStatusLabel;
  certStatus: TOcspCertStatusLabel;
  cache: "hit" | "miss" | "skipped" | "coalesced";
}) => {
  safely(() => {
    if (!isTelemetryEnabled()) return;
    ocspResponseCounter.add(1, {
      "ocsp.status": params.status,
      "ocsp.cert_status": params.certStatus,
      "ocsp.cache": params.cache
    });
  });
};

export const recordOcspSigningDurationMetric = (params: { durationMs: number }) => {
  safely(() => {
    if (!isTelemetryEnabled()) return;
    ocspSigningDurationHistogram.record(params.durationMs);
  });
};