Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import * as k8s from '@kubernetes/client-node'
import { hccLogger } from './logger'
import {
type CreateKind,
createsTotal,
existenceReadsTotal,
writeSkipsTotal,
writesTotal,
} from './metrics'
/** Extract HTTP status code from a K8s client error (handles both error formats). */
export function getErrorCode(error: unknown): number | undefined {
const e = error as { code?: number; response?: { statusCode?: number } }
return e.code ?? e.response?.statusCode
}
/** Observe only the create request, preserving its response and error identity. */
export async function observeCreate<T>(kind: CreateKind, create: () => Promise<T>): Promise<T> {
try {
const result = await create()
createsTotal.inc({ kind, outcome: 'created' })
return result
} catch (error) {
const outcome = error != null && getErrorCode(error) === 409 ? 'conflict' : 'error'
createsTotal.inc({ kind, outcome })
throw error
}
}
/** Outcome of a read-first create-or-replace. Callers may ignore this value. */
export type ResourceApplyResult = 'created' | 'replaced' | 'up_to_date' | 'missing' | 'not_allowed'
/** Counters for one reconciler apply pass. `writes + skips === objects` is the invariant. */
export type ApplyCountStats = { objects: number; writes: number; skips: number }
/** True when the apply mutated the cluster (create or replace). */
export function applyResultIsWrite(result: ResourceApplyResult): boolean {
switch (result) {
case 'created':
case 'replaced':
return true
case 'up_to_date':
case 'missing':
case 'not_allowed':
return false
default: {
const exhaustive: never = result
throw new Error(`unhandled apply result: ${String(exhaustive)}`)
}
}
}
/** Bucket one apply outcome. Both GFS and LlmHook must use this mapping. */
export function accumulateApplyResult(stats: ApplyCountStats, result: ResourceApplyResult): void {
stats.objects += 1
if (applyResultIsWrite(result)) stats.writes += 1
else stats.skips += 1
}
/** Observe one existence GET without changing its value or error handling. */
export async function observeExistenceRead<T>(
kind: CreateKind,
read: () => Promise<T>
): Promise<T> {
try {
const result = await read()
existenceReadsTotal.inc({ kind, outcome: 'found' })
return result
} catch (error) {
const outcome = error != null && getErrorCode(error) === 404 ? 'absent' : 'error'
existenceReadsTotal.inc({ kind, outcome })
throw error
}
}
/**
* Read before creating, then retain each caller's convergence policy. The first
* observed object is consumed once by the existing retry loop; a conflict must
* always reread the API. SDK calls and their metric labels stay at the call site.
*/
export async function ensureResource<T extends { metadata?: { name?: string } }>(opts: {
read: () => Promise<T>
create: () => Promise<unknown>
existing?: T | null
/** False preserves a caller's non-throwing failure without counting a skip. */
converge: (read: () => Promise<T>, observed?: T) => Promise<void | false | ResourceApplyResult>
onSkipped: () => void
mutationAllowed?: () => boolean
}): Promise<ResourceApplyResult> {
const { read, create, converge, onSkipped, mutationAllowed } = opts
if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
const validate = (value: T): T => {
if (!value || typeof value.metadata?.name !== 'string' || !value.metadata.name.trim()) {
throw new Error('Kubernetes existence read must resolve to an object with metadata.name')
}
return value
}
// The existing retry may deliberately return void after a missing read.
// Retain that policy while distinguishing it from completed preservation.
// This is invocation-local live-read state, not state of the cached snapshot.
// POST409 convergence never counts skipped, even when its fresh read succeeds.
let latestReadSucceeded = true
const readFresh = async (): Promise<T> => {
try {
const value = validate(await read())
latestReadSucceeded = true
return value
} catch (error) {
latestReadSucceeded = false
throw error
}
}
let existing = opts.existing
if (existing === undefined) {
try {
existing = await readFresh()
} catch (error) {
if (error == null || getErrorCode(error) !== 404) throw error
existing = null
}
} else if (existing !== null) {
validate(existing)
}
if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
if (existing !== null) {
let snapshot: T | undefined = existing
const readOnce = async (): Promise<T> => {
if (snapshot !== undefined) {
const observed = snapshot
snapshot = undefined
return observed
}
return readFresh()
}
const result = await converge(readOnce, existing)
if (result !== false && latestReadSucceeded && (!mutationAllowed || mutationAllowed())) {
onSkipped()
}
if (result === false) return 'not_allowed'
if (typeof result === 'string') return result
return latestReadSucceeded ? 'up_to_date' : 'missing'
}
try {
await create()
return 'created'
} catch (error) {
if (error == null || getErrorCode(error) !== 409) throw error
if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
// Absence observed before the POST cannot survive a create conflict.
const result = await converge(readFresh)
if (result === false) return 'not_allowed'
if (typeof result === 'string') return result
return latestReadSucceeded ? 'up_to_date' : 'missing'
}
}
/**
* Re-read + replace with optimistic-lock retry. K8s rejects a replace whose
* `metadata.resourceVersion` does not match the current value; the only
* recovery is read-modify-write. Concurrent reconcilers (McpServer/Context/
* Host watch events firing for the same downstream resource) routinely race
* here, so we retry up to maxAttempts times before propagating.
*
* Between attempts we sleep a jittered backoff so concurrent retriers don't
* lockstep-collide on the same resourceVersion. Without jitter, N watchers
* all read the same RV, all attempt replace, all but one fail, all retry
* simultaneously, repeat — amplifying API load instead of converging.
*/
export async function replaceWithConflictRetry<
T extends { metadata?: { resourceVersion?: string }; kind?: string },
>(
opts: {
description: string
logPrefix: string
read: () => Promise<T>
replace: (body: T) => Promise<unknown>
mergeExisting?: (body: T, existing: T) => T
/**
* Called after the desired object has been merged with server-owned fields.
* Returning true avoids a no-op replace (and therefore avoids a needless
* resourceVersion/generation bump) while retaining the conflict-retry path
* when a meaningful change is still required.
*/
isUpToDate?: (body: T, existing: T) => boolean
/** Reject an object whose identity or ownership is unsafe to replace. */
validateExisting?: (existing: T) => void
/** Rechecked immediately before every Kubernetes write attempt. */
mutationAllowed?: () => boolean
/** Admission-sensitive callers must not treat an absent object as applied. */
missingIsError?: boolean
maxAttempts?: number
} & (
| { body: T; resolveBody?: () => T | Promise<T> }
/** Rebuild desired state before each attempt without an unused eager body. */
| { body?: never; resolveBody: () => T | Promise<T> }
)
): Promise<ResourceApplyResult> {
const {
description,
logPrefix,
read,
replace,
mergeExisting,
isUpToDate,
validateExisting,
mutationAllowed,
missingIsError = false,
maxAttempts = 3,
} = opts
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
let existing: T
try {
existing = await read()
} catch (err) {
if (getErrorCode(err) === 404) {
if (missingIsError) throw err
hccLogger.warn('Kubernetes resource disappeared; skipping replace', {
scope: logPrefix,
description,
})
return 'missing'
}
throw err
}
validateExisting?.(existing)
// The options union requires body whenever resolveBody is absent.
const desired: T = opts.resolveBody ? await opts.resolveBody() : opts.body!
const base: T = {
...desired,
metadata: { ...desired.metadata, resourceVersion: existing.metadata?.resourceVersion },
}
const next = mergeExisting ? mergeExisting(base, existing) : base
if (isUpToDate?.(next, existing)) {
if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
writeSkipsTotal.inc({ kind: next.kind ?? 'unknown' })
return 'up_to_date'
}
if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
try {
await replace(next)
writesTotal.inc({ kind: next.kind ?? 'unknown' })
hccLogger.info('Kubernetes resource updated', { scope: logPrefix, description, attempt })
return 'replaced'
} catch (err) {
if (getErrorCode(err) === 409 && attempt < maxAttempts) {
// Jittered backoff: 25–125ms on attempt 1, 50–250ms on attempt 2.
const baseMs = 25 * attempt
const jitterMs = Math.floor(Math.random() * baseMs * 4)
await new Promise(r => setTimeout(r, baseMs + jitterMs))
continue
}
throw err
}
}
throw new Error(`replaceWithConflictRetry exhausted attempts for ${description}`)
}
function mergeAnnotations(
desired?: Record<string, string>,
existing?: Record<string, string>
): Record<string, string> | undefined {
const merged = { ...(existing ?? {}), ...(desired ?? {}) }
return Object.keys(merged).length > 0 ? merged : undefined
}
/**
* Preserve operator/runtime annotations while keeping controller-authored labels
* and specs authoritative. This intentionally does not preserve arbitrary
* labels because Clerum ownership and cleanup depend on them.
*/
export function preserveObjectAnnotations<
T extends { metadata?: { annotations?: Record<string, string> } },
>(desired: T, existing: T): T {
const annotations = mergeAnnotations(
desired.metadata?.annotations,
existing.metadata?.annotations
)
return {
...desired,
metadata: {
...desired.metadata,
annotations,
},
}
}
/**
* Preserve Deployment annotations at both object and pod-template level.
* Pod-template annotations include operational restart markers such as
* kubectl.kubernetes.io/restartedAt.
*
* This merges every live annotation. GFS applyDeployment does not use it:
* that path keeps object annotations and drops live-only pod-template keys
* other than kubectl.kubernetes.io/restartedAt. Host, SFS, McpServer, and
* LlmHook still merge every template annotation (#698).
*/
export function preserveDeploymentAnnotations<
T extends {
metadata?: { annotations?: Record<string, string> }
spec?: { template?: { metadata?: { annotations?: Record<string, string> } } }
},
>(desired: T, existing: T): T {
const objectPreserved = preserveObjectAnnotations(desired, existing)
const templateAnnotations = mergeAnnotations(
desired.spec?.template?.metadata?.annotations,
existing.spec?.template?.metadata?.annotations
)
return {
...objectPreserved,
spec: {
...objectPreserved.spec,
template: {
...objectPreserved.spec?.template,
metadata: {
...objectPreserved.spec?.template?.metadata,
annotations: templateAnnotations,
},
},
},
}
}
/**
* Preserve immutable Service fields assigned by Kubernetes while keeping the
* desired Service spec otherwise authoritative.
*/
export function preserveServiceAssignedFields<
T extends {
metadata?: { annotations?: Record<string, string> }
spec?: {
clusterIP?: string
clusterIPs?: string[]
ipFamilies?: string[]
ipFamilyPolicy?: string
}
},
>(desired: T, existing: T): T {
const next = preserveObjectAnnotations(desired, existing)
return {
...next,
spec: {
...next.spec,
clusterIP: existing.spec?.clusterIP ?? next.spec?.clusterIP,
clusterIPs: existing.spec?.clusterIPs ?? next.spec?.clusterIPs,
ipFamilies: existing.spec?.ipFamilies ?? next.spec?.ipFamilies,
ipFamilyPolicy: existing.spec?.ipFamilyPolicy ?? next.spec?.ipFamilyPolicy,
},
}
}
/**
* True when the desired PodDisruptionBudget is equivalent to the live object.
* policy/v1 does not default-fill spec fields on this object, so comparison is
* the whole PDB after stripping server-owned metadata. Labels, finalizers, and
* ownerReferences are compared in full (mergeExisting keeps annotations only).
* Doubt or a malformed object returns false (fail-open-to-write).
*/
export function podDisruptionBudgetMatchesDesired(
desired: k8s.V1PodDisruptionBudget | undefined,
existing: k8s.V1PodDisruptionBudget | undefined
): boolean {
try {
if (!desired?.spec || !existing?.spec) return false
return (
JSON.stringify(normalizePodDisruptionBudgetForComparison(desired)) ===
JSON.stringify(normalizePodDisruptionBudgetForComparison(existing))
)
} catch {
return false
}
}
function normalizePodDisruptionBudgetForComparison(pdb: k8s.V1PodDisruptionBudget): unknown {
const normalized = structuredClone(pdb)
stripServerOwnedMetadata(normalized)
return canonicalizeValue(normalized)
}
/**
* True when the merged desired Service is equivalent to the live object, so a
* replace would be a no-op. Canonicalize first: the apiserver default-fills
* type/sessionAffinity/internalTrafficPolicy/protocol and omitted targetPort,
* and key order is not stable (#307). Arrays stay in author order (#214).
* Doubt or a malformed object returns false (fail-open-to-write).
*/
export function serviceMatchesDesired(
desired: k8s.V1Service | undefined,
existing: k8s.V1Service | undefined
): boolean {
try {
if (!desired?.spec || !existing?.spec) return false
return (
JSON.stringify(normalizeServiceForComparison(desired)) ===
JSON.stringify(normalizeServiceForComparison(existing))
)
} catch {
return false
}
}
function stripServerOwnedMetadata(object: {
status?: unknown
metadata?: {
resourceVersion?: unknown
uid?: unknown
generation?: unknown
creationTimestamp?: unknown
managedFields?: unknown
selfLink?: unknown
}
}): void {
delete object.status
delete object.metadata?.resourceVersion
delete object.metadata?.uid
delete object.metadata?.generation
delete object.metadata?.creationTimestamp
delete object.metadata?.managedFields
delete object.metadata?.selfLink
}
function normalizeServiceForComparison(service: k8s.V1Service): unknown {
const normalized = structuredClone(service)
stripServerOwnedMetadata(normalized)
const spec = normalized.spec
if (spec) {
if (spec.type === 'ClusterIP') delete spec.type
if (spec.sessionAffinity === 'None') delete spec.sessionAffinity
if (spec.internalTrafficPolicy === 'Cluster') delete spec.internalTrafficPolicy
for (const port of spec.ports ?? []) {
if (port.protocol === 'TCP') delete port.protocol
if (port.targetPort === undefined) port.targetPort = port.port
}
}
return canonicalizeValue(normalized)
}
/**
* True when the merged desired ConfigMap is equivalent to the live object.
* ConfigMaps have no apps/v1-style spec defaulting, so comparison is the
* whole object after stripping server-owned metadata. Labels, finalizers,
* and ownerReferences are compared in full (mergeExisting keeps annotations
* only), so a foreign stamp there is permanent write churn, not a skip.
* Doubt or a malformed object returns false (fail-open-to-write).
*/
export function configMapMatchesDesired(
desired: k8s.V1ConfigMap | undefined,
existing: k8s.V1ConfigMap | undefined
): boolean {
try {
if (!desired || !existing) return false
return (
JSON.stringify(normalizeConfigMapForComparison(desired)) ===
JSON.stringify(normalizeConfigMapForComparison(existing))
)
} catch {
return false
}
}
function normalizeConfigMapForComparison(configMap: k8s.V1ConfigMap): unknown {
const normalized = structuredClone(configMap)
stripServerOwnedMetadata(normalized)
return canonicalizeValue(normalized)
}
/**
* True when the desired NetworkPolicy is equivalent to the live object, so a
* replace would be a no-op. Canonicalize first: the apiserver default-fills
* policyTypes and ports[].protocol=TCP, and key order is not stable (#307).
* Arrays stay in author order (#214). Doubt or a malformed object returns
* false (fail-open-to-write). Distinct from `sameNetworkPolicySpec` (spec-only
* safety-inventory filter, no default-fill) and `egressSignature` (#299
* no-churn fingerprint). Do not unify those predicates here (#473).
*/
export function networkPolicyMatchesDesired(
desired: k8s.V1NetworkPolicy | undefined,
existing: k8s.V1NetworkPolicy | undefined
): boolean {
try {
if (!desired?.spec || !existing?.spec) return false
return (
JSON.stringify(normalizeNetworkPolicyForComparison(desired)) ===
JSON.stringify(normalizeNetworkPolicyForComparison(existing))
)
} catch {
return false
}
}
function normalizeNetworkPolicyForComparison(policy: k8s.V1NetworkPolicy): unknown {
const normalized = structuredClone(policy) as k8s.V1NetworkPolicy & { status?: unknown }
stripServerOwnedMetadata(normalized)
const spec = normalized.spec
if (spec) {
// Kubernetes SetDefaults_NetworkPolicy fills when PolicyTypes is omitted or empty.
if (!spec.policyTypes?.length) {
spec.policyTypes = (spec.egress?.length ?? 0) > 0 ? ['Ingress', 'Egress'] : ['Ingress']
}
for (const rule of [...(spec.ingress ?? []), ...(spec.egress ?? [])]) {
for (const port of rule.ports ?? []) {
if (port.protocol === 'TCP') delete port.protocol
}
}
// Empty slices persist the same as an omitted field for the apiserver.
if (spec.ingress?.length === 0) delete spec.ingress
if (spec.egress?.length === 0) delete spec.egress
}
return canonicalizeValue(normalized)
}
/**
* True when the merged desired Deployment is equivalent to the live object.
* Kubernetes persists server metadata and defaulted Deployment/PodSpec fields
* that HCC does not author. Remove only those known fields before comparing;
* every HCC-authored field stays exact so an intentional removal or change
* still causes a rollout.
*
* Safe-strip lemma: a normalizer mutation of the form "delete field X iff
* X === documented-default, applied identically to both sides" cannot hide
* any real change for any consumer. Anything outside that form — unconditional
* deletes, value normalisation, per-writer branches — can hide changes and is
* forbidden. Keep this allowlist conservative: an unknown admission mutation
* must compare as drift rather than be silently ignored.
*
* Pod-template annotations are merged by preserveDeploymentAnnotations before
* this comparison. Labels, finalizers, and ownerReferences are compared in
* full: a third-party stamp on those fields is drift and keeps the gate
* writing (fail-open), because mergeExisting does not absorb them.
* Doubt or a malformed object returns false
* (fail-open-to-write), matching serviceMatchesDesired / networkPolicyMatchesDesired.
*/
export function deploymentMatchesDesired(
desired: k8s.V1Deployment | undefined,
existing: k8s.V1Deployment | undefined
): boolean {
try {
if (!desired?.spec || !existing?.spec) return false
return (
JSON.stringify(normalizeDeploymentForComparison(desired)) ===
JSON.stringify(normalizeDeploymentForComparison(existing))
)
} catch {
return false
}
}
export function normalizeDeploymentForComparison(deployment: k8s.V1Deployment): unknown {
const normalized = structuredClone(deployment)
stripServerOwnedMetadata(normalized)
const spec = normalized.spec
if (spec) {
if (spec.progressDeadlineSeconds === 600) delete spec.progressDeadlineSeconds
if (spec.revisionHistoryLimit === 10) delete spec.revisionHistoryLimit
if (spec.minReadySeconds === 0) delete spec.minReadySeconds
if (spec.paused === false) delete spec.paused
if (
spec.strategy?.type === 'RollingUpdate' &&
spec.strategy.rollingUpdate?.maxSurge === '25%' &&
spec.strategy.rollingUpdate.maxUnavailable === '25%'
) {
delete spec.strategy
}
const template = spec.template
if (template) {
delete template.metadata?.creationTimestamp
const podSpec = template.spec
if (podSpec) {
if (podSpec.restartPolicy === 'Always') delete podSpec.restartPolicy
if (podSpec.dnsPolicy === 'ClusterFirst') delete podSpec.dnsPolicy
if (podSpec.schedulerName === 'default-scheduler') delete podSpec.schedulerName
if (podSpec.terminationGracePeriodSeconds === 30)
delete podSpec.terminationGracePeriodSeconds
if (podSpec.enableServiceLinks === true) delete podSpec.enableServiceLinks
if (podSpec.preemptionPolicy === 'PreemptLowerPriority') delete podSpec.preemptionPolicy
// Cross-field default: kube mirrors serviceAccount from serviceAccountName.
// Delete the deprecated alias first; then strip SA === 'default'. Swapping
// these two lines would leave a dangling alias and force a write every pass.
if (podSpec.serviceAccount === podSpec.serviceAccountName) delete podSpec.serviceAccount
if (podSpec.serviceAccountName === 'default') delete podSpec.serviceAccountName
if (Object.keys(podSpec.securityContext ?? {}).length === 0) delete podSpec.securityContext
for (const container of [
...(podSpec.initContainers ?? []),
...(podSpec.containers ?? []),
]) {
normalizeContainerDefaults(container)
}
for (const volume of podSpec.volumes ?? []) normalizeVolumeDefaults(volume)
}
}
}
return canonicalizeValue(normalized)
}
export function normalizeContainerDefaults(container: k8s.V1Container): void {
if (Object.keys(container.resources ?? {}).length === 0) delete container.resources
if (container.terminationMessagePath === '/dev/termination-log') {
delete container.terminationMessagePath
}
if (container.terminationMessagePolicy === 'File') delete container.terminationMessagePolicy
for (const probe of [container.startupProbe, container.livenessProbe, container.readinessProbe]) {
if (!probe) continue
if (probe.initialDelaySeconds === 0) delete probe.initialDelaySeconds
if (probe.timeoutSeconds === 1) delete probe.timeoutSeconds
if (probe.successThreshold === 1) delete probe.successThreshold
if (probe.failureThreshold === 3) delete probe.failureThreshold
if (probe.httpGet?.scheme === 'HTTP') delete probe.httpGet.scheme
}
for (const env of container.env ?? []) {
if (env.value === '') delete env.value
if (env.valueFrom?.fieldRef?.apiVersion === 'v1') {
delete env.valueFrom.fieldRef.apiVersion
}
}
}
export function normalizeVolumeDefaults(volume: k8s.V1Volume): void {
if (volume.secret?.defaultMode === 420) delete volume.secret.defaultMode
if (volume.secret?.optional === false) delete volume.secret.optional
if (volume.configMap?.defaultMode === 420) delete volume.configMap.defaultMode
if (volume.configMap?.optional === false) delete volume.configMap.optional
if (volume.downwardAPI?.defaultMode === 420) delete volume.downwardAPI.defaultMode
if (volume.projected?.defaultMode === 420) delete volume.projected.defaultMode
if (volume.persistentVolumeClaim?.readOnly === false) {
delete volume.persistentVolumeClaim.readOnly
}
}
/** Read-first create or converge a NetworkPolicy with the existing retry policy. */
export async function applyNetworkPolicy(
api: k8s.NetworkingV1Api,
name: string,
namespace: string,
policy: k8s.V1NetworkPolicy,
logPrefix = '[NetPol]',
mutationAllowed?: () => boolean,
validateExisting?: (existing: k8s.V1NetworkPolicy) => void,
missingIsError = false,
existing?: k8s.V1NetworkPolicy | null
): Promise<ResourceApplyResult> {
return ensureResource<k8s.V1NetworkPolicy>({
existing,
mutationAllowed,
read: () =>
observeExistenceRead('NetworkPolicy', () =>
api.readNamespacedNetworkPolicy({ name, namespace })
),
create: async () => {
await observeCreate('NetworkPolicy', () =>
api.createNamespacedNetworkPolicy({ namespace, body: policy })
)
hccLogger.info('NetworkPolicy created', { scope: logPrefix, policy: name, namespace })
},
onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),
converge: read =>
replaceWithConflictRetry<k8s.V1NetworkPolicy>({
description: `policy "${name}" in ${namespace}`,
logPrefix,
body: policy,
read,
replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),
mutationAllowed,
validateExisting,
missingIsError,
isUpToDate: networkPolicyMatchesDesired,
}),
})
}
/**
* Recursive key-sort used by the no-op gates (Role, Service, Deployment,
* NetworkPolicy apply). Arrays keep author order. Undefined-valued keys are
* dropped. A second copy of this walk is a #307-class drift: the skip would
* silently stop matching.
*
* Distinct from `canonicalStringify` (top-level Secret key list) and from the
* egressSignature walker in networkPolicyReconciler, which keeps undefined
* keys. Do not unify those (#473 / #299).
*/
export function canonicalizeValue(value: unknown): unknown {
if (Array.isArray(value)) return value.map(canonicalizeValue)
if (typeof value !== 'object' || value === null) return value
const canonical: Record<string, unknown> = {}
for (const key of Object.keys(value).sort()) {
const entry = (value as Record<string, unknown>)[key]
if (entry !== undefined) canonical[key] = canonicalizeValue(entry)
}
return canonical
}
/**
* Stable JSON for hashing: object keys are emitted in sorted order, so the
* digest of a Secret's data depends on its content and not on the order the
* API server happened to return the keys in.
*
* Shared by the Host channel-reader credentials revision and the McpServer
* connector credentials revision (issue #223): both hash Secret content into a
* pod-template annotation, and a divergence between the two canonical forms
* would be invisible until a rollout silently stopped happening.
*/
export function canonicalStringify(obj: Record<string, unknown>): string {
const sorted = Object.keys(obj).sort()
return JSON.stringify(sorted.map(k => [k, obj[k]]))
}