:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Attributes } from '@opentelemetry/api'
import { Counter, Gauge } from 'prom-client'
import { instrumentFn, setSpanAttributes, withTracingSpan } from '~/common/tracing/tracing-utils'
import { defaultConfig } from '../config/config'
import { logger } from './logger'
import { sleep } from './utils'
const lazyLoaderCacheHits = new Counter({
name: 'lazy_loader_cache_hits',
help: 'The number of times we have hit the cache',
labelNames: ['name', 'hit'],
})
const lazyLoaderFullCacheHits = new Counter({
name: 'lazy_loader_full_cache_hits',
help: 'The number of times we have hit the cache for all keys',
labelNames: ['name', 'hit'],
})
const lazyLoaderBufferUsage = new Counter({
name: 'lazy_loader_buffer_usage',
help: 'The number of times we have used the buffer indicating better batching',
labelNames: ['name', 'hit'],
})
const lazyLoaderQueuedCacheHits = new Counter({
name: 'lazy_loader_queued_cache_hits',
help: 'The number of times we have hit the cached loading promise for a key',
labelNames: ['name', 'hit'],
})
const lazyLoaderCacheSize = new Gauge({
name: 'lazy_loader_cache_size',
help: 'Current number of entries in the cache',
labelNames: ['name'],
})
/**
* We have a common pattern across consumers where we want to:
* - Load a value lazily
* - Minimize queries to the DB for multiple values (e.g. teams for events)
* - Keep that value cached ensuring any caller to retrieve it will get the value
* - "Refresh" the value after a certain age
* - "Drop" the value after a much longer age
*
* Follow up improvements:
* - Soft and hard refresh times - if soft it can be refreshed in the background, non blocking
* - Parallel loading defense - multiple calls for the same value in parallel only loads once
*/
/**
* Retry policy for the loader. When set, a loader call that throws a retriable error
* (`error.isRetriable === true`) is retried until `maxElapsedMs` is exhausted. Useful for absorbing
* transient dependency blips (e.g. a Postgres pooler scale-down) so a single failed background load
* doesn't propagate to every caller — including fire-and-forget ones, where it would crash the process.
*/
export type LoaderRetryOptions = {
/** Base delay between attempts. */
retryIntervalMs: number
/** Random extra delay in `[0, retryJitterMs)` added to each interval to avoid a thundering herd. */
retryJitterMs?: number
/** Total time budget for retrying. Once exceeded, the last error is rethrown. */
maxElapsedMs: number
}
/**
* Shared retry policy for loaders backed by transient-blip-prone dependencies (e.g. Postgres
* behind PgBouncer). Sized to ride out a pooler restart without stalling callers for long.
*/
export const DEFAULT_LOADER_RETRY: LoaderRetryOptions = {
retryIntervalMs: 250,
retryJitterMs: 250,
maxElapsedMs: 5000,
}
export type LazyLoaderOptions<T> = {
name: string
/** Function to load the values */
loader: (key: string[]) => Promise<Record<string, T | null | undefined>>
/** Retry policy for transient loader failures. Off by default (no retry). */
loaderRetry?: LoaderRetryOptions
/** How long to cache the value */
refreshAgeMs?: number
/** How long to cache null values */
refreshNullAgeMs?: number
/** How long to cache the value before refreshing in the background - must be smaller than refreshAgeMs */
refreshBackgroundAgeMs?: number
/** How much jitter to add to the refresh time */
refreshJitterMs?: number
/** How long to buffer loads for - if set to 0 then it will load immediately without buffering */
bufferMs?: number
/** Maximum number of entries in the cache - LRU eviction when exceeded */
maxSize?: number
}
export type LoadOptions = {
/**
* Issue the buffered load now instead of waiting out `bufferMs`. For a caller that already
* holds the whole batch, such as a chunk prefetch, the buffer only delays the query.
*/
flush?: boolean
}
type LazyLoaderMap<T> = Record<string, T | null | undefined>
/**
* How a `loadViaCache` call was served: every key from the cache, by invoking the loader, or by
* waiting on a load another caller already had in flight.
*/
export type LazyLoaderSpanOutcome = 'loaded' | 'waited_pending'
/**
* A cached value together with the deadlines that govern it. These live on one object so that
* writing a value and stamping its deadlines cannot come apart: any code path that sets the
* value has to produce the deadlines too.
*/
type CacheEntry<T> = {
value: T | null
lastUsed: number
/** Past this, the next lookup reloads before returning. */
cacheUntil: number
/** Past this, the next lookup returns the cached value and reloads in the background. */
backgroundRefreshAfter?: number
}
export class LazyLoader<T> {
private cache: Record<string, CacheEntry<T> | undefined>
private pendingLoads: Record<string, Promise<void> | undefined>
private refreshAgeMs: number
private refreshNullAgeMs: number
private refreshBackgroundAgeMs?: number
private refreshJitterMs: number
private maxSize: number
private cacheSize: number = 0
private buffer:
| {
keys: Set<string>
promise: Promise<void>
/** Fires the buffered load now; a no-op once the buffer timer has fired. */
flush: () => void
}
| undefined
constructor(private readonly options: LazyLoaderOptions<T>) {
// Keys come from callers and can be ingest tokens or distinct ids, so these maps have no
// prototype. On a plain object a key like `__proto__` resolves to `Object.prototype`
// instead of missing, and writing through it mutates a builtin the whole process shares.
this.cache = Object.create(null)
this.pendingLoads = Object.create(null)
this.refreshAgeMs = this.options.refreshAgeMs ?? 1000 * 60 * 5 // 5 minutes
this.refreshNullAgeMs = this.options.refreshNullAgeMs ?? this.refreshAgeMs
this.refreshBackgroundAgeMs = this.options.refreshBackgroundAgeMs
this.refreshJitterMs = this.options.refreshJitterMs ?? this.refreshAgeMs / 5
this.maxSize = this.options.maxSize ?? defaultConfig.LAZY_LOADER_MAX_SIZE
if (this.refreshBackgroundAgeMs && this.refreshBackgroundAgeMs > this.refreshAgeMs) {
throw new Error('refreshBackgroundAgeMs must be smaller than refreshAgeMs')
}
}
/** A snapshot of the cached values. Mutating the result does not affect the cache. */
public getCache(): Record<string, T | null> {
const values: Record<string, T | null> = Object.create(null)
for (const [key, entry] of Object.entries(this.cache)) {
if (entry) {
values[key] = entry.value
}
}
return values
}
public async get(key: string): Promise<T | null> {
const loaded = await this.loadViaCache([key])
return loaded[key] ?? null
}
public async getMany(keys: string[], options?: LoadOptions): Promise<Record<string, T | null>> {
return await this.loadViaCache(keys, options)
}
public markForRefresh(key: string | string[]): void {
for (const k of Array.isArray(key) ? key : [key]) {
const entry = this.cache[k]
if (entry) {
// A cacheUntil of 0 forces the next lookup to reload before returning. The
// background deadline goes with it so the entry never holds two that disagree.
entry.cacheUntil = 0
entry.backgroundRefreshAfter = undefined
}
}
}
public clear(): void {
this.cache = Object.create(null)
this.cacheSize = 0
// this.pendingLoads = {} // NOTE: We don't clear this
this.updateCacheSizeMetric()
}
private setValues(map: LazyLoaderMap<T>): void {
const now = Date.now()
const keys = Object.keys(map)
for (const key of keys) {
const value = map[key] ?? null
const jitter = Math.floor(Math.random() * this.refreshJitterMs)
const refreshAge = value === null ? this.refreshNullAgeMs : this.refreshAgeMs
const cacheUntil = now + refreshAge + jitter
// A null takes refreshNullAgeMs for both deadlines, putting them on the same instant,
// so nulls hard-refresh and never background-refresh.
const backgroundRefreshAfter = this.refreshBackgroundAgeMs
? now + (value === null ? this.refreshNullAgeMs : this.refreshBackgroundAgeMs) + jitter
: undefined
if (this.cache[key] === undefined) {
this.cacheSize++
}
this.cache[key] = { value, lastUsed: now, cacheUntil, backgroundRefreshAfter }
}
this.evictLRU()
this.updateCacheSizeMetric()
}
/**
* Ensure that a range of values are preloaded and cached.
*
* If already cached, the lastUsed value is updated to now
* If not cached, the value is loaded as part of the batch and added to the cache.
* If the value is older than the refreshAge, it is loaded from the database.
*/
private async loadViaCache(keys: string[], options?: LoadOptions): Promise<Record<string, T | null>> {
// No prototype, for the same reason as the cache: keys are caller-supplied, and this
// object is handed back to callers who may iterate or spread it.
const results: Record<string, T | null> = Object.create(null)
const keysToLoad = new Set<string>()
for (const key of keys) {
const cached = this.cache[key]
if (cached !== undefined) {
results[key] = cached.value
cached.lastUsed = Date.now()
const cacheUntil = cached.cacheUntil
const backgroundRefreshAfter = cached.backgroundRefreshAfter
if (Date.now() > cacheUntil) {
keysToLoad.add(key)
lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()
continue
}
if (backgroundRefreshAfter && Date.now() > backgroundRefreshAfter) {
void this.load([key]).catch((err) => {
logger.warn(`[LazyLoader:${this.options.name}] Background refresh failed`, {
key,
error: String(err),
})
})
lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit_background' }).inc()
continue
}
} else {
keysToLoad.add(key)
lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()
continue
}
lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()
}
if (keysToLoad.size === 0) {
lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()
return results
}
lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()
// Only a miss is instrumented: hits would outnumber every other span in a trace, and on a hit the
// instrumentation costs more than the lookup.
return await instrumentFn({ key: `lazyLoader.loadViaCache`, tag: this.options.name, span: false }, () =>
withTracingSpan(
'instrumented_function',
'lazyLoader.loadViaCache',
{ tag: this.options.name },
async () => {
// A span that only waits on another caller's in-flight load has no query child of its
// own, so record the distinction here or it looks like a slow cache lookup.
const allPending = Array.from(keysToLoad).every((key) => this.pendingLoads[key] !== undefined)
setSpanAttributes(this.spanAttributes(keys, keysToLoad, allPending ? 'waited_pending' : 'loaded'))
await this.load(Array.from(keysToLoad), options)
for (const key of keys) {
results[key] = this.cache[key]?.value ?? null
}
return results
}
)
)
}
private spanAttributes(keys: string[], keysToLoad: Set<string>, outcome: LazyLoaderSpanOutcome): Attributes {
return {
'lazyloader.name': this.options.name,
'lazyloader.keys': keys.length,
'lazyloader.misses': keysToLoad.size,
'lazyloader.outcome': outcome,
}
}
/**
* Schedules the keys to be loaded with a buffer to allow batching multiple keys
* This is somewhat complex but simplifies the usage around the codebase as you can safely do multiple gets without worrying about firing off duplicate DB requests
*
* Loaded values land in `this.cache` via `setValues`; callers read them back from there.
*/
private async load(keys: string[], options?: LoadOptions): Promise<void> {
const bufferMs = this.options.bufferMs ?? defaultConfig.LAZY_LOADER_DEFAULT_BUFFER_MS
const keyPromises: Promise<void>[] = []
for (const key of keys) {
let pendingLoad = this.pendingLoads[key]
if (pendingLoad) {
keyPromises.push(pendingLoad)
lazyLoaderQueuedCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()
continue
}
lazyLoaderQueuedCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()
if (!this.buffer) {
this.buffer = this.createBuffer(bufferMs)
lazyLoaderBufferUsage.labels({ name: this.options.name, hit: 'miss' }).inc()
} else {
lazyLoaderBufferUsage.labels({ name: this.options.name, hit: 'hit' }).inc()
}
// The values land in the cache via setValues, so callers read them from there.
this.buffer.keys.add(key)
pendingLoad = this.buffer.promise.finally(() => {
delete this.pendingLoads[key]
})
this.pendingLoads[key] = pendingLoad
keyPromises.push(pendingLoad)
}
if (options?.flush) {
// Keys other callers buffered meanwhile ride along; they only get served sooner.
this.buffer?.flush()
}
await Promise.all(keyPromises)
}
/**
* A buffer collects keys until its timer fires, or until `flush` fires it early, then calls
* the loader once for all of them. Whichever fires first wins; the other is a no-op.
*/
private createBuffer(bufferMs: number): NonNullable<typeof this.buffer> {
const keys = new Set<string>()
let fired = false
let fire: () => void = () => {}
const promise = new Promise<string[]>((resolve) => {
const timer = setTimeout(() => fire(), bufferMs)
fire = () => {
if (fired) {
return
}
fired = true
clearTimeout(timer)
this.buffer = undefined
resolve(Array.from(keys))
}
}).then(async (bufferedKeys) => {
logger.debug('[LazyLoader]', this.options.name, 'Loading: ', bufferedKeys)
this.setValues(await this.invokeLoader(bufferedKeys))
})
return { keys, promise, flush: () => fire() }
}
/**
* Invoke the loader and normalize its result into a map this class owns: no prototype, an entry
* for every requested key, and `undefined` collapsed to `null`. Keys the loader returns beyond
* those requested are kept, so a loader can warm the cache with extras. The loader hands back a
* plain object, where a key like `__proto__` resolves up the prototype chain instead of reading
* as absent.
*/
private async invokeLoader(keys: string[]): Promise<Record<string, T | null>> {
const loaded = await this.invokeLoaderWithRetry(keys)
const map: Record<string, T | null> = Object.create(null)
// A key the loader omits means "no value", not "no answer", so seed every requested key
// before overlaying what came back.
for (const key of keys) {
map[key] = null
}
for (const [key, value] of Object.entries(loaded)) {
map[key] = value ?? null
}
return map
}
/**
* Invoke the loader, optionally retrying transient failures.
*
* Each attempt re-invokes `loader(keys)` so the ids/tokens are re-evaluated against the source at
* retry time rather than reusing a stale result. Only retriable errors (`error.isRetriable === true`)
* are retried; anything else is rethrown immediately so genuine bugs surface rather than being masked.
*/
private async invokeLoaderWithRetry(keys: string[]): Promise<LazyLoaderMap<T>> {
const retry = this.options.loaderRetry
if (!retry) {
return await this.options.loader(keys)
}
const deadline = performance.now() + retry.maxElapsedMs
let attempt = 0
for (;;) {
try {
return await this.options.loader(keys)
} catch (error) {
attempt++
if (error?.isRetriable !== true) {
// Surface non-transient errors instead of masking them with retries.
throw error
}
if (performance.now() >= deadline) {
logger.warn('🔁', `[LazyLoader:${this.options.name}] Loader retries exhausted, giving up`, {
attempt,
keys: keys.length,
error: String(error),
})
throw error
}
const jitter = retry.retryJitterMs ? Math.floor(Math.random() * retry.retryJitterMs) : 0
logger.warn('🔁', `[LazyLoader:${this.options.name}] Loader failed, retrying`, {
attempt,
keys: keys.length,
error: String(error),
})
await sleep(retry.retryIntervalMs + jitter)
}
}
}
private evictLRU(): void {
if (this.cacheSize <= this.maxSize) {
return
}
// Evict extra headroom so we don't re-sort on every subsequent insert.
// Apply headroom only to caches large enough to benefit from fewer sorts.
const headroom = this.maxSize > 100 ? Math.ceil(this.maxSize * 0.1) : 0
const toEvict = this.cacheSize - this.maxSize + headroom
const cacheKeys = Object.keys(this.cache)
cacheKeys.sort((a, b) => (this.cache[a]?.lastUsed ?? 0) - (this.cache[b]?.lastUsed ?? 0))
const evictCount = Math.min(toEvict, cacheKeys.length)
for (let i = 0; i < evictCount; i++) {
delete this.cache[cacheKeys[i]]
this.cacheSize--
}
}
private updateCacheSizeMetric(): void {
lazyLoaderCacheSize.labels({ name: this.options.name }).set(this.cacheSize)
}
}