Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import { Router } from 'express';
import { timingSafeEqual } from 'crypto';
import { Registry, collectDefaultMetrics, Counter, Gauge, Histogram } from 'prom-client';
import { logger, setAgentEventActorReceiptMetricObserver } from '@librechat/data-schemas';
import type { Request, Response, NextFunction, RequestHandler } from 'express';
import type { Mongoose } from 'mongoose';
import type { AgentStartupMilestone, AgentStartupResult } from '~/agents/phases';
import type { LocatorTraversalFailure } from '../protection/diagnostics';
import { agentStartupMilestones, agentStartupResults } from '~/agents/phases';
const PATH_NORMALIZATIONS: [RegExp, string][] = [
[/^(\/api\/insights(?:\/access)?)\/?$/, '$1'],
[/^\/api\/agents\/chat\/stream\/[^/]+(?=\/|$)/, '/api/agents/chat/stream/#id'],
[/^\/api\/agents\/chat\/status\/[^/]+(?=\/|$)/, '/api/agents/chat/status/#id'],
[/^\/api\/files\/code\/download\/[^/]+\/[^/]+(?=\/|$)/, '/api/files/code/download/#id/#id'],
[/^\/api\/files\/download-url\/[^/]+\/[^/]+(?=\/|$)/, '/api/files/download-url/#id/#id'],
[/^\/api\/files\/download\/[^/]+\/[^/]+(?=\/|$)/, '/api/files/download/#id/#id'],
[/^\/api\/files\/[^/]+\/preview(?=\/|$)/, '/api/files/#id/preview'],
[/^\/api\/skills\/[^/]+\/files(?:\/.*)?(?=\/|$)/, '/api/skills/#id/files'],
[/^\/api\/messages\/artifact\/[^/]+(?=\/|$)/, '/api/messages/artifact/#id'],
[/^\/api\/messages\/[^/]+\/[^/]+(?=\/|$)/, '/api/messages/#id/#id'],
[/^\/api\/convos\/[^/]+\/messages\/[^/]+(?=\/|$)/, '/api/convos/#id/messages/#id'],
[/^\/api\/messages\/[^/]+(?=\/|$)/, '/api/messages/#id'],
[/^\/api\/convos\/[^/]+(?=\/|$)/, '/api/convos/#id'],
[/^\/api\/files\/[^/]+(?=\/|$)/, '/api/files/#id'],
[/^\/api\/agents\/[^/]+(?=\/|$)/, '/api/agents/#id'],
[/^\/api\/assistants\/[^/]+(?=\/|$)/, '/api/assistants/#id'],
[/^\/api\/share\/[^/]+(?=\/|$)/, '/api/share/#token'],
[/^\/share\/[^/]+(?=\/|$)/, '/share/#id'],
[/^\/api\/(tags|tools|runs|sessions)\/[0-9a-f]{24}(?=\/|$)/i, '/api/$1/#id'],
[
/^\/api\/(tools|sessions)\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?=\/|$)/i,
'/api/$1/#id',
],
];
const STATIC_PATHS = new Set([
'/',
'/health',
'/metrics',
'/api/auth/login',
'/api/config',
'/api/insights',
'/api/insights/access',
'/api/agents/chat/abort',
'/api/agents/chat/active',
'/api/agents/v1/chat/completions',
'/api/agents/v1/responses',
'/api/files',
'/api/files/config',
'/api/files/images',
'/api/files/images/avatar',
'/api/files/speech/stt',
]);
const UPLOAD_PATHS = new Set([
'/api/files',
'/api/files/images',
'/api/files/images/avatar',
'/api/files/speech/stt',
'/api/skills/#id/files',
]);
const UPLOAD_METHODS = new Set(['POST', 'PUT', 'PATCH']);
const LOW_CARDINALITY_PATHS: RegExp[] = [
/^\/api\/agents\/chat\/stream\/#id$/,
/^\/api\/agents\/chat\/status\/#id$/,
/^\/api\/files\/#id\/preview$/,
/^\/api\/files\/(code\/download|download-url|download)\/#id\/#id$/,
/^\/api\/skills\/#id\/files$/,
/^\/api\/messages\/#id$/,
/^\/api\/messages\/#id\/#id$/,
/^\/api\/messages\/artifact\/#id$/,
/^\/api\/convos\/#id$/,
/^\/api\/convos\/#id\/messages\/#id$/,
/^\/api\/(files|agents|assistants|tags|tools|runs|sessions)\/#id$/,
/^\/api\/share\/#token$/,
/^\/share\/#id(?:\/edit)?$/,
];
const isLowCardinalityPath = (path: string): boolean =>
STATIC_PATHS.has(path) || LOW_CARDINALITY_PATHS.some((pattern) => pattern.test(path));
const normalizeKnownPath = (path: string): string => {
for (const [pattern, replacement] of PATH_NORMALIZATIONS) {
if (pattern.test(path)) {
return path.replace(pattern, replacement);
}
}
return path;
};
const normalizeUnknownPath = (path: string): string => {
if (STATIC_PATHS.has(path)) {
return path;
}
if (path === '/api' || path.startsWith('/api/')) {
return '/api/#path';
}
if (path === '/images' || path.startsWith('/images/')) {
return '/images/#path';
}
if (path === '/avatars' || path.startsWith('/avatars/')) {
return '/avatars/#path';
}
if (path === '/t' || path.startsWith('/t/')) {
return '/t/#path';
}
return '/#path';
};
export const normalizePath = (rawPath: string): string => {
const [pathWithoutQuery] = rawPath.split('?');
const path = pathWithoutQuery.startsWith('/') ? pathWithoutQuery : `/${pathWithoutQuery}`;
const normalized = normalizeKnownPath(path || '/');
if (isLowCardinalityPath(normalized)) {
return normalized;
}
return normalizeUnknownPath(path);
};
export interface PrometheusMetrics {
metricsMiddleware: (req: Request, res: Response, next: NextFunction) => void;
metricsRouter: Router;
}
export interface AgentEventActorStorageMetricsSnapshot {
retainedByResolution: Record<
'checkpoint_verified' | 'action_compensated' | 'history_repaired',
number
>;
expiryEligible: number;
retryDeliveries: number;
deadDeliveries: number;
pendingReconciliations: number;
oldestPendingAgeSeconds: number;
}
export interface MetricsOptions {
collectAgentEventActorStorageMetrics?: () => Promise<AgentEventActorStorageMetricsSnapshot>;
}
const AGENT_EVENT_ACTOR_STORAGE_METRICS_CACHE_MS = 60_000;
export type OpenIDUserLookupResult = 'found' | 'not_found' | 'migration' | 'auth_failed' | 'error';
export type GenerationJobStore = 'memory' | 'redis';
export type GenerationJobResult = 'created' | 'completed' | 'error' | 'aborted' | 'abort_failed';
export type GenerationStreamSubscriptionType = 'initial' | 'resume' | 'resume_state';
export type GenerationStreamSubscriptionResult =
| 'success'
| 'not_found'
| 'error'
| 'found'
| 'missing';
export type GenerationStreamRecoveryMethod = 'redis' | 'snapshot';
export type GenerationStreamRecoveryOutcome = 'success' | 'failed' | 'not_required';
export type GenerationStreamAttachmentOutcome =
| 'attached'
| 'bootstrap_slow'
| 'disconnected'
| 'never_attached';
export type RumProxyEndpoint = 'traces' | 'logs' | 'unknown';
export type RumProxyResult =
| 'success'
| 'replay_disabled'
| 'auth_drop'
| 'auth_error'
| 'bad_request'
| 'rate_limited'
| 'unsupported_media_type'
| 'not_configured'
| 'collector_4xx'
| 'collector_5xx'
| 'collector_error'
| 'collector_timeout';
export type ShareLinkOperation = 'create' | 'update';
export type ShareLinkRejectionCode = 'TARGET_MESSAGE_NOT_FOUND' | 'NO_MESSAGES';
export type RedisClient = 'ioredis' | 'keyv';
export type RedisOperationStatus = 'success' | 'error';
type OpenIDUserLookupMetrics = {
recordLookup: (result: OpenIDUserLookupResult, durationSeconds: number) => void;
};
let openIDUserLookupMetrics: OpenIDUserLookupMetrics = {
recordLookup: () => undefined,
};
export function recordOpenIDUserLookup(
result: OpenIDUserLookupResult,
durationSeconds: number,
): void {
openIDUserLookupMetrics.recordLookup(result, durationSeconds);
}
type MongooseQueryMetrics = {
recordQuery: (model: string, operation: string, status: string, durationSeconds: number) => void;
};
let mongooseQueryMetrics: MongooseQueryMetrics = {
recordQuery: () => undefined,
};
type GenerationJobMetrics = {
recordJob: (store: GenerationJobStore, result: GenerationJobResult) => void;
setJobsInFlight: (store: GenerationJobStore, count: number) => void;
recordSubscription: (
store: GenerationJobStore,
type: GenerationStreamSubscriptionType,
result: GenerationStreamSubscriptionResult,
) => void;
recordResumePendingEvents: (store: GenerationJobStore, count: number) => void;
recordEarlyBufferOverflow: (store: GenerationJobStore) => void;
recordRecovery: (
store: GenerationJobStore,
method: GenerationStreamRecoveryMethod,
outcome: GenerationStreamRecoveryOutcome,
durationSeconds: number,
reconstructedEvents: number,
reconstructedContent: number,
) => void;
recordAttachment: (
store: GenerationJobStore,
outcome: GenerationStreamAttachmentOutcome,
delaySeconds?: number,
) => void;
};
let generationJobMetrics: GenerationJobMetrics = {
recordJob: () => undefined,
setJobsInFlight: () => undefined,
recordSubscription: () => undefined,
recordResumePendingEvents: () => undefined,
recordEarlyBufferOverflow: () => undefined,
recordRecovery: () => undefined,
recordAttachment: () => undefined,
};
type AgentStartupMetrics = {
recordMilestone: (milestone: AgentStartupMilestone, durationSeconds: number) => void;
recordResult: (result: AgentStartupResult) => void;
};
const agentStartupMilestoneSet = new Set<string>(agentStartupMilestones);
const agentStartupResultSet = new Set<string>(agentStartupResults);
let agentStartupMetrics: AgentStartupMetrics = {
recordMilestone: () => undefined,
recordResult: () => undefined,
};
type RumProxyMetrics = {
recordRequest: (endpoint: RumProxyEndpoint, result: RumProxyResult) => void;
};
let rumProxyMetrics: RumProxyMetrics = {
recordRequest: () => undefined,
};
type ShareLinkMetrics = {
recordRejection: (operation: ShareLinkOperation, code: ShareLinkRejectionCode) => void;
};
let shareLinkMetrics: ShareLinkMetrics = {
recordRejection: () => undefined,
};
type RedisOperationMetrics = {
recordOperation: (
client: RedisClient,
useCase: string,
operation: string,
status: RedisOperationStatus,
durationSeconds: number,
) => void;
};
let redisOperationMetrics: RedisOperationMetrics = {
recordOperation: () => undefined,
};
let observeLocatorTraversal: (failure: LocatorTraversalFailure) => void = () => undefined;
/** Application sink supplied explicitly to content inspection callers. */
export function reportLocatorTraversalFailure(failure: LocatorTraversalFailure): void {
logger.warn(`[content-filter] Locator traversal incomplete ${JSON.stringify(failure)}`, failure);
observeLocatorTraversal(failure);
}
const resetMetricRecorders = (): void => {
observeLocatorTraversal = () => undefined;
openIDUserLookupMetrics = {
recordLookup: () => undefined,
};
mongooseQueryMetrics = {
recordQuery: () => undefined,
};
generationJobMetrics = {
recordJob: () => undefined,
setJobsInFlight: () => undefined,
recordSubscription: () => undefined,
recordResumePendingEvents: () => undefined,
recordEarlyBufferOverflow: () => undefined,
recordRecovery: () => undefined,
recordAttachment: () => undefined,
};
agentStartupMetrics = {
recordMilestone: () => undefined,
recordResult: () => undefined,
};
rumProxyMetrics = {
recordRequest: () => undefined,
};
shareLinkMetrics = {
recordRejection: () => undefined,
};
redisOperationMetrics = {
recordOperation: () => undefined,
};
setAgentEventActorReceiptMetricObserver();
};
export function recordGenerationJob(store: GenerationJobStore, result: GenerationJobResult): void {
generationJobMetrics.recordJob(store, result);
}
export function setGenerationJobsInFlight(store: GenerationJobStore, count: number): void {
generationJobMetrics.setJobsInFlight(store, count);
}
export function recordGenerationStreamSubscription(
store: GenerationJobStore,
type: GenerationStreamSubscriptionType,
result: GenerationStreamSubscriptionResult,
): void {
generationJobMetrics.recordSubscription(store, type, result);
}
export function recordGenerationStreamResumePendingEvents(
store: GenerationJobStore,
count: number,
): void {
generationJobMetrics.recordResumePendingEvents(store, count);
}
export function recordGenerationStreamEarlyBufferOverflow(store: GenerationJobStore): void {
generationJobMetrics.recordEarlyBufferOverflow(store);
}
export function recordGenerationStreamRecovery(
store: GenerationJobStore,
method: GenerationStreamRecoveryMethod,
outcome: GenerationStreamRecoveryOutcome,
durationSeconds: number,
reconstructedEvents: number,
reconstructedContent: number,
): void {
generationJobMetrics.recordRecovery(
store,
method,
outcome,
durationSeconds,
reconstructedEvents,
reconstructedContent,
);
}
export function recordGenerationStreamAttachment(
store: GenerationJobStore,
outcome: GenerationStreamAttachmentOutcome,
delaySeconds?: number,
): void {
generationJobMetrics.recordAttachment(store, outcome, delaySeconds);
}
export function recordAgentStartupMilestone(
milestone: AgentStartupMilestone,
durationSeconds: number,
): void {
if (
!agentStartupMilestoneSet.has(milestone) ||
!Number.isFinite(durationSeconds) ||
durationSeconds < 0
) {
return;
}
agentStartupMetrics.recordMilestone(milestone, durationSeconds);
}
export function recordAgentStartupResult(result: AgentStartupResult): void {
if (!agentStartupResultSet.has(result)) {
return;
}
agentStartupMetrics.recordResult(result);
}
export function recordRumProxyRequest(endpoint: RumProxyEndpoint, result: RumProxyResult): void {
rumProxyMetrics.recordRequest(endpoint, result);
}
export function recordShareLinkRejection(
operation: ShareLinkOperation,
code: ShareLinkRejectionCode,
): void {
shareLinkMetrics.recordRejection(operation, code);
}
export function recordRedisOperation(
client: RedisClient,
useCase: string,
operation: string,
status: RedisOperationStatus,
durationSeconds: number,
): void {
redisOperationMetrics.recordOperation(client, useCase, operation, status, durationSeconds);
}
const getElapsedSeconds = (startedAt: bigint): number =>
Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
export const isMetricsConfigured = (): boolean => Boolean(process.env.METRICS_SECRET);
const createUnauthorizedMetricsRouter = (): Router => {
const metricsRouter = Router();
metricsRouter.get('/', (_req, res) => {
res.status(401).end();
});
return metricsRouter;
};
const normalizeMongooseLabel = (value: unknown): string => {
if (typeof value !== 'string' || !value) return 'unknown';
return value.replace(/[^a-zA-Z0-9_:-]/g, '_').slice(0, 64) || 'unknown';
};
const getHeader = (headers: Record<string, unknown>, name: string): unknown => {
const lowerName = name.toLowerCase();
for (const [key, value] of Object.entries(headers)) {
if (key.toLowerCase() === lowerName) return value;
}
return undefined;
};
const headerIncludes = (value: unknown, expected: string): boolean => {
if (Array.isArray(value)) {
return value.some((entry) => headerIncludes(entry, expected));
}
return typeof value === 'string' && value.toLowerCase().includes(expected);
};
const isEventStreamContentType = (value: unknown): boolean =>
headerIncludes(value, 'text/event-stream');
const isMultipartContentType = (value: unknown): boolean =>
headerIncludes(value, 'multipart/form-data');
const getRequestContentLength = (req: Request): number | null => {
const contentLength = req.headers['content-length'];
const rawContentLength = Array.isArray(contentLength) ? contentLength[0] : contentLength;
const bodyBytes = rawContentLength == null ? NaN : Number(rawContentLength);
return Number.isFinite(bodyBytes) && bodyBytes >= 0 ? bodyBytes : null;
};
const isUploadRequest = (req: Request, normalizedPath: string): boolean => {
if (!UPLOAD_METHODS.has(req.method)) return false;
if (isMultipartContentType(req.headers['content-type'])) return true;
return UPLOAD_PATHS.has(normalizedPath);
};
export function recordMongooseQuery(
model: string,
operation: string,
status: string,
durationSeconds: number,
): void {
mongooseQueryMetrics.recordQuery(
normalizeMongooseLabel(model),
normalizeMongooseLabel(operation),
normalizeMongooseLabel(status),
durationSeconds,
);
}
export function instrumentMongooseQueryMetrics(mongoose: Mongoose): void {
if (!isMetricsConfigured()) return;
const instrumented = Symbol.for('librechat.mongooseQueryMetrics.instrumented');
const queryPrototype = mongoose.Query?.prototype as
| (typeof mongoose.Query.prototype & { [instrumented]?: boolean })
| undefined;
if (!queryPrototype || queryPrototype[instrumented]) return;
const originalExec = queryPrototype.exec;
queryPrototype.exec = function instrumentedExec(
this: typeof queryPrototype & {
model?: { modelName?: string };
op?: string;
},
...args: Parameters<typeof originalExec>
) {
const startedAt = process.hrtime.bigint();
const model = normalizeMongooseLabel(this.model?.modelName);
const operation = normalizeMongooseLabel(this.op);
let result: ReturnType<typeof originalExec>;
try {
result = originalExec.apply(this, args);
} catch (error) {
recordMongooseQuery(model, operation, 'error', getElapsedSeconds(startedAt));
throw error;
}
return Promise.resolve(result).then(
(result) => {
recordMongooseQuery(model, operation, 'success', getElapsedSeconds(startedAt));
return result;
},
(error) => {
recordMongooseQuery(model, operation, 'error', getElapsedSeconds(startedAt));
throw error;
},
);
} as typeof originalExec;
queryPrototype[instrumented] = true;
}
export function createMetrics(options: MetricsOptions = {}): PrometheusMetrics {
if (!isMetricsConfigured()) {
resetMetricRecorders();
return {
metricsMiddleware: (_req: Request, _res: Response, next: NextFunction) => next(),
metricsRouter: createUnauthorizedMetricsRouter(),
};
}
const registry = new Registry();
collectDefaultMetrics({ register: registry });
observeLocatorTraversal = () => undefined;
const locatorTraversalFailuresTotal = new Counter({
name: 'content_filter_locator_traversal_failures_total',
help: 'Incomplete resolved file locator traversals',
labelNames: ['operation', 'reason'] as const,
registers: [registry],
});
const locatorTraversalSize = new Histogram({
name: 'content_filter_locator_traversal_size',
help: 'Structural counts at an incomplete resolved file locator traversal',
labelNames: ['operation', 'reason', 'dimension'] as const,
buckets: [0, 1, 8, 24, 64, 256, 1024, 4096, 16384],
registers: [registry],
});
observeLocatorTraversal = (failure: LocatorTraversalFailure): void => {
const labels = { operation: failure.operation, reason: failure.reason };
locatorTraversalFailuresTotal.inc(labels);
for (const dimension of [
'visitedNodes',
'depth',
'messageCount',
'resolvedFileCount',
] as const) {
locatorTraversalSize.observe({ ...labels, dimension }, failure[dimension]);
}
};
const httpRequests = new Counter({
name: 'http_requests_total',
help: 'Total HTTP requests',
labelNames: ['method', 'path', 'status'] as const,
registers: [registry],
});
const httpDuration = new Histogram({
name: 'http_request_duration_seconds',
help: 'HTTP request latency in seconds',
labelNames: ['method', 'path', 'status'] as const,
buckets: [0.05, 0.1, 0.3, 0.5, 1, 2, 5],
registers: [registry],
});
const httpRequestsInFlight = new Gauge({
name: 'http_requests_in_flight',
help: 'HTTP requests currently being handled',
labelNames: ['method', 'path'] as const,
registers: [registry],
});
const httpRequestBodyBytes = new Histogram({
name: 'http_request_body_bytes',
help: 'HTTP request body size in bytes from the Content-Length header',
labelNames: ['method', 'path'] as const,
buckets: [1_000, 10_000, 100_000, 1_000_000, 5_000_000, 10_000_000, 25_000_000, 50_000_000],
registers: [registry],
});
const sseStreams = new Counter({
name: 'sse_streams_total',
help: 'Total SSE streams opened',
labelNames: ['method', 'path', 'status'] as const,
registers: [registry],
});
const sseStreamsInFlight = new Gauge({
name: 'sse_streams_in_flight',
help: 'SSE streams currently open',
labelNames: ['method', 'path'] as const,
registers: [registry],
});
const sseStreamDuration = new Histogram({
name: 'sse_stream_duration_seconds',
help: 'SSE stream open duration in seconds',
labelNames: ['method', 'path', 'status'] as const,
buckets: [1, 5, 10, 30, 60, 120, 300, 600, 1_200, 1_800],
registers: [registry],
});
const uploadRequests = new Counter({
name: 'upload_requests_total',
help: 'Total upload requests',
labelNames: ['method', 'path', 'status'] as const,
registers: [registry],
});
const uploadRequestsInFlight = new Gauge({
name: 'upload_requests_in_flight',
help: 'Upload requests currently being handled',
labelNames: ['method', 'path'] as const,
registers: [registry],
});
const uploadRequestDuration = new Histogram({
name: 'upload_request_duration_seconds',
help: 'Upload request duration in seconds',
labelNames: ['method', 'path', 'status'] as const,
buckets: [0.1, 0.3, 0.5, 1, 2, 5, 10, 30, 60, 120, 300],
registers: [registry],
});
const uploadBytes = new Counter({
name: 'upload_bytes_total',
help: 'Upload request bytes from the Content-Length header',
labelNames: ['method', 'path'] as const,
registers: [registry],
});
const openIDUserLookupTotal = new Counter({
name: 'openid_user_lookup_total',
help: 'OpenID user lookup attempts',
labelNames: ['result'] as const,
registers: [registry],
});
const openIDUserLookupDuration = new Histogram({
name: 'openid_user_lookup_duration_seconds',
help: 'OpenID user lookup latency in seconds',
labelNames: ['result'] as const,
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
registers: [registry],
});
openIDUserLookupMetrics = {
recordLookup: (result, durationSeconds) => {
openIDUserLookupTotal.inc({ result });
openIDUserLookupDuration.observe({ result }, durationSeconds);
},
};
const mongooseQueries = new Counter({
name: 'mongoose_queries_total',
help: 'Mongoose queries by model, operation, and status',
labelNames: ['model', 'operation', 'status'] as const,
registers: [registry],
});
const mongooseQueryDuration = new Histogram({
name: 'mongoose_query_duration_seconds',
help: 'Mongoose query duration in seconds by model, operation, and status',
labelNames: ['model', 'operation', 'status'] as const,
buckets: [0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
registers: [registry],
});
mongooseQueryMetrics = {
recordQuery: (model, operation, status, durationSeconds) => {
const labels = { model, operation, status };
mongooseQueries.inc(labels);
mongooseQueryDuration.observe(labels, durationSeconds);
},
};
const generationJobs = new Counter({
name: 'generation_jobs_total',
help: 'Generation jobs by backing store and result',
labelNames: ['store', 'result'] as const,
registers: [registry],
});
const generationJobsInFlight = new Gauge({
name: 'generation_jobs_in_flight',
help: 'Generation jobs currently running in this process',
labelNames: ['store'] as const,
registers: [registry],
});
const generationStreamSubscriptions = new Counter({
name: 'generation_stream_subscriptions_total',
help: 'Generation stream subscription attempts by backing store, type, and result',
labelNames: ['store', 'type', 'result'] as const,
registers: [registry],
});
const generationStreamResumePendingEvents = new Counter({
name: 'generation_stream_resume_pending_events_total',
help: 'Pending events delivered while resuming generation streams',
labelNames: ['store'] as const,
registers: [registry],
});
const generationStreamEarlyBufferOverflows = new Counter({
name: 'generation_stream_early_buffer_overflows_total',
help: 'Early event replay buffers discarded after exceeding hard size bounds',
labelNames: ['store'] as const,
registers: [registry],
});
const generationStreamRecoveries = new Counter({
name: 'generation_stream_recoveries_total',
help: 'Early buffer recovery attempts by backing store, source, and outcome',
labelNames: ['store', 'method', 'outcome'] as const,
registers: [registry],
});
const generationStreamRecoveryDuration = new Histogram({
name: 'generation_stream_recovery_duration_seconds',
help: 'Time spent reconstructing an overflowed early generation stream',
labelNames: ['store', 'method', 'outcome'] as const,
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],
registers: [registry],
});
const generationStreamRecoveryEvents = new Histogram({
name: 'generation_stream_recovery_events',
help: 'Event count reconstructed during early buffer recovery',
labelNames: ['store', 'method', 'outcome'] as const,
buckets: [1, 10, 100, 1_000, 5_000, 10_000, 50_000],
registers: [registry],
});
const generationStreamRecoveryContent = new Histogram({
name: 'generation_stream_recovery_content_parts',
help: 'Content part count reconstructed during early buffer recovery',
labelNames: ['store', 'method', 'outcome'] as const,
buckets: [1, 5, 10, 25, 50, 100, 500, 1_000],
registers: [registry],
});
const generationStreamAttachments = new Counter({
name: 'generation_stream_attachment_outcomes_total',
help: 'Generation stream attachment lifecycle outcomes',
labelNames: ['store', 'outcome'] as const,
registers: [registry],
});
const generationStreamFirstAttachmentDelay = new Histogram({
name: 'generation_stream_first_attachment_delay_seconds',
help: 'Time from generation creation to its first subscriber attachment',
labelNames: ['store'] as const,
buckets: [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300],
registers: [registry],
});
const agentStartupMilestoneDuration = new Histogram({
name: 'agent_startup_milestone_duration_seconds',
help: 'Cumulative agent chat startup latency from request ingress to each milestone',
labelNames: ['milestone'] as const,
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300],
registers: [registry],
});
const agentStartups = new Counter({
name: 'agent_startups_total',
help: 'Agent chat startup attempts by terminal result',
labelNames: ['result'] as const,
registers: [registry],
});
const rumProxyRequests = new Counter({
name: 'rum_proxy_requests_total',
help: 'RUM proxy requests by endpoint and result',
labelNames: ['endpoint', 'result'] as const,
registers: [registry],
});
const shareLinkRejections = new Counter({
name: 'share_link_rejections_total',
help: 'Shared link publication rejections by operation and bounded domain code',
labelNames: ['operation', 'code'] as const,
registers: [registry],
});
const redisOperations = new Counter({
name: 'redis_operations_total',
help: 'Logical Redis operations by client, use case, operation, and status',
labelNames: ['client', 'use_case', 'operation', 'status'] as const,
registers: [registry],
});
const redisOperationDuration = new Histogram({
name: 'redis_operation_duration_seconds',
help: 'Logical Redis operation latency in seconds',
labelNames: ['client', 'use_case', 'operation', 'status'] as const,
buckets: [0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
registers: [registry],
});
const agentEventActorReceiptOperations = new Counter({
name: 'agent_event_actor_receipt_operations_total',
help: 'Event actor receipt storage operations by bounded outcome and resolution',
labelNames: ['operation', 'outcome', 'resolution'] as const,
registers: [registry],
});
const agentEventActorReceiptsRetained = new Gauge({
name: 'agent_event_actor_receipts_retained',
help: 'Delivery-owned event actor receipts currently retained for replay',
labelNames: ['resolution'] as const,
registers: [registry],
});
const agentEventActorReceiptsExpiryEligible = new Gauge({
name: 'agent_event_actor_receipts_expiry_eligible',
help: 'Retained event actor receipts whose Mongo TTL deadline has elapsed',
registers: [registry],
});
const agentEventActorReconciliationsPending = new Gauge({
name: 'agent_event_actor_reconciliations_pending',
help: 'Active event actor reconciliation markers awaiting a terminal delivery receipt',
registers: [registry],
});
const agentEventActorOldestReconciliationAge = new Gauge({
name: 'agent_event_actor_oldest_reconciliation_age_seconds',
help: 'Age in seconds of the oldest active event actor reconciliation marker',
registers: [registry],
});
const agentEventActorDeliveries = new Gauge({
name: 'agent_event_actor_deliveries',
help: 'Current retrying and dead delivery rows visible to the receipt ledger',
labelNames: ['state'] as const,
registers: [registry],
});
setAgentEventActorReceiptMetricObserver(({ operation, outcome, resolution }) => {
agentEventActorReceiptOperations.inc({
operation,
outcome,
resolution: resolution ?? 'none',
});
});
let actorStorageMetricsCache:
| { snapshot: AgentEventActorStorageMetricsSnapshot; expiresAt: number }
| undefined;
let actorStorageMetricsCollection: Promise<
AgentEventActorStorageMetricsSnapshot | undefined
> | null = null;
const collectActorStorageMetrics = async () => {
const now = Date.now();
if (actorStorageMetricsCache != null && actorStorageMetricsCache.expiresAt > now) {
return actorStorageMetricsCache.snapshot;
}
actorStorageMetricsCollection ??= Promise.resolve(
options.collectAgentEventActorStorageMetrics?.(),
)
.then((snapshot) => {
if (snapshot != null) {
actorStorageMetricsCache = {
snapshot,
expiresAt: Date.now() + AGENT_EVENT_ACTOR_STORAGE_METRICS_CACHE_MS,
};
}
return snapshot;
})
.finally(() => {
actorStorageMetricsCollection = null;
});
return actorStorageMetricsCollection;
};
generationJobMetrics = {
recordJob: (store, result) => generationJobs.inc({ store, result }),
setJobsInFlight: (store, count) => generationJobsInFlight.set({ store }, count),
recordSubscription: (store, type, result) =>
generationStreamSubscriptions.inc({ store, type, result }),
recordResumePendingEvents: (store, count) =>
generationStreamResumePendingEvents.inc({ store }, count),
recordEarlyBufferOverflow: (store) => generationStreamEarlyBufferOverflows.inc({ store }),
recordRecovery: (
store,
method,
outcome,
durationSeconds,
reconstructedEvents,
reconstructedContent,
) => {
const labels = { store, method, outcome };
generationStreamRecoveries.inc(labels);
generationStreamRecoveryDuration.observe(labels, durationSeconds);
generationStreamRecoveryEvents.observe(labels, reconstructedEvents);
generationStreamRecoveryContent.observe(labels, reconstructedContent);
},
recordAttachment: (store, outcome, delaySeconds) => {
generationStreamAttachments.inc({ store, outcome });
if (outcome === 'attached' && delaySeconds != null) {
generationStreamFirstAttachmentDelay.observe({ store }, delaySeconds);
}
},
};
agentStartupMetrics = {
recordMilestone: (milestone, durationSeconds) =>
agentStartupMilestoneDuration.observe({ milestone }, durationSeconds),
recordResult: (result) => agentStartups.inc({ result }),
};
rumProxyMetrics = {
recordRequest: (endpoint, result) => rumProxyRequests.inc({ endpoint, result }),
};
shareLinkMetrics = {
recordRejection: (operation, code) => shareLinkRejections.inc({ operation, code }),
};
redisOperationMetrics = {
recordOperation: (client, useCase, operation, status, durationSeconds) => {
const labels = { client, use_case: useCase, operation, status };
redisOperations.inc(labels);
redisOperationDuration.observe(labels, durationSeconds);
},
};
const metricsMiddleware = (req: Request, res: Response, next: NextFunction): void => {
const end = httpDuration.startTimer();
const labels = { method: req.method, path: normalizePath(req.path) };
const uploadTracked = isUploadRequest(req, labels.path);
const uploadStartedAt = uploadTracked ? process.hrtime.bigint() : null;
let sseTracked = false;
let sseStartedAt: bigint | null = null;
let completed = false;
const markSSEStream = () => {
if (completed || res.writableEnded || res.destroyed) return;
if (sseTracked) return;
sseTracked = true;
sseStartedAt = process.hrtime.bigint();
sseStreamsInFlight.inc(labels);
};
const originalSetHeader = res.setHeader;
res.setHeader = function setHeader(this: Response, ...args: Parameters<Response['setHeader']>) {
const [name, value] = args;
const result = originalSetHeader.apply(this, args);
if (String(name).toLowerCase() === 'content-type' && isEventStreamContentType(value)) {
markSSEStream();
}
return result;
} as Response['setHeader'];
const originalWriteHead = res.writeHead;
res.writeHead = function writeHead(this: Response, ...args: [number, unknown?, unknown?]) {
const [, reasonPhrase, headers] = args;
const responseHeaders =
typeof reasonPhrase === 'object' && reasonPhrase != null ? reasonPhrase : headers;
if (
isEventStreamContentType(
getHeader((responseHeaders ?? {}) as Record<string, unknown>, 'content-type'),
) ||
isEventStreamContentType(res.getHeader('content-type'))
) {
markSSEStream();
}
return originalWriteHead.apply(this, args as Parameters<typeof originalWriteHead>);
} as Response['writeHead'];
httpRequestsInFlight.inc(labels);
if (uploadTracked) {
uploadRequestsInFlight.inc(labels);
}
const complete = (completedBy: 'finish' | 'close') => {
if (completed) return;
completed = true;
const requestLabels = {
...labels,
status: completedBy === 'close' ? 499 : res.statusCode,
};
httpRequests.inc(requestLabels);
end(requestLabels);
httpRequestsInFlight.dec(labels);
const bodyBytes = getRequestContentLength(req);
if (bodyBytes != null) {
httpRequestBodyBytes.observe(labels, bodyBytes);
}
if (sseTracked) {
sseStreams.inc(requestLabels);
sseStreamsInFlight.dec(labels);
if (sseStartedAt) {
sseStreamDuration.observe(requestLabels, getElapsedSeconds(sseStartedAt));
}
}
if (uploadTracked) {
uploadRequests.inc(requestLabels);
uploadRequestsInFlight.dec(labels);
if (uploadStartedAt) {
uploadRequestDuration.observe(requestLabels, getElapsedSeconds(uploadStartedAt));
}
if (bodyBytes != null) {
uploadBytes.inc(labels, bodyBytes);
}
}
};
res.once('finish', () => complete('finish'));
res.once('close', () => complete('close'));
next();
};
const metricsRouter = Router();
const metricsHandler: RequestHandler = (req, res): void => {
const secret = process.env.METRICS_SECRET;
const auth = req.headers['authorization'];
if (!secret || !auth) {
res.status(401).end();
return;
}
const bearerToken = auth.match(/^bearer\s+(.+)$/i);
if (!bearerToken) {
res.status(401).end();
return;
}
const token = bearerToken[1];
const encode = (s: string) => new TextEncoder().encode(s);
const expected = encode(secret);
const actual = encode(token);
if (expected.byteLength !== actual.byteLength || !timingSafeEqual(expected, actual)) {
res.status(401).end();
return;
}
void Promise.resolve()
.then(async () => {
const snapshot = await collectActorStorageMetrics();
if (snapshot == null) {
return;
}
for (const [resolution, count] of Object.entries(snapshot.retainedByResolution)) {
agentEventActorReceiptsRetained.set({ resolution }, count);
}
agentEventActorReceiptsExpiryEligible.set(snapshot.expiryEligible);
agentEventActorReconciliationsPending.set(snapshot.pendingReconciliations);
agentEventActorOldestReconciliationAge.set(snapshot.oldestPendingAgeSeconds);
agentEventActorDeliveries.set({ state: 'retry' }, snapshot.retryDeliveries);
agentEventActorDeliveries.set({ state: 'dead' }, snapshot.deadDeliveries);
})
.then(() => registry.metrics())
.then((metrics) => {
res.set('Content-Type', registry.contentType);
res.end(metrics);
})
.catch((err) => {
logger.error('[metrics] Failed to collect metrics:', err);
res.status(500).end();
});
};
metricsRouter.get('/', metricsHandler);
return { metricsMiddleware, metricsRouter };
}