The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
import { and, desc, eq, gte, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { toolAccessAuditEvents, toolConnections, toolCallEvents, toolRuntimeMetricCounters, toolRuntimeSlots } from "@paperclipai/db";
import type { DeploymentExposure, DeploymentMode, ToolRuntimeAlertRecommendation, ToolRuntimeHealthSummary } from "@paperclipai/shared";
import { TOOL_RUNTIME_AUDIT_WRITE_FAILURE_METRIC } from "../../tool-runtime-metrics.js";
function asRecord(value: unknown): Record<string, unknown> {
if (value && typeof value === "object" && !Array.isArray(value))
return value as Record<string, unknown>;
return {};
}
function runtimeAlert(
input: ToolRuntimeAlertRecommendation,
): ToolRuntimeAlertRecommendation {
return input;
}
function buildRuntimeAlerts(input: {
stuckStartingSlots: number;
stuckRunningSlots: number;
timeoutRate: number;
timeoutCount: number;
failureRate: number;
failureCount: number;
capacityDeferrals: number;
restartAttempts: number;
restartSuppressions: number;
degradedConnections: number;
disabledConnections: number;
missingSecretFailures: number;
auditWriteFailures: number;
}): ToolRuntimeAlertRecommendation[] {
const runbookSection = "doc/MCP-RUNTIME-OPERATIONS.md";
const timeoutSeverity =
input.timeoutCount >= 10 || input.timeoutRate >= 25
? "critical"
: input.timeoutCount >= 3 && input.timeoutRate >= 10
? "warning"
: "warning";
const failureSeverity =
input.failureCount >= 10 || input.failureRate >= 25
? "critical"
: input.failureCount >= 5 && input.failureRate >= 10
? "warning"
: "warning";
const restartSeverity =
input.restartSuppressions > 0 ? "critical" : "warning";
return [
runtimeAlert({
name: "mcp_runtime_stuck_starting_slot",
severity: "critical",
status: input.stuckStartingSlots > 0 ? "firing" : "ok",
threshold: "Any starting slot older than 5 minutes.",
observed: `${input.stuckStartingSlots} stuck starting slot(s).`,
description:
"A local stdio runtime slot is stuck before it reaches running state.",
firstResponderAction:
"Inspect the slot health/logs, stop the slot, restart it once, then disable the connection if the slot sticks again.",
runbookSection,
}),
runtimeAlert({
name: "mcp_runtime_stuck_running_slot",
severity: "critical",
status: input.stuckRunningSlots > 0 ? "firing" : "ok",
threshold: "Any running slot with no progress for 5 minutes.",
observed: `${input.stuckRunningSlots} stuck running slot(s).`,
description:
"A runtime slot is running but has not recorded progress inside the supervisor stuck-slot window.",
firstResponderAction:
"Inspect recent audit events and active tool calls; restart the slot only after confirming no healthy call is still in progress.",
runbookSection,
}),
runtimeAlert({
name: "mcp_runtime_high_timeout_rate",
severity: timeoutSeverity,
status:
input.timeoutCount >= 3 && input.timeoutRate >= 10 ? "firing" : "ok",
threshold:
"Warning at >=3 timeouts and >=10% timeout rate in 1 hour; critical at >=10 timeouts or >=25%.",
observed: `${input.timeoutCount} timeout(s), ${input.timeoutRate}% timeout rate.`,
description:
"Tool gateway calls are timing out or being runtime-deferred at an elevated rate.",
firstResponderAction:
"Check upstream MCP health, Paperclip runtime capacity, and recent gateway audit failures before retrying workloads.",
runbookSection,
}),
runtimeAlert({
name: "mcp_runtime_high_error_rate",
severity: failureSeverity,
status:
input.failureCount >= 5 && input.failureRate >= 10 ? "firing" : "ok",
threshold:
"Warning at >=5 failures and >=10% failure rate in 1 hour; critical at >=10 failures or >=25%.",
observed: `${input.failureCount} failure(s), ${input.failureRate}% failure rate.`,
description:
"Tool gateway calls are failing after policy authorization.",
firstResponderAction:
"Group audit failures by reasonCode, then fix credentials/config or disable the affected connection.",
runbookSection,
}),
runtimeAlert({
name: "mcp_runtime_capacity_deferrals_repeated",
severity: input.capacityDeferrals >= 10 ? "critical" : "warning",
status: input.capacityDeferrals >= 3 ? "firing" : "ok",
threshold:
"Warning at >=3 capacity deferrals in 1 hour; critical at >=10.",
observed: `${input.capacityDeferrals} capacity deferral(s) in 1 hour.`,
description:
"The runtime supervisor is refusing local stdio work because company or host slot capacity is exhausted.",
firstResponderAction:
"Stop idle/stale slots, lower noisy workloads, or raise slot caps only after confirming host capacity.",
runbookSection,
}),
runtimeAlert({
name: "mcp_runtime_restart_storm",
severity: restartSeverity,
status:
input.restartSuppressions > 0 || input.restartAttempts >= 3
? "firing"
: "ok",
threshold:
"Warning at >=3 restarts in 1 hour; critical on any restart suppression.",
observed: `${input.restartAttempts} restart attempt(s), ${input.restartSuppressions} suppression(s).`,
description:
"Runtime slots are restarting repeatedly or have hit restart-storm suppression.",
firstResponderAction:
"Stop the affected slot, inspect stderr/audit reason codes, and keep the connection disabled until the template/upstream is fixed.",
runbookSection,
}),
runtimeAlert({
name: "mcp_runtime_connection_health_degraded",
severity: input.degradedConnections > 0 ? "critical" : "warning",
status:
input.degradedConnections > 0 || input.disabledConnections > 0
? "firing"
: "ok",
threshold:
"Any active enabled connection with degraded/failed/missing-secret health, or any disabled enabled-path connection.",
observed: `${input.degradedConnections} degraded connection(s), ${input.disabledConnections} disabled connection(s).`,
description:
"A configured MCP connection is not healthy or has been disabled.",
firstResponderAction:
"Run a connection health check, refresh catalog after recovery, or keep the connection disabled and route agents to alternatives.",
runbookSection,
}),
runtimeAlert({
name: "mcp_runtime_missing_secret_failures",
severity: input.missingSecretFailures >= 3 ? "critical" : "warning",
status: input.missingSecretFailures > 0 ? "firing" : "ok",
threshold:
"Warning on any missing-secret failure; critical at >=3 in 1 hour.",
observed: `${input.missingSecretFailures} missing-secret failure(s) in 1 hour.`,
description:
"A connection or tool call needed a bound secret that could not be resolved.",
firstResponderAction:
"Check secret bindings and provider health without printing secret values; rotate or rebind missing secrets.",
runbookSection,
}),
runtimeAlert({
name: "mcp_runtime_audit_write_failures",
severity: "critical",
status: input.auditWriteFailures > 0 ? "firing" : "ok",
threshold: "Any audit write failure.",
observed: `${input.auditWriteFailures} audit write failure(s) in 1 hour.`,
description:
"Tool gateway audit writes failed, reducing incident traceability.",
firstResponderAction:
"Treat as a control-plane incident: check database writes, activity log writes, and retry only after audit durability is restored.",
runbookSection,
}),
];
}
function numberValue(value: unknown): number | null {
const parsed = typeof value === "number" ? value : Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
function percent(numerator: number, denominator: number): number {
if (denominator <= 0) return 0;
return Math.round((numerator / denominator) * 1000) / 10;
}
function percentile(values: number[], p: number): number | null {
if (values.length === 0) return null;
const sorted = [...values].sort((a, b) => a - b);
const index = Math.min(
sorted.length - 1,
Math.max(0, Math.ceil((p / 100) * sorted.length) - 1),
);
return sorted[index] ?? null;
}
type RuntimeHealthOptions = { deploymentMode?: DeploymentMode; deploymentExposure?: DeploymentExposure };
export function createToolRuntimeHealth(db: Db, options: RuntimeHealthOptions, now: () => Date, trustedRuntimeHost: () => string | null) {
async function runtimeHealth(
companyId: string,
): Promise<ToolRuntimeHealthSummary> {
const generatedAt = now();
const windowStartedAt = new Date(generatedAt.getTime() - 60 * 60 * 1000);
const stuckSlotMs = 5 * 60 * 1000;
const [
slots,
connections,
auditRows,
callEvents,
auditWriteFailureCounterRows,
] = await Promise.all([
db
.select()
.from(toolRuntimeSlots)
.where(eq(toolRuntimeSlots.companyId, companyId)),
db
.select()
.from(toolConnections)
.where(eq(toolConnections.companyId, companyId)),
db
.select()
.from(toolAccessAuditEvents)
.where(
and(
eq(toolAccessAuditEvents.companyId, companyId),
gte(toolAccessAuditEvents.createdAt, windowStartedAt),
),
)
.orderBy(desc(toolAccessAuditEvents.createdAt)),
db
.select()
.from(toolCallEvents)
.where(
and(
eq(toolCallEvents.companyId, companyId),
gte(toolCallEvents.createdAt, windowStartedAt),
),
)
.orderBy(desc(toolCallEvents.createdAt)),
db
.select({
count: sql<number>`coalesce(sum(${toolRuntimeMetricCounters.count}), 0)::int`,
})
.from(toolRuntimeMetricCounters)
.where(
and(
eq(toolRuntimeMetricCounters.companyId, companyId),
eq(
toolRuntimeMetricCounters.metric,
TOOL_RUNTIME_AUDIT_WRITE_FAILURE_METRIC,
),
gte(toolRuntimeMetricCounters.bucketStartAt, windowStartedAt),
),
),
]);
const activeSlots = slots.filter(
(slot) =>
slot.status === "starting" ||
slot.status === "running" ||
slot.status === "idle",
);
const staleActiveSlots = activeSlots.filter((slot) => {
const lastProgressAt =
slot.lastUsedAt ?? slot.startedAt ?? slot.updatedAt;
return generatedAt.getTime() - lastProgressAt.getTime() > stuckSlotMs;
});
const callTerminalEvents = callEvents.filter(
(event) =>
event.eventType === "call_completed" ||
event.eventType === "call_failed" ||
event.eventType === "call_denied",
);
const toolCallsLastHour = callTerminalEvents.length;
const toolTimeoutsLastHour = callTerminalEvents.filter(
(event) => event.outcome === "timeout",
).length;
const toolFailuresLastHour = callTerminalEvents.filter(
(event) => event.outcome === "failure",
).length;
const durations = auditRows
.map((row) => numberValue(asRecord(row.details).durationMs))
.filter((value): value is number => value !== null && value >= 0);
const capacityDeferrals = auditRows.filter(
(row) =>
row.action === "runtime_deferred" ||
row.reasonCode === "runtime_company_capacity_exhausted" ||
row.reasonCode === "runtime_host_capacity_exhausted",
).length;
const restartAttempts = auditRows.filter(
(row) =>
row.action === "runtime_started" && row.reasonCode !== "lazy_start",
).length;
const restartSuppressions = auditRows.filter(
(row) =>
row.action === "runtime_restart_suppressed" ||
row.reasonCode === "runtime_restart_suppressed",
).length;
const idleEvictions = auditRows.filter(
(row) =>
row.action === "runtime_stopped" &&
row.reasonCode === "idle_ttl_expired",
).length;
const missingSecretFailures = auditRows.filter(
(row) =>
row.reasonCode === "missing_secret" ||
(row.outcome === "failure" && row.reasonCode?.includes("secret")),
).length;
const legacyAuditWriteFailures = auditRows.filter(
(row) =>
row.action === "runtime_audit_write_failed" ||
row.reasonCode === "audit_write_failed",
).length;
const auditWriteFailuresMetric =
Number(auditWriteFailureCounterRows[0]?.count ?? 0) +
legacyAuditWriteFailures;
const enabledPathConnections = connections.filter(
(connection) => connection.status === "active" && connection.enabled,
);
const activeConnections = enabledPathConnections.length;
const disabledConnections = connections.filter(
(connection) => connection.status === "disabled",
).length;
const degradedConnections = enabledPathConnections.filter((connection) =>
["degraded", "failed", "error", "missing_secret"].includes(
connection.healthStatus,
),
).length;
const metrics = {
windowStartedAt,
windowEndedAt: generatedAt,
activeSlots: activeSlots.length,
startingSlots: slots.filter((slot) => slot.status === "starting").length,
runningSlots: slots.filter((slot) => slot.status === "running").length,
idleSlots: slots.filter((slot) => slot.status === "idle").length,
failedSlots: slots.filter(
(slot) => slot.status === "failed" || slot.status === "error",
).length,
stoppedSlots: slots.filter(
(slot) => slot.status === "stopped" || slot.status === "disabled",
).length,
stuckStartingSlots: staleActiveSlots.filter(
(slot) => slot.status === "starting",
).length,
stuckRunningSlots: staleActiveSlots.filter(
(slot) => slot.status === "running",
).length,
capacityDeferralsLastHour: capacityDeferrals,
restartAttemptsLastHour: restartAttempts,
restartSuppressionsLastHour: restartSuppressions,
idleEvictionsLastHour: idleEvictions,
toolCallsLastHour,
toolTimeoutsLastHour,
toolFailuresLastHour,
timeoutRateLastHour: percent(toolTimeoutsLastHour, toolCallsLastHour),
failureRateLastHour: percent(toolFailuresLastHour, toolCallsLastHour),
averageToolLatencyMsLastHour:
durations.length > 0
? Math.round(
durations.reduce((sum, value) => sum + value, 0) /
durations.length,
)
: null,
p95ToolLatencyMsLastHour: percentile(durations, 95),
missingSecretFailuresLastHour: missingSecretFailures,
auditWriteFailuresLastHour: auditWriteFailuresMetric,
activeConnections,
disabledConnections,
degradedConnections,
remoteHttpConnections: connections.filter(
(connection) =>
connection.status !== "archived" &&
connection.transport === "mcp_remote",
).length,
localStdioConnections: connections.filter(
(connection) =>
connection.status !== "archived" &&
connection.transport === "local_stdio",
).length,
};
const recommendations = buildRuntimeAlerts({
stuckStartingSlots: metrics.stuckStartingSlots,
stuckRunningSlots: metrics.stuckRunningSlots,
timeoutRate: metrics.timeoutRateLastHour,
timeoutCount: metrics.toolTimeoutsLastHour,
failureRate: metrics.failureRateLastHour,
failureCount: metrics.toolFailuresLastHour,
capacityDeferrals,
restartAttempts,
restartSuppressions,
degradedConnections,
disabledConnections,
missingSecretFailures,
auditWriteFailures: metrics.auditWriteFailuresLastHour,
});
const firing = recommendations.filter((alert) => alert.status === "firing");
const status = firing.some((alert) => alert.severity === "critical")
? "critical"
: firing.length > 0
? "degraded"
: "ok";
const deploymentMode = options.deploymentMode ?? "local_trusted";
const deploymentExposure = options.deploymentExposure ?? "private";
const localStdioSupported =
deploymentMode === "local_trusted" || Boolean(trustedRuntimeHost());
return {
status,
generatedAt,
runbookPath: "doc/MCP-RUNTIME-OPERATIONS.md",
metrics,
supportMatrix: {
remoteHttp: {
supported: true,
note: "mcp_remote MCP connections are supported in hosted cloud and local deployments.",
},
localStdio: {
supported: localStdioSupported,
note: localStdioSupported
? "local_stdio is available for local trusted mode or through the configured trusted MCP runtime host."
: `local_stdio should stay disabled for ${deploymentMode}/${deploymentExposure}; use mcp_remote or configure a trusted runtime worker.`,
},
},
alerts: firing,
recommendations,
};
}
return runtimeHealth;
}