Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Scoped event-loop lag attribution for slow-operation diagnostics (#12329).
*
* A stopwatch around an awaited phase measures asynchronous wait (storage, the
* libuv threadpool, a queue) plus delayed scheduling, and cannot tell them
* apart. While at least one span is open, one shared tick is rescheduled every
* `intervalMs`; the time it spends overdue is time the loop could not run
* timers — blocking JavaScript, GC, host CPU starvation or process suspension.
* Each span charges that overdue time to the phase it was in, so a slow phase
* splits into `lagMs` (scheduling) and the remainder (asynchronous wait).
*
* Nothing is scheduled while no span is open. Resolution is about one interval,
* and an overdue stretch no longer than `toleranceMs` is ignored as timer
* jitter. Phase names are caller-supplied constants; summaries are numbers only.
*
* A span charged more than `attributeAboveMs` of lag also carries `load`
* (#12852): this process's CPU time, GC time and involuntary context switches
* over the span, plus host load, so lag can be told apart as busy-process
* (`cpuMs` near the span's wall time; `gcMs` names the GC share) versus
* starved-process (low `cpuMs`, `load1` at or above `cpus`).
*/
import os from 'node:os';
import { PerformanceObserver } from 'node:perf_hooks';
/** Resolves the globals at call time, so fake timers in tests govern it. */
export const systemClock = {
now: () => performance.now(),
setTimer: (fn, ms) => setTimeout(fn, ms),
clearTimer: (timer) => clearTimeout(timer),
};
/**
* A deterministic clock with the same contract. `advance(ms)` models a
* responsive loop (every timer fires on schedule); `stall(ms)` models a blocked
* one (time passes and overdue timers wait for the next `advance`).
*/
export function __createManualClock() {
let now = 0;
let seq = 0;
const timers = new Map();
const nextDue = (limit) => [...timers].filter(([, t]) => t.at <= limit)
.sort(([a, x], [b, y]) => x.at - y.at || a - b)[0];
return {
now: () => now,
setTimer: (fn, ms) => { timers.set(++seq, { fn, at: now + ms }); return seq; },
clearTimer: (id) => { timers.delete(id); },
advance(ms) {
const target = now + ms;
for (let due = nextDue(target); due; due = nextDue(target)) {
timers.delete(due[0]);
now = Math.max(now, due[1].at);
due[1].fn();
}
now = target;
},
stall(ms) { now += ms; },
};
}
export const roundMs = (ms) => Math.min(Number.MAX_SAFE_INTEGER, Math.max(0, Math.round(ms)));
/**
* GC time by occurrence, not delivery: Node queues `gc` entries to a later
* loop turn (`takeRecords()` does not see them synchronously), so each entry is
* charged to the open windows by its `startTime`. A GC in a span's last loop
* turn may arrive after the span is read. The observer runs only while a
* window is open, and entries older than every open window are dropped.
*/
function createGcMeter() {
const windows = new Set();
let entries = [];
let observer = null;
const record = (list) => {
for (const entry of list.getEntries()) entries.push([entry.startTime, entry.duration]);
};
return {
begin() {
const gcWindow = { since: performance.now() };
if (windows.size === 0) {
observer = new PerformanceObserver(record);
observer.observe({ entryTypes: ['gc'] });
}
windows.add(gcWindow);
let open = true;
return {
ms: () => entries.reduce((ms, [start, duration]) => (start >= gcWindow.since ? ms + duration : ms), 0),
end() {
if (!open) return;
open = false;
windows.delete(gcWindow);
if (windows.size === 0) {
observer.disconnect();
observer = null;
entries = [];
return;
}
let oldest = Infinity;
for (const { since } of windows) oldest = Math.min(oldest, since);
entries = entries.filter(([start]) => start >= oldest);
},
};
},
};
}
/** Process and host counters for lag attribution; each is resolved at call time. */
export const systemResources = {
cpuUsage: () => process.cpuUsage(),
loadavg: () => os.loadavg(),
cpus: () => os.availableParallelism(),
involuntaryContextSwitches: () => process.resourceUsage().involuntaryContextSwitches,
gc: createGcMeter(),
};
/**
* Start counting this process's CPU, GC and involuntary context switches.
* `read()` returns the deltas so far plus host load; `end()` stops the GC
* window and is idempotent. Numbers only.
*/
export function beginLoadSample(resources = systemResources) {
const cpuStart = resources.cpuUsage();
const switchesStart = resources.involuntaryContextSwitches();
const gc = resources.gc.begin();
return {
read() {
const cpu = resources.cpuUsage();
const [load1] = resources.loadavg();
return {
cpuMs: roundMs((cpu.user - cpuStart.user + cpu.system - cpuStart.system) / 1000),
load1: Math.round(load1 * 100) / 100,
cpus: resources.cpus(),
gcMs: roundMs(gc.ms()),
invCtx: Math.max(0, resources.involuntaryContextSwitches() - switchesStart),
};
},
end: () => gc.end(),
};
}
/** ` cpuMs=… load1=… cpus=… gcMs=… invCtx=…` for a log line, or '' without a sample. */
export const formatLoadAttribution = (load) => (load
? ` cpuMs=${load.cpuMs} load1=${load.load1} cpus=${load.cpus} gcMs=${load.gcMs} invCtx=${load.invCtx}`
: '');
/**
* @param {{ clock?: typeof systemClock, intervalMs?: number, toleranceMs?: number,
* resources?: typeof systemResources, attributeAboveMs?: number }} [options]
* @returns {{ begin: (phase: string) => { mark: (phase: string) => void, snapshot: () => object, end: () => object } }}
*/
export function createEventLoopLagProbe({
clock = systemClock, intervalMs = 100, toleranceMs = 20, resources = systemResources, attributeAboveMs = 1000,
} = {}) {
const spans = new Set();
let timer = null;
let dueAt = 0;
// Charge the overdue part of [accountedTo, at] to the span's current phase.
const account = (span, at) => {
const from = Math.max(dueAt, span.accountedTo);
span.accountedTo = Math.max(span.accountedTo, at);
const lag = at - from;
if (lag <= toleranceMs) return;
span.phases[span.phase].lagMs += lag;
span.lagMs += lag;
span.lagMaxMs = Math.max(span.lagMaxMs, lag);
};
// The tick only observes; it must never keep the process alive.
const schedule = (at) => {
dueAt = at + intervalMs;
timer = clock.setTimer(tick, intervalMs);
timer?.unref?.();
};
function tick() {
const at = clock.now();
for (const span of spans) account(span, at);
schedule(at);
}
const enter = (span, phase, at) => {
span.phase = phase;
span.phaseStartedAt = at;
span.phaseEnteredAt = at;
span.phases[phase] ??= { ms: 0, lagMs: 0 };
};
const close = (span, at) => {
account(span, at);
span.phases[span.phase].ms += at - span.phaseStartedAt;
span.phaseStartedAt = at;
};
const summarize = (span, at) => ({
phase: span.phase,
phaseMs: roundMs(at - span.phaseEnteredAt),
totalMs: roundMs(at - span.startedAt),
lagMs: roundMs(span.lagMs),
lagMaxMs: roundMs(span.lagMaxMs),
phases: Object.fromEntries(Object.entries(span.phases)
.map(([name, { ms, lagMs }]) => [name, { ms: roundMs(ms), lagMs: roundMs(lagMs) }])),
load: span.lagMs > attributeAboveMs ? span.sample.read() : null,
});
return {
begin(phase) {
const at = clock.now();
if (timer === null) schedule(at);
const span = { startedAt: at, accountedTo: at, lagMs: 0, lagMaxMs: 0, phases: {}, sample: beginLoadSample(resources) };
enter(span, phase, at);
spans.add(span);
let ended = null;
return {
mark(next) {
if (ended) return;
const now = clock.now();
close(span, now);
enter(span, next, now);
},
snapshot() {
if (ended) return ended;
const now = clock.now();
close(span, now);
return summarize(span, now);
},
end() {
if (ended) return ended;
const now = clock.now();
close(span, now);
ended = summarize(span, now);
span.sample.end();
spans.delete(span);
if (spans.size === 0 && timer !== null) {
clock.clearTimer(timer);
timer = null;
}
return ended;
},
};
},
};
}
/** Sum the named phase and its `name.` sub-phases from a span summary. */
export function phaseGroup(summary, name) {
let ms = 0;
let lagMs = 0;
for (const [phase, value] of Object.entries(summary.phases)) {
if (phase !== name && !phase.startsWith(`${name}.`)) continue;
ms += value.ms;
lagMs += value.lagMs;
}
return { ms, lagMs };
}