Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Autonomous Jobs — self-diagnostics script handler (#2464).
*
* A deterministic `type: 'script'` job that surfaces CoS self-healing failures
* (claim-work / layered-intelligence / any task category) so operators get an
* automated pulse on system health. On each run it:
* 1. Reads the task-learning per-category metrics (`byTaskType`) — the same
* completion data the Learning tab renders.
* 2. For every category with failures in the RECENT WINDOW (issue #2617: the
* `recentOutcomes` ring windowed by `computeWindowedStats`, NOT the
* never-decaying lifetime `failed` counter), records a structured
* single-line log entry (slug, windowed total/failed counts, lastCompleted,
* avg/max duration). A category that failed historically but has a clean
* recent window drops off as the window rolls.
* 3. Aggregates the failing categories, ordered by impact, into ONE persistent
* GitHub summary issue — reusing an existing issue marked `monitoring` so the
* loop never files a fresh duplicate each cycle.
* 4. Applies a `needs attention` label while recent failures exist (and removes
* it once the windowed failing set is empty) so the same summary isn't
* re-reported as stale.
*
* NO LLM calls — this is a pure metrics-read + `gh` issue write, safe to run on a
* schedule under the cold-bootstrap AI policy. The pure helpers (compute/sort,
* title/body/log builders) are side-effect-free and unit-tested; the I/O handler
* takes injectable deps so tests drive it without a live `gh` or filesystem.
*/
import { runCliBounded } from '../../lib/runCliBounded.js'
import { PATHS, safeJSONParse } from '../../lib/fileUtils.js'
import { loadLearningData, computeWindowedStats } from '../taskLearning/store.js'
// The label that marks the ONE persistent summary issue to reuse across runs, and
// the attention label the loop toggles so a still-open summary isn't re-filed.
export const MONITORING_LABEL = 'monitoring'
export const NEEDS_ATTENTION_LABEL = 'needs attention'
// Stable dedup marker embedded in the summary issue body. Lets a run recognize
// its own prior summary even if the `monitoring` label was hand-applied to an
// unrelated issue, and lets a human/label read find it.
export const DIAGNOSTICS_SLUG = 'cos-self-diagnostics'
const SLUG_MARKER = `<!-- lil-slug: ${DIAGNOSTICS_SLUG} -->`
const STABLE_TITLE = 'CoS self-diagnostics: self-healing failures'
/**
* Reduce the learning `byTaskType` map to the categories with at least one
* failure in the RECENT WINDOW (issue #2617), projected onto the fields the
* summary + logs need, ordered by impact. Pure. Counts come from
* `computeWindowedStats` over each bucket's `recentOutcomes` ring (bounded by
* both sample count and age) — NOT the lifetime `failed` counter, which never
* decays and would keep a long-fixed category on the monitoring issue forever.
* A category with a clean (or empty) recent window is excluded, so stale rows
* drop off as the window rolls and the `needs attention` label can clear.
* Impact order: most recent failures first, then lowest windowed success rate,
* then most windowed runs (a big-sample low-rate category outranks a one-off
* flake).
*
* @param {Record<string, object>} byTaskType
* @param {{ minCompleted?: number, now?: number }} [opts] - `minCompleted`
* ignores thin categories below this windowed run count (default 1 — any
* category with a recent failure qualifies); `now` is the clock seam
* forwarded to the windowing (epoch ms).
* @returns {Array<{ slug, total, failed, succeeded, successRate, lastCompleted, avgDurationMs, maxDurationMs }>}
*/
export function computeFailingCategories(byTaskType, { minCompleted = 1, now = Date.now() } = {}) {
if (!byTaskType || typeof byTaskType !== 'object') return []
return Object.entries(byTaskType)
.map(([slug, m]) => {
const w = computeWindowedStats(m?.recentOutcomes, { now })
return {
slug,
total: w.windowedCompleted,
failed: w.windowedFailed,
succeeded: w.windowedSucceeded,
// Post-filter rows always have windowed samples, so the null sentinel
// (empty window) only shows up in rows the filter drops.
successRate: w.windowedSuccessRate ?? 0,
lastCompleted: m?.lastCompleted ?? null,
avgDurationMs: m?.avgDurationMs ?? null,
maxDurationMs: m?.maxDurationMs ?? null
}
})
.filter(c => c.failed > 0 && c.total >= minCompleted)
.sort((a, b) =>
b.failed - a.failed ||
a.successRate - b.successRate ||
b.total - a.total
)
}
/**
* Human-friendly ms → minutes/seconds. Pure. Null-safe (`—` when absent).
* Distinct from `fileUtils.formatDuration` (which floors to whole minutes and
* renders sub-minute as "0m") and the client `formatDurationMs`: diagnostics
* durations are often seconds, and absent must read as `—`, not a zero duration.
*/
export function formatDurationShort(ms) {
if (!Number.isFinite(ms) || ms < 0) return '—'
if (ms < 1000) return `${Math.round(ms)}ms`
const totalSec = Math.round(ms / 1000)
if (totalSec < 60) return `${totalSec}s`
const min = Math.floor(totalSec / 60)
const sec = totalSec % 60
return sec ? `${min}m ${sec}s` : `${min}m`
}
/**
* Structured single-line log entry for one failing category. Pure — the handler
* emits it via console so logs stay single-line + emoji-prefixed (AGENTS.md).
*/
export function formatCategoryLogLine(c) {
return `🩺 ${c.slug}: ${c.failed}/${c.total} recent failed (${c.successRate}% success) · ` +
`last ${c.lastCompleted || 'n/a'} · avg ${formatDurationShort(c.avgDurationMs)} · max ${formatDurationShort(c.maxDurationMs)}`
}
/** Stable summary-issue title. Pure. */
export function buildDiagnosticsIssueTitle() {
return STABLE_TITLE
}
/**
* Markdown body for the summary issue. Pure. Lists failing categories in impact
* order as a table, embeds the dedup slug marker, and stamps the run time so a
* reused issue shows freshness. `failing:[]` renders an explicit all-clear.
*/
export function buildDiagnosticsIssueBody(failing, { generatedAt = new Date().toISOString() } = {}) {
const lines = []
lines.push('_Automated CoS self-diagnostics — regenerated each run. Do not hand-edit; edits are overwritten._')
lines.push('')
if (!failing || failing.length === 0) {
lines.push('✅ **All self-healing categories are passing** — no categories with failures in the recent window.')
} else {
lines.push(`Found **${failing.length}** self-healing categor${failing.length === 1 ? 'y' : 'ies'} with recent failures (recency-windowed — long-fixed categories drop off as the window rolls), ordered by impact:`)
lines.push('')
lines.push('| Category | Failed / Total (recent) | Success rate (recent) | Last completed | Avg duration | Max duration |')
lines.push('| --- | --- | --- | --- | --- | --- |')
for (const c of failing) {
lines.push(`| \`${c.slug}\` | ${c.failed} / ${c.total} | ${c.successRate}% | ${c.lastCompleted || '—'} | ${formatDurationShort(c.avgDurationMs)} | ${formatDurationShort(c.maxDurationMs)} |`)
}
lines.push('')
lines.push('These are candidates for human diagnosis (model routing, tooling, or pipeline issues). Investigate the highest-impact rows first.')
}
lines.push('')
lines.push(`_Last run: ${generatedAt}_`)
lines.push('')
lines.push(SLUG_MARKER)
return lines.join('\n')
}
// ---------------------------------------------------------------------------
// I/O layer — `gh` shell + the handler. Injectable deps keep them testable.
// ---------------------------------------------------------------------------
// The injectable default `exec` for the gh calls below; bounded so a stalled gh can't wedge the job.
const runCli = runCliBounded
/**
* Locate OUR persistent summary issue to reuse. Matches only an open `monitoring`
* issue that is identifiably ours — by the embedded slug marker, or (forward-compat
* for a summary filed before the marker existed) by our exact stable title. It
* deliberately does NOT fall back to "the first arbitrary `monitoring` issue": the
* handler edit-overwrites the body it returns, so reusing an unrelated issue the
* user labeled `monitoring` for their own tracking would clobber it. When none is
* ours, returns `{ ok:true, issue:null }` so the handler files its own dedicated
* summary. `ok:false` means the read FAILED (do NOT then create — a transient `gh`
* blip would file a duplicate; sentinel rule).
*/
export async function findMonitoringIssue({ cwd = PATHS.root, exec = runCli } = {}) {
// High limit so our long-lived summary (kept at its original creation date)
// can't fall out of a small recency window on an install that accumulates many
// open `monitoring`-labeled issues — which would defeat dedup and file a fresh
// duplicate. gh sorts newest-first; the summary sinks over time.
const { code, stdout } = await exec('gh',
['issue', 'list', '--label', MONITORING_LABEL, '--state', 'open', '--limit', '200', '--json', 'number,title,body,labels,url'],
{ cwd })
if (code !== 0) return { ok: false, issue: null }
if (!stdout.trim()) return { ok: true, issue: null }
const parsed = safeJSONParse(stdout, null, { logError: false })
if (!Array.isArray(parsed)) return { ok: false, issue: null }
const norm = parsed.map(i => ({
number: i.number ?? null,
title: i.title || '',
body: i.body || '',
url: i.url || null,
labels: (i.labels || []).map(l => (typeof l === 'string' ? l : l?.name)).filter(Boolean)
}))
const ours = norm.find(i => i.body.includes(SLUG_MARKER)) || norm.find(i => i.title === STABLE_TITLE)
return { ok: true, issue: ours || null }
}
/**
* Ensure the labels exist before the first `issue create` (gh fails creating an
* issue with a non-existent label). Idempotent — `--force` is a no-op when the
* label already exists.
*/
export async function ensureDiagnosticsLabels({ cwd = PATHS.root, exec = runCli } = {}) {
const labels = [
{ name: MONITORING_LABEL, color: '0e8a16', desc: 'Persistent monitoring / self-diagnostics summary issue' },
{ name: NEEDS_ATTENTION_LABEL, color: 'd93f0b', desc: 'Flagged by CoS self-diagnostics — needs human diagnosis' }
]
for (const l of labels) {
await exec('gh', ['label', 'create', l.name, '--color', l.color, '--description', l.desc, '--force'], { cwd })
}
}
const MALFORMED_LABEL = /^(?:model:[^ ]+|effort:[^ ]+|planner:[^ ]+|plan|tests|bug|data-safety|upgrade-rehearsal|security|needs-decision) (?:(--color )?#?([0-9A-Fa-f]{6}))(?: .*)?$/
/**
* Remove labels accidentally created by passing a whole "name color description"
* string as one CLI argument. A label is deleted only after both issue and PR
* searches succeed and report no usage; any ambiguous read is fail-closed.
*/
async function pruneMalformedLabels({ cwd = PATHS.root, exec = runCli } = {}) {
const { code, stdout, stderr } = await exec('gh', ['label', 'list', '--limit', '500', '--json', 'name'], { cwd })
if (code !== 0) {
console.warn(`⚠️ Self-diagnostics: could not list labels for pruning: ${stderr.trim()}`)
return { pruned: 0, skipped: 0 }
}
const labels = safeJSONParse(stdout, null, { logError: false })
if (!Array.isArray(labels) || labels.some(label => typeof label?.name !== 'string')) {
console.warn('⚠️ Self-diagnostics: could not parse labels for pruning')
return { pruned: 0, skipped: 0 }
}
let pruned = 0
let skipped = 0
for (const { name } of labels) {
const malformed = name.match(MALFORMED_LABEL)
// Keep unrelated labels with number-like values (e.g. "ticket 123456").
// Only the label prefixes known to have been produced by the broken shell
// loops are candidates; a bare six-letter word such as "decade" is hex too.
if (!malformed || (!malformed[1] && !/[0-9]/.test(malformed[2]))) continue
// gh's --label filters split commas into multiple labels, so those names
// cannot be checked safely through the issue/pr list commands.
if (name.includes(',')) {
skipped++
console.warn(`⚠️ Self-diagnostics: cannot safely verify comma-containing malformed label ${JSON.stringify(name)}; leaving it in place`)
continue
}
const usage = await Promise.all([
exec('gh', ['issue', 'list', '--state', 'all', '--label', name, '--limit', '1', '--json', 'number'], { cwd }),
exec('gh', ['pr', 'list', '--state', 'all', '--label', name, '--limit', '1', '--json', 'number'], { cwd })
])
const [issues, pullRequests] = usage
const issueNumbers = issues.code === 0 ? safeJSONParse(issues.stdout, null, { logError: false }) : null
const prNumbers = pullRequests.code === 0 ? safeJSONParse(pullRequests.stdout, null, { logError: false }) : null
if (!Array.isArray(issueNumbers) || !Array.isArray(prNumbers) ||
issueNumbers.some(item => !Number.isInteger(item?.number)) ||
prNumbers.some(item => !Number.isInteger(item?.number))) {
skipped++
console.warn(`⚠️ Self-diagnostics: could not verify usage for malformed label ${JSON.stringify(name)}; leaving it in place`)
continue
}
if (issueNumbers.length || prNumbers.length) {
skipped++
const usages = [
...issueNumbers.map(item => `issue #${item.number}`),
...prNumbers.map(item => `PR #${item.number}`)
]
console.warn(`⚠️ Self-diagnostics: malformed label ${JSON.stringify(name)} is in use by ${usages.join(', ')}; leaving it in place`)
continue
}
const deleted = await exec('gh', ['label', 'delete', name, '--yes'], { cwd })
if (deleted.code !== 0) {
skipped++
console.warn(`⚠️ Self-diagnostics: failed to delete malformed label ${JSON.stringify(name)}: ${deleted.stderr.trim()}`)
continue
}
pruned++
}
console.log(`🧹 Pruned ${pruned} malformed labels`)
return { pruned, skipped }
}
/**
* Self-diagnostics handler (SCRIPT_HANDLERS['self-diagnostics']).
*
* @param {object} [deps]
* @param {() => Promise<object>} [deps.loadLearning] - learning-data loader
* @param {(cmd,args,opts)=>Promise<{code,stdout,stderr}>} [deps.exec] - CLI runner
* @param {string} [deps.cwd] - repo root for `gh`
* @param {() => string} [deps.now] - ISO clock (test seam)
* @param {() => Promise<{ok:boolean,status:string}>} [deps.forgeGate] - gh probe (test seam)
* @returns {Promise<{ failingCount, categories, issue, forgeUnavailable? }>}
*/
export async function runSelfDiagnostics({
loadLearning = loadLearningData,
exec = runCli,
cwd = PATHS.root,
now = () => new Date().toISOString(),
forgeGate = null
} = {}) {
const data = await loadLearning()
// Window against the injected clock so tests are deterministic (issue #2617).
const failing = computeFailingCategories(data?.byTaskType, { now: new Date(now()).getTime() })
if (failing.length === 0) {
console.log('🩺 Self-diagnostics: no self-healing categories with recent failures — all clear')
} else {
console.log(`🩺 Self-diagnostics: ${failing.length} failing categor${failing.length === 1 ? 'y' : 'ies'} (top: ${failing[0].slug} ${failing[0].failed}/${failing[0].total})`)
for (const c of failing) console.log(formatCategoryLogLine(c))
}
// The metrics half above is local and always worth logging; everything below
// reads or writes the forge. Probe once and skip that half rather than issuing
// four `gh` calls that each fail into an ambiguous empty result (#3358).
const gate = forgeGate || (async () => {
const { ensureForgeReachable } = await import('../github.js')
return ensureForgeReachable('self-diagnostics')
})
const forge = await gate()
if (!forge?.ok) {
return { failingCount: failing.length, categories: failing, issue: null, forgeUnavailable: true }
}
const generatedAt = now()
const title = buildDiagnosticsIssueTitle()
const body = buildDiagnosticsIssueBody(failing, { generatedAt })
const found = await findMonitoringIssue({ cwd, exec })
if (!found.ok) {
// Read failed — refuse to file, or a transient gh blip duplicates the summary.
console.warn('⚠️ Self-diagnostics: could not read monitoring issues (gh unavailable?) — skipping issue update')
return { failingCount: failing.length, categories: failing, issue: null }
}
const hasFailures = failing.length > 0
if (found.issue) {
// Reuse the existing monitoring issue: refresh the body, and toggle the
// attention label to match the current state (add on failures, clear on all-green).
const editArgs = ['issue', 'edit', String(found.issue.number), '--body', body]
const addingLabel = hasFailures && !found.issue.labels.includes(NEEDS_ATTENTION_LABEL)
if (addingLabel) {
await ensureDiagnosticsLabels({ cwd, exec })
editArgs.push('--add-label', NEEDS_ATTENTION_LABEL)
} else if (!hasFailures && found.issue.labels.includes(NEEDS_ATTENTION_LABEL)) {
editArgs.push('--remove-label', NEEDS_ATTENTION_LABEL)
}
await pruneMalformedLabels({ cwd, exec })
const { code, stderr } = await exec('gh', editArgs, { cwd })
if (code !== 0) {
console.warn(`⚠️ Self-diagnostics: failed to update issue #${found.issue.number}: ${stderr.trim()}`)
return { failingCount: failing.length, categories: failing, issue: null }
}
console.log(`🩺 Self-diagnostics: updated monitoring issue #${found.issue.number} (${failing.length} failing)`)
return { failingCount: failing.length, categories: failing, issue: { number: found.issue.number, url: found.issue.url, created: false } }
}
if (!hasFailures) {
// Nothing failing and no existing summary — don't create noise.
await pruneMalformedLabels({ cwd, exec })
console.log('🩺 Self-diagnostics: no failures and no existing summary issue — nothing to file')
return { failingCount: 0, categories: [], issue: null }
}
await ensureDiagnosticsLabels({ cwd, exec })
await pruneMalformedLabels({ cwd, exec })
const createArgs = ['issue', 'create', '--title', title, '--body', body, '--label', MONITORING_LABEL, '--label', NEEDS_ATTENTION_LABEL]
const { code, stdout, stderr } = await exec('gh', createArgs, { cwd })
if (code !== 0) {
console.warn(`⚠️ Self-diagnostics: failed to create monitoring issue: ${stderr.trim()}`)
return { failingCount: failing.length, categories: failing, issue: null }
}
const url = (stdout.trim().match(/(https?:\/\/\S+)/) || [])[1] || stdout.trim()
const number = Number((url.match(/(\d+)\s*$/) || [])[1]) || null
console.log(`🩺 Self-diagnostics: filed monitoring issue ${url} (${failing.length} failing)`)
return { failingCount: failing.length, categories: failing, issue: { number, url, created: true } }
}