Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

18.5 KB
Raw Blame
/**
 * Autonomous Jobs — self-diagnostics script handler (#2464).
 *
 * A deterministic `type: 'script'` job that surfaces CoS self-healing failures
 * (claim-work / layered-intelligence / any task category) so operators get an
 * automated pulse on system health. On each run it:
 *   1. Reads the task-learning per-category metrics (`byTaskType`) — the same
 *      completion data the Learning tab renders.
 *   2. For every category with failures in the RECENT WINDOW (issue #2617: the
 *      `recentOutcomes` ring windowed by `computeWindowedStats`, NOT the
 *      never-decaying lifetime `failed` counter), records a structured
 *      single-line log entry (slug, windowed total/failed counts, lastCompleted,
 *      avg/max duration). A category that failed historically but has a clean
 *      recent window drops off as the window rolls.
 *   3. Aggregates the failing categories, ordered by impact, into ONE persistent
 *      GitHub summary issue — reusing an existing issue marked `monitoring` so the
 *      loop never files a fresh duplicate each cycle.
 *   4. Applies a `needs attention` label while recent failures exist (and removes
 *      it once the windowed failing set is empty) so the same summary isn't
 *      re-reported as stale.
 *
 * NO LLM calls — this is a pure metrics-read + `gh` issue write, safe to run on a
 * schedule under the cold-bootstrap AI policy. The pure helpers (compute/sort,
 * title/body/log builders) are side-effect-free and unit-tested; the I/O handler
 * takes injectable deps so tests drive it without a live `gh` or filesystem.
 */

import { runCliBounded } from '../../lib/runCliBounded.js'
import { PATHS, safeJSONParse } from '../../lib/fileUtils.js'
import { loadLearningData, computeWindowedStats } from '../taskLearning/store.js'

// The label that marks the ONE persistent summary issue to reuse across runs, and
// the attention label the loop toggles so a still-open summary isn't re-filed.
export const MONITORING_LABEL = 'monitoring'
export const NEEDS_ATTENTION_LABEL = 'needs attention'

// Stable dedup marker embedded in the summary issue body. Lets a run recognize
// its own prior summary even if the `monitoring` label was hand-applied to an
// unrelated issue, and lets a human/label read find it.
export const DIAGNOSTICS_SLUG = 'cos-self-diagnostics'
const SLUG_MARKER = `<!-- lil-slug: ${DIAGNOSTICS_SLUG} -->`

const STABLE_TITLE = 'CoS self-diagnostics: self-healing failures'

/**
 * Reduce the learning `byTaskType` map to the categories with at least one
 * failure in the RECENT WINDOW (issue #2617), projected onto the fields the
 * summary + logs need, ordered by impact. Pure. Counts come from
 * `computeWindowedStats` over each bucket's `recentOutcomes` ring (bounded by
 * both sample count and age) — NOT the lifetime `failed` counter, which never
 * decays and would keep a long-fixed category on the monitoring issue forever.
 * A category with a clean (or empty) recent window is excluded, so stale rows
 * drop off as the window rolls and the `needs attention` label can clear.
 * Impact order: most recent failures first, then lowest windowed success rate,
 * then most windowed runs (a big-sample low-rate category outranks a one-off
 * flake).
 *
 * @param {Record<string, object>} byTaskType
 * @param {{ minCompleted?: number, now?: number }} [opts] - `minCompleted`
 *   ignores thin categories below this windowed run count (default 1 — any
 *   category with a recent failure qualifies); `now` is the clock seam
 *   forwarded to the windowing (epoch ms).
 * @returns {Array<{ slug, total, failed, succeeded, successRate, lastCompleted, avgDurationMs, maxDurationMs }>}
 */
export function computeFailingCategories(byTaskType, { minCompleted = 1, now = Date.now() } = {}) {
  if (!byTaskType || typeof byTaskType !== 'object') return []
  return Object.entries(byTaskType)
    .map(([slug, m]) => {
      const w = computeWindowedStats(m?.recentOutcomes, { now })
      return {
        slug,
        total: w.windowedCompleted,
        failed: w.windowedFailed,
        succeeded: w.windowedSucceeded,
        // Post-filter rows always have windowed samples, so the null sentinel
        // (empty window) only shows up in rows the filter drops.
        successRate: w.windowedSuccessRate ?? 0,
        lastCompleted: m?.lastCompleted ?? null,
        avgDurationMs: m?.avgDurationMs ?? null,
        maxDurationMs: m?.maxDurationMs ?? null
      }
    })
    .filter(c => c.failed > 0 && c.total >= minCompleted)
    .sort((a, b) =>
      b.failed - a.failed ||
      a.successRate - b.successRate ||
      b.total - a.total
    )
}

/**
 * Human-friendly ms → minutes/seconds. Pure. Null-safe (`—` when absent).
 * Distinct from `fileUtils.formatDuration` (which floors to whole minutes and
 * renders sub-minute as "0m") and the client `formatDurationMs`: diagnostics
 * durations are often seconds, and absent must read as `—`, not a zero duration.
 */
export function formatDurationShort(ms) {
  if (!Number.isFinite(ms) || ms < 0) return '—'
  if (ms < 1000) return `${Math.round(ms)}ms`
  const totalSec = Math.round(ms / 1000)
  if (totalSec < 60) return `${totalSec}s`
  const min = Math.floor(totalSec / 60)
  const sec = totalSec % 60
  return sec ? `${min}m ${sec}s` : `${min}m`
}

/**
 * Structured single-line log entry for one failing category. Pure — the handler
 * emits it via console so logs stay single-line + emoji-prefixed (AGENTS.md).
 */
export function formatCategoryLogLine(c) {
  return `🩺 ${c.slug}: ${c.failed}/${c.total} recent failed (${c.successRate}% success) · ` +
    `last ${c.lastCompleted || 'n/a'} · avg ${formatDurationShort(c.avgDurationMs)} · max ${formatDurationShort(c.maxDurationMs)}`
}

/** Stable summary-issue title. Pure. */
export function buildDiagnosticsIssueTitle() {
  return STABLE_TITLE
}

/**
 * Markdown body for the summary issue. Pure. Lists failing categories in impact
 * order as a table, embeds the dedup slug marker, and stamps the run time so a
 * reused issue shows freshness. `failing:[]` renders an explicit all-clear.
 */
export function buildDiagnosticsIssueBody(failing, { generatedAt = new Date().toISOString() } = {}) {
  const lines = []
  lines.push('_Automated CoS self-diagnostics — regenerated each run. Do not hand-edit; edits are overwritten._')
  lines.push('')
  if (!failing || failing.length === 0) {
    lines.push('✅ **All self-healing categories are passing** — no categories with failures in the recent window.')
  } else {
    lines.push(`Found **${failing.length}** self-healing categor${failing.length === 1 ? 'y' : 'ies'} with recent failures (recency-windowed — long-fixed categories drop off as the window rolls), ordered by impact:`)
    lines.push('')
    lines.push('| Category | Failed / Total (recent) | Success rate (recent) | Last completed | Avg duration | Max duration |')
    lines.push('| --- | --- | --- | --- | --- | --- |')
    for (const c of failing) {
      lines.push(`| \`${c.slug}\` | ${c.failed} / ${c.total} | ${c.successRate}% | ${c.lastCompleted || '—'} | ${formatDurationShort(c.avgDurationMs)} | ${formatDurationShort(c.maxDurationMs)} |`)
    }
    lines.push('')
    lines.push('These are candidates for human diagnosis (model routing, tooling, or pipeline issues). Investigate the highest-impact rows first.')
  }
  lines.push('')
  lines.push(`_Last run: ${generatedAt}_`)
  lines.push('')
  lines.push(SLUG_MARKER)
  return lines.join('\n')
}

// ---------------------------------------------------------------------------
// I/O layer — `gh` shell + the handler. Injectable deps keep them testable.
// ---------------------------------------------------------------------------

// The injectable default `exec` for the gh calls below; bounded so a stalled gh can't wedge the job.
const runCli = runCliBounded

/**
 * Locate OUR persistent summary issue to reuse. Matches only an open `monitoring`
 * issue that is identifiably ours — by the embedded slug marker, or (forward-compat
 * for a summary filed before the marker existed) by our exact stable title. It
 * deliberately does NOT fall back to "the first arbitrary `monitoring` issue": the
 * handler edit-overwrites the body it returns, so reusing an unrelated issue the
 * user labeled `monitoring` for their own tracking would clobber it. When none is
 * ours, returns `{ ok:true, issue:null }` so the handler files its own dedicated
 * summary. `ok:false` means the read FAILED (do NOT then create — a transient `gh`
 * blip would file a duplicate; sentinel rule).
 */
export async function findMonitoringIssue({ cwd = PATHS.root, exec = runCli } = {}) {
  // High limit so our long-lived summary (kept at its original creation date)
  // can't fall out of a small recency window on an install that accumulates many
  // open `monitoring`-labeled issues — which would defeat dedup and file a fresh
  // duplicate. gh sorts newest-first; the summary sinks over time.
  const { code, stdout } = await exec('gh',
    ['issue', 'list', '--label', MONITORING_LABEL, '--state', 'open', '--limit', '200', '--json', 'number,title,body,labels,url'],
    { cwd })
  if (code !== 0) return { ok: false, issue: null }
  if (!stdout.trim()) return { ok: true, issue: null }
  const parsed = safeJSONParse(stdout, null, { logError: false })
  if (!Array.isArray(parsed)) return { ok: false, issue: null }
  const norm = parsed.map(i => ({
    number: i.number ?? null,
    title: i.title || '',
    body: i.body || '',
    url: i.url || null,
    labels: (i.labels || []).map(l => (typeof l === 'string' ? l : l?.name)).filter(Boolean)
  }))
  const ours = norm.find(i => i.body.includes(SLUG_MARKER)) || norm.find(i => i.title === STABLE_TITLE)
  return { ok: true, issue: ours || null }
}

/**
 * Ensure the labels exist before the first `issue create` (gh fails creating an
 * issue with a non-existent label). Idempotent — `--force` is a no-op when the
 * label already exists.
 */
export async function ensureDiagnosticsLabels({ cwd = PATHS.root, exec = runCli } = {}) {
  const labels = [
    { name: MONITORING_LABEL, color: '0e8a16', desc: 'Persistent monitoring / self-diagnostics summary issue' },
    { name: NEEDS_ATTENTION_LABEL, color: 'd93f0b', desc: 'Flagged by CoS self-diagnostics — needs human diagnosis' }
  ]
  for (const l of labels) {
    await exec('gh', ['label', 'create', l.name, '--color', l.color, '--description', l.desc, '--force'], { cwd })
  }
}

const MALFORMED_LABEL = /^(?:model:[^ ]+|effort:[^ ]+|planner:[^ ]+|plan|tests|bug|data-safety|upgrade-rehearsal|security|needs-decision) (?:(--color )?#?([0-9A-Fa-f]{6}))(?: .*)?$/

/**
 * Remove labels accidentally created by passing a whole "name color description"
 * string as one CLI argument. A label is deleted only after both issue and PR
 * searches succeed and report no usage; any ambiguous read is fail-closed.
 */
async function pruneMalformedLabels({ cwd = PATHS.root, exec = runCli } = {}) {
  const { code, stdout, stderr } = await exec('gh', ['label', 'list', '--limit', '500', '--json', 'name'], { cwd })
  if (code !== 0) {
    console.warn(`⚠️ Self-diagnostics: could not list labels for pruning: ${stderr.trim()}`)
    return { pruned: 0, skipped: 0 }
  }

  const labels = safeJSONParse(stdout, null, { logError: false })
  if (!Array.isArray(labels) || labels.some(label => typeof label?.name !== 'string')) {
    console.warn('⚠️ Self-diagnostics: could not parse labels for pruning')
    return { pruned: 0, skipped: 0 }
  }

  let pruned = 0
  let skipped = 0
  for (const { name } of labels) {
    const malformed = name.match(MALFORMED_LABEL)
    // Keep unrelated labels with number-like values (e.g. "ticket 123456").
    // Only the label prefixes known to have been produced by the broken shell
    // loops are candidates; a bare six-letter word such as "decade" is hex too.
    if (!malformed || (!malformed[1] && !/[0-9]/.test(malformed[2]))) continue
    // gh's --label filters split commas into multiple labels, so those names
    // cannot be checked safely through the issue/pr list commands.
    if (name.includes(',')) {
      skipped++
      console.warn(`⚠️ Self-diagnostics: cannot safely verify comma-containing malformed label ${JSON.stringify(name)}; leaving it in place`)
      continue
    }

    const usage = await Promise.all([
      exec('gh', ['issue', 'list', '--state', 'all', '--label', name, '--limit', '1', '--json', 'number'], { cwd }),
      exec('gh', ['pr', 'list', '--state', 'all', '--label', name, '--limit', '1', '--json', 'number'], { cwd })
    ])
    const [issues, pullRequests] = usage
    const issueNumbers = issues.code === 0 ? safeJSONParse(issues.stdout, null, { logError: false }) : null
    const prNumbers = pullRequests.code === 0 ? safeJSONParse(pullRequests.stdout, null, { logError: false }) : null
    if (!Array.isArray(issueNumbers) || !Array.isArray(prNumbers) ||
      issueNumbers.some(item => !Number.isInteger(item?.number)) ||
      prNumbers.some(item => !Number.isInteger(item?.number))) {
      skipped++
      console.warn(`⚠️ Self-diagnostics: could not verify usage for malformed label ${JSON.stringify(name)}; leaving it in place`)
      continue
    }
    if (issueNumbers.length || prNumbers.length) {
      skipped++
      const usages = [
        ...issueNumbers.map(item => `issue #${item.number}`),
        ...prNumbers.map(item => `PR #${item.number}`)
      ]
      console.warn(`⚠️ Self-diagnostics: malformed label ${JSON.stringify(name)} is in use by ${usages.join(', ')}; leaving it in place`)
      continue
    }

    const deleted = await exec('gh', ['label', 'delete', name, '--yes'], { cwd })
    if (deleted.code !== 0) {
      skipped++
      console.warn(`⚠️ Self-diagnostics: failed to delete malformed label ${JSON.stringify(name)}: ${deleted.stderr.trim()}`)
      continue
    }
    pruned++
  }

  console.log(`🧹 Pruned ${pruned} malformed labels`)
  return { pruned, skipped }
}

/**
 * Self-diagnostics handler (SCRIPT_HANDLERS['self-diagnostics']).
 *
 * @param {object} [deps]
 * @param {() => Promise<object>} [deps.loadLearning] - learning-data loader
 * @param {(cmd,args,opts)=>Promise<{code,stdout,stderr}>} [deps.exec] - CLI runner
 * @param {string} [deps.cwd] - repo root for `gh`
 * @param {() => string} [deps.now] - ISO clock (test seam)
 * @param {() => Promise<{ok:boolean,status:string}>} [deps.forgeGate] - gh probe (test seam)
 * @returns {Promise<{ failingCount, categories, issue, forgeUnavailable? }>}
 */
export async function runSelfDiagnostics({
  loadLearning = loadLearningData,
  exec = runCli,
  cwd = PATHS.root,
  now = () => new Date().toISOString(),
  forgeGate = null
} = {}) {
  const data = await loadLearning()
  // Window against the injected clock so tests are deterministic (issue #2617).
  const failing = computeFailingCategories(data?.byTaskType, { now: new Date(now()).getTime() })

  if (failing.length === 0) {
    console.log('🩺 Self-diagnostics: no self-healing categories with recent failures — all clear')
  } else {
    console.log(`🩺 Self-diagnostics: ${failing.length} failing categor${failing.length === 1 ? 'y' : 'ies'} (top: ${failing[0].slug} ${failing[0].failed}/${failing[0].total})`)
    for (const c of failing) console.log(formatCategoryLogLine(c))
  }

  // The metrics half above is local and always worth logging; everything below
  // reads or writes the forge. Probe once and skip that half rather than issuing
  // four `gh` calls that each fail into an ambiguous empty result (#3358).
  const gate = forgeGate || (async () => {
    const { ensureForgeReachable } = await import('../github.js')
    return ensureForgeReachable('self-diagnostics')
  })
  const forge = await gate()
  if (!forge?.ok) {
    return { failingCount: failing.length, categories: failing, issue: null, forgeUnavailable: true }
  }

  const generatedAt = now()
  const title = buildDiagnosticsIssueTitle()
  const body = buildDiagnosticsIssueBody(failing, { generatedAt })

  const found = await findMonitoringIssue({ cwd, exec })
  if (!found.ok) {
    // Read failed — refuse to file, or a transient gh blip duplicates the summary.
    console.warn('⚠️ Self-diagnostics: could not read monitoring issues (gh unavailable?) — skipping issue update')
    return { failingCount: failing.length, categories: failing, issue: null }
  }

  const hasFailures = failing.length > 0

  if (found.issue) {
    // Reuse the existing monitoring issue: refresh the body, and toggle the
    // attention label to match the current state (add on failures, clear on all-green).
    const editArgs = ['issue', 'edit', String(found.issue.number), '--body', body]
    const addingLabel = hasFailures && !found.issue.labels.includes(NEEDS_ATTENTION_LABEL)
    if (addingLabel) {
      await ensureDiagnosticsLabels({ cwd, exec })
      editArgs.push('--add-label', NEEDS_ATTENTION_LABEL)
    } else if (!hasFailures && found.issue.labels.includes(NEEDS_ATTENTION_LABEL)) {
      editArgs.push('--remove-label', NEEDS_ATTENTION_LABEL)
    }
    await pruneMalformedLabels({ cwd, exec })
    const { code, stderr } = await exec('gh', editArgs, { cwd })
    if (code !== 0) {
      console.warn(`⚠️ Self-diagnostics: failed to update issue #${found.issue.number}: ${stderr.trim()}`)
      return { failingCount: failing.length, categories: failing, issue: null }
    }
    console.log(`🩺 Self-diagnostics: updated monitoring issue #${found.issue.number} (${failing.length} failing)`)
    return { failingCount: failing.length, categories: failing, issue: { number: found.issue.number, url: found.issue.url, created: false } }
  }

  if (!hasFailures) {
    // Nothing failing and no existing summary — don't create noise.
    await pruneMalformedLabels({ cwd, exec })
    console.log('🩺 Self-diagnostics: no failures and no existing summary issue — nothing to file')
    return { failingCount: 0, categories: [], issue: null }
  }

  await ensureDiagnosticsLabels({ cwd, exec })
  await pruneMalformedLabels({ cwd, exec })
  const createArgs = ['issue', 'create', '--title', title, '--body', body, '--label', MONITORING_LABEL, '--label', NEEDS_ATTENTION_LABEL]
  const { code, stdout, stderr } = await exec('gh', createArgs, { cwd })
  if (code !== 0) {
    console.warn(`⚠️ Self-diagnostics: failed to create monitoring issue: ${stderr.trim()}`)
    return { failingCount: failing.length, categories: failing, issue: null }
  }
  const url = (stdout.trim().match(/(https?:\/\/\S+)/) || [])[1] || stdout.trim()
  const number = Number((url.match(/(\d+)\s*$/) || [])[1]) || null
  console.log(`🩺 Self-diagnostics: filed monitoring issue ${url} (${failing.length} failing)`)
  return { failingCount: failing.length, categories: failing, issue: { number, url, created: true } }
}