Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Post-merge gate audit for claim runs (#12549).
*
* The merge-admission lease and the "required CI is green" rule are prompt
* instructions, and a claim agent has full forge access, so nothing stops it
* merging around them. This runs after the run completes and asks the forge:
* did each PR the run merged hold the repository's admission lease, and were all
* checks on the merged head green before `mergedAt`? A miss is recorded on the
* run, in the run-events ledger and as a notification. When the flagged head's
* checks then fail, ONE recovery task is queued for the default branch so
* concurrent runs inherit a single fix rather than each writing their own.
*
* Detection only: it never merges, reverts or edits a repository.
*/
import { execGh } from './github.js';
import { resolveForgeExecOptions } from './forgeExecOptions.js';
import { emitLog } from './cosEvents.js';
import { appendRunEvent } from './agentRunEventLog.js';
import { addNotification, NOTIFICATION_TYPES, PRIORITY_LEVELS } from './notifications.js';
import { getAppById, getActiveApps, updateApp } from './apps.js';
import { RECOVERY_TASK_PREFIX } from './recoveryTasks.js';
import { getOriginInfo } from '../lib/gitRemote.js';
import { githubRepoSpec, githubApiHost } from '../lib/workTracker.js';
import { safeJSONParse } from '../lib/fileUtils.js';
import { isPlainObject } from '../lib/objects.js';
import { assessMergeGate, awaitsCoordinatorMark, classifyHeadChecks, foldHeadChecks, MERGE_GATE_VIOLATIONS } from '../lib/mergeGateAudit.js';
// A flagged head whose checks never settle is dropped after this long; the
// sweep owns it until then.
export const MERGE_GATE_WATCH_TTL_MS = 6 * 60 * 60 * 1000;
// Fast follow-up after a flag, so a head failing in minutes is reported in
// minutes; the 30-minute CoS sweep is the durable fallback after a restart.
export const MERGE_GATE_POLL_INTERVAL_MS = 2 * 60 * 1000;
const MERGE_GATE_POLL_CAP = 15;
const LIST_LIMIT = 50;
const SHORT_SHA = 12;
// The coordinator marks its receipt just after its merge call returns; a merge
// this recent whose receipt is still unmarked is judged on the next sweep.
const COORDINATOR_MARK_GRACE_MS = 2 * 60 * 1000;
const MERGED_PR_FIELDS = 'number,url,headRefOid,mergedAt,baseRefName';
const readList = (value) => (Array.isArray(value) ? value : []);
const auditKey = (pr) => `${pr.number}@${String(pr.headRefOid).toLowerCase()}`;
const watchKey = (repository, headSha) => `${repository}#${String(headSha).toLowerCase()}`;
const readWatches = (app) => readList(app?.mergeGateWatches).filter((watch) => isPlainObject(watch) && watch.key);
// Every attempt and every posted status, not the latest per name: the audit asks
// what existed at the merge, and a rerun or re-posted context since then replaces
// the answer. Commit statuses cover CI that reports outside the check-run API.
const CHECK_RUN_JQ = '.check_runs[] | {id, name, status, conclusion, suite: .check_suite.id, started_at, completed_at}';
const CHECK_SUITE_JQ = '.check_suites[] | {id, created_at}';
const STATUS_JQ = '.[] | {context, state, created_at}';
async function readGhLines({ origin, repoSpec, forgeExec, path, jq }) {
const raw = await execGh([
'api', '--hostname', githubApiHost(origin.host), '--paginate', path, '--jq', jq,
], undefined, { ...forgeExec, backoffKey: repoSpec }).catch(() => null);
if (raw === null) return null;
const rows = String(raw).split('\n').map((line) => line.trim()).filter(Boolean).map((line) => safeJSONParse(line, null));
return rows.every(isPlainObject) ? rows : null;
}
/** The checks of `headSha` as they stood at `mergedAt` (`foldHeadChecks`), or null when any read failed. */
async function readHeadCheckRuns({ origin, repoSpec, forgeExec, headSha, mergedAt }) {
const base = `repos/${origin.fullName}/commits/${headSha}`;
const [runs, suites, statuses] = await Promise.all([
readGhLines({ origin, repoSpec, forgeExec, path: `${base}/check-runs?filter=all&per_page=100`, jq: CHECK_RUN_JQ }),
readGhLines({ origin, repoSpec, forgeExec, path: `${base}/check-suites?per_page=100`, jq: CHECK_SUITE_JQ }),
readGhLines({ origin, repoSpec, forgeExec, path: `${base}/statuses?per_page=100`, jq: STATUS_JQ }),
]);
if (!runs || !suites || !statuses) return null;
return foldHeadChecks({ runs, suites, statuses }, mergedAt);
}
async function readMergedPrs({ repoSpec, forgeExec, branch, since }) {
const raw = await execGh([
'pr', 'list', '--repo', repoSpec, '--head', branch, '--state', 'merged',
'--json', MERGED_PR_FIELDS, '--limit', '5',
], undefined, { ...forgeExec, backoffKey: repoSpec }).catch(() => null);
const prs = raw === null ? null : safeJSONParse(raw, null);
if (!Array.isArray(prs)) return null;
// A reused branch name can carry an earlier run's PR.
return prs.filter((pr) => isPlainObject(pr) && /^[0-9a-f]{40}$/i.test(String(pr.headRefOid || ''))
&& Number.isFinite(Date.parse(pr.mergedAt)) && (since === null || Date.parse(pr.mergedAt) >= since));
}
async function getAgentRecord(agentId) {
const lifecycle = await import('./cosAgentLifecycle.js');
return lifecycle.getAgentRecord(agentId);
}
/** A lease the run still holds (it exited without releasing), from the trusted admission state. */
async function readHeldLease(agent, repository) {
const { readMergeAdmissionStateForSafetyCheck } = await import('./cosState.js');
const snapshot = await readMergeAdmissionStateForSafetyCheck().catch(() => null);
const lease = snapshot?.trusted ? snapshot.mergeAdmissions?.[repository] : null;
return isPlainObject(lease) && lease.kind === 'agent' && lease.agentId === agent.id
&& lease.startedAt === agent.startedAt && lease.taskId === agent.taskId ? lease : null;
}
function recoveryContext(watch, failing, appName) {
return [
`A pull request merged in ${appName} (${watch.repository}) before its head's checks finished green`
+ `${watch.codes.includes(MERGE_GATE_VIOLATIONS.NO_ADMISSION_LEASE) ? ' and without the merge-admission lease' : ''}, `
+ 'and those checks have now failed.',
'',
`Merged PR: ${watch.prUrl}`,
`Merged head: ${watch.headSha}`,
`Base branch: ${watch.baseBranch}`,
`Failing checks: ${failing.slice(0, 10).join(', ')}`,
`Merging run: ${watch.agentId}`,
'',
`Make ${watch.baseBranch} green again, in one pull request: either fix what the failing checks report or revert the merge.`,
'Other claim runs hit the same failure and may already be writing their own fixes. '
+ `Before changing anything, check open pull requests and recent commits on ${watch.baseBranch} for a fix that already landed or is in flight, `
+ 'and reuse it instead of writing a competing one.',
].join('\n');
}
async function fileRecoveryTask(watch, failing, app) {
const { addTask } = await import('./cosTaskStore.js');
const created = await addTask({
// First line is the queue's dedupe identity (base branch + head + app): no PR number, so two PRs backed by one head share a recovery.
description: `${RECOVERY_TASK_PREFIX} Default branch ${watch.baseBranch} is red after a PR merged past the merge gate (${watch.headSha.slice(0, SHORT_SHA)}) in ${app.name || watch.repository}`,
priority: 'HIGH',
app: app.id,
isRecovery: true,
metadata: { mergeGateRecovery: { repository: watch.repository, headSha: watch.headSha, prUrl: watch.prUrl, baseBranch: watch.baseBranch } },
context: recoveryContext(watch, failing, app.name || watch.repository),
useWorktree: true,
openPR: true,
}, 'user');
if (created?.duplicate) {
emitLog('info', `🚦 Merge-gate recovery for ${watch.headSha.slice(0, SHORT_SHA)} already queued as ${created.id}`, { agentId: watch.agentId });
return null;
}
emitLog('warn', `🚦 Queued merge-gate recovery for ${watch.baseBranch} (PR #${watch.prNumber}, ${failing.length} failing check(s))`, { agentId: watch.agentId, prUrl: watch.prUrl });
return created?.id || null;
}
// apps.json is one store: serialize this module's read-modify-write of its field.
let watchWriteTail = Promise.resolve();
function mutateWatches(appId, mutate) {
const next = watchWriteTail.catch(() => undefined).then(async () => {
const app = await getAppById(appId);
if (!app) return null;
return updateApp(appId, { mergeGateWatches: mutate(readWatches(app)) });
});
watchWriteTail = next;
return next;
}
async function notifyViolation(watch, violations, failing) {
const codes = violations.map((violation) => violation.message).join('; ');
return addNotification({
type: NOTIFICATION_TYPES.AGENT_WARNING,
priority: PRIORITY_LEVELS.HIGH,
title: `PR #${watch.prNumber} bypassed the merge gate`,
description: `Run ${watch.agentId} ${codes}.${failing.length ? ` ${failing.length} check(s) on the merged head already failed.` : ''}`,
link: watch.prUrl,
metadata: { agentId: watch.agentId, prNumber: watch.prNumber, headSha: watch.headSha },
}).catch((err) => console.error(`❌ Failed to notify about merge-gate violation on PR #${watch.prNumber}: ${err.message}`));
}
async function recordViolation(agent, watch, violations, checkNames) {
emitLog('warn', `🚦 Merge gate bypassed by ${agent.id}: PR #${watch.prNumber} ${violations.map((violation) => violation.message).join('; ')}`,
{ agentId: agent.id, taskId: agent.taskId, prUrl: watch.prUrl });
await appendRunEvent({
kind: 'run.merge-gate-violation',
runId: agent.metadata?.runId,
agentId: agent.id,
taskId: agent.taskId,
eventId: `merge-gate:${agent.id}:${watch.prNumber}:${watch.headSha}`,
data: { prNumber: watch.prNumber, headSha: watch.headSha, mergedAt: watch.mergedAt, codes: watch.codes, checks: checkNames.slice(0, 10) },
});
}
/**
* Audit the PRs a completed claim run merged.
*
* @param {object} params
* @param {string} params.agentId
* @param {object|null} params.task - the CoS task
* @param {object|null} params.agentState - the persisted agent record
* @returns {Promise<{ audited: number, violations: number, unresolved?: boolean, skipReason?: string }>}
*/
export async function auditClaimMergeGate({ agentId, task, agentState }) {
const metadata = agentState?.metadata || {};
const branches = [...new Set([...readList(metadata.claimBranches), metadata.claimBranch].filter((b) => typeof b === 'string' && b))];
const none = (skipReason) => ({ audited: 0, violations: 0, skipReason });
if (branches.length === 0) return none('not-a-claim-run');
if (!metadata.sourceWorkspace) return none('no-source-workspace');
const origin = await getOriginInfo(metadata.sourceWorkspace).catch(() => null);
const repoSpec = githubRepoSpec(origin);
if (!repoSpec) return none('not-a-github-repo');
const appId = task?.metadata?.app || null;
const app = appId ? await getAppById(appId).catch(() => null) : null;
const forgeExec = await resolveForgeExecOptions(metadata.sourceWorkspace, { forgeAccount: app?.forgeAccount });
const { repositoryKey } = await import('./cosMergeAdmission.js');
const repository = repositoryKey(origin);
const startedMs = Date.parse(agentState?.startedAt);
const since = Number.isFinite(startedMs) ? startedMs : null;
const audited = new Set(readList(metadata.mergeGateAudits));
const newAudits = [];
const found = [];
let unresolved = false;
for (const branch of branches) {
const prs = await readMergedPrs({ repoSpec, forgeExec, branch, since });
if (prs === null) { unresolved = true; continue; }
for (const pr of prs.filter((candidate) => !audited.has(auditKey(candidate)))) {
const headSha = String(pr.headRefOid).toLowerCase();
const checks = await readHeadCheckRuns({ origin, repoSpec, forgeExec, headSha, mergedAt: pr.mergedAt });
// No verdict without readable evidence; the durable sweep retries the audit.
if (checks === null) { unresolved = true; continue; }
// The coordinator may have merged and marked its receipt while the forge was
// being read, so judge against the run's record as it stands now.
const current = (await getAgentRecord(agentId).catch(() => null))?.metadata || metadata;
const receipts = readList(current.mergeHandoffReceipts);
if (awaitsCoordinatorMark({ receipts, prUrl: pr.url, headSha }) && Date.now() - Date.parse(pr.mergedAt) < COORDINATOR_MARK_GRACE_MS) {
unresolved = true;
continue;
}
const heldLease = await readHeldLease({ ...agentState, id: agentId }, repository);
const verdict = assessMergeGate({
repository, prUrl: pr.url, headSha, mergedAt: pr.mergedAt, checks, heldLease, receipts,
history: [...readList(current.mergeAdmissionHistory), ...(current.lastMergeAdmission ? [current.lastMergeAdmission] : [])],
});
newAudits.push(auditKey(pr));
if (verdict.violations.length === 0) continue;
found.push({ pr, headSha, verdict });
}
}
if (unresolved) await persistPendingAudit({ agentId, taskId: agentState?.taskId ?? null, app });
const violationRecords = [];
for (const { pr, headSha, verdict } of found) {
const codes = verdict.violations.map((violation) => violation.code);
const watch = {
key: watchKey(repository, headSha), repository, agentId, taskId: agentState?.taskId ?? null,
prUrl: pr.url, prNumber: pr.number, headSha, baseBranch: pr.baseRefName || 'the default branch',
mergedAt: pr.mergedAt, flaggedAt: new Date().toISOString(), codes,
};
const checkNames = verdict.checks.notGreenAtMerge;
await recordViolation({ ...agentState, id: agentId }, watch, verdict.violations, checkNames);
await notifyViolation(watch, verdict.violations, verdict.checks.failing);
violationRecords.push({ prUrl: pr.url, prNumber: pr.number, headSha, mergedAt: pr.mergedAt, detectedAt: watch.flaggedAt,
codes, notGreenAtMerge: checkNames.slice(0, 10) });
if (app && verdict.checks.readable) await watchFlaggedHead(watch, verdict.checks, app);
}
if (newAudits.length > 0 || violationRecords.length > 0) {
const { updateAgent } = await import('./cosAgentLifecycle.js');
await updateAgent(agentId, { metadata: {
mergeGateAudits: [...audited, ...newAudits].slice(-LIST_LIMIT),
...(violationRecords.length ? { mergeGateViolations: [...readList(metadata.mergeGateViolations), ...violationRecords].slice(-LIST_LIMIT) } : {}),
} }).catch((err) => emitLog('warn', `🚦 Could not record merge-gate audit on ${agentId}: ${err.message}`, { agentId }));
}
return { audited: newAudits.length, violations: found.length, unresolved };
}
// The forge could not be read: keep the audit as work for the durable sweep
// rather than losing the run's only chance of being checked.
async function persistPendingAudit({ agentId, taskId, app }) {
if (!app) {
emitLog('warn', `🚦 Merge-gate audit for ${agentId} could not read the forge and has no app to retry under`, { agentId });
return;
}
const key = `audit:${agentId}`;
await mutateWatches(app.id, (existing) => (existing.some((entry) => entry.key === key) ? existing
: [...existing, { key, kind: 'audit', agentId, taskId, flaggedAt: new Date().toISOString() }]))
.catch((err) => emitLog('warn', `🚦 Could not queue a merge-gate audit retry for ${agentId}: ${err.message}`, { agentId }));
}
// Already-failed checks queue the recovery now; checks still running arm a watch
// that the sweep resolves. Settled-and-green needs no recovery: the default
// branch is not red.
async function watchFlaggedHead(watch, checks, app) {
if (checks.failing.length > 0) {
// A failed queue write keeps the watch, so the sweep retries it.
const queued = await fileRecoveryTask(watch, checks.failing, app).then(() => true, (err) => {
emitLog('warn', `🚦 Merge-gate recovery for PR #${watch.prNumber} not queued: ${err.message}`, { agentId: watch.agentId });
return false;
});
if (queued) return;
} else if (checks.settled) return;
await mutateWatches(app.id, (existing) => (existing.some((entry) => entry.key === watch.key) ? existing : [...existing, watch]))
.catch((err) => emitLog('warn', `🚦 Could not watch flagged head of PR #${watch.prNumber}: ${err.message}`, { agentId: watch.agentId }));
scheduleWatchPolls(app.id);
}
const pollers = new Map();
// Bounded fast poll after a flag; every callback owns its rejections.
function scheduleWatchPolls(appId) {
if (pollers.has(appId)) return;
let polls = 0;
const tick = () => {
polls += 1;
sweepMergeGateWatches({ appId }).then((result) => {
if (result.remaining === 0 || polls >= MERGE_GATE_POLL_CAP) pollers.delete(appId);
else pollers.set(appId, setTimeout(tick, MERGE_GATE_POLL_INTERVAL_MS).unref());
}).catch((err) => {
console.error(`❌ Merge-gate watch poll for app ${appId} failed: ${err.message}`);
pollers.delete(appId);
});
};
pollers.set(appId, setTimeout(tick, MERGE_GATE_POLL_INTERVAL_MS).unref());
}
/** Test seam: stop any armed fast polls. */
export function __cancelMergeGatePolls() {
for (const timer of pollers.values()) clearTimeout(timer);
pollers.clear();
}
// One sweep per scope: a poll for one app must not borrow another app's result.
const sweepsInFlight = new Map();
// True once the audit ran to a verdict. A run whose record is gone cannot be
// audited again; the entry waits out its TTL like any other unreadable evidence.
async function retryPendingAudit(watch, app) {
const agentState = await getAgentRecord(watch.agentId).catch(() => null);
if (!agentState) return false;
const result = await auditClaimMergeGate({ agentId: watch.agentId, task: { id: watch.taskId, metadata: { app: app.id } }, agentState })
.catch((err) => {
emitLog('warn', `🚦 Merge-gate audit retry for ${watch.agentId} failed: ${err.message}`, { agentId: watch.agentId });
return { unresolved: true };
});
return !result.unresolved;
}
async function sweepApp(app) {
const watches = readWatches(app);
if (watches.length === 0) return { checked: 0, recovered: 0, remaining: 0 };
const origin = await getOriginInfo(app.repoPath).catch(() => null);
const repoSpec = githubRepoSpec(origin);
if (!repoSpec) return { checked: 0, recovered: 0, remaining: watches.length };
const forgeExec = await resolveForgeExecOptions(app.repoPath, { forgeAccount: app.forgeAccount });
const settledKeys = new Set();
let recovered = 0;
for (const watch of watches) {
if (Date.now() - Date.parse(watch.flaggedAt) >= MERGE_GATE_WATCH_TTL_MS) {
emitLog('warn', `🚦 Dropping merge-gate ${watch.kind === 'audit' ? `audit of ${watch.agentId}: the forge stayed unreadable` : `watch for PR #${watch.prNumber}: checks did not settle`} in time`, { agentId: watch.agentId });
settledKeys.add(watch.key);
continue;
}
if (watch.kind === 'audit') {
if (await retryPendingAudit(watch, app)) settledKeys.add(watch.key);
continue;
}
const checks = classifyHeadChecks(await readHeadCheckRuns({ origin, repoSpec, forgeExec, headSha: watch.headSha, mergedAt: watch.mergedAt }), watch.mergedAt);
if (checks.failing.length > 0) {
const queued = await fileRecoveryTask(watch, checks.failing, app).then(() => true, (err) => {
emitLog('warn', `🚦 Merge-gate recovery for PR #${watch.prNumber} not queued: ${err.message}`, { agentId: watch.agentId });
return false;
});
if (queued) { settledKeys.add(watch.key); recovered += 1; }
} else if (checks.readable && checks.settled) {
settledKeys.add(watch.key);
}
}
if (settledKeys.size > 0) await mutateWatches(app.id, (current) => current.filter((entry) => !settledKeys.has(entry.key)));
return { checked: watches.length, recovered, remaining: watches.length - settledKeys.size };
}
/**
* Resolve every flagged head still being watched: failing checks queue the one
* recovery task, settled green checks close the watch.
*
* @param {{ appId?: string }} [options] limit to one app
* @returns {Promise<{ checked: number, recovered: number, remaining: number }>}
*/
export function sweepMergeGateWatches({ appId = null } = {}) {
// A concurrent sweep of the same scope joins the one in flight; recovery dedupe rides on the task queue.
const scope = appId || '*';
if (sweepsInFlight.has(scope)) return sweepsInFlight.get(scope);
const sweep = (async () => {
const totals = { checked: 0, recovered: 0, remaining: 0 };
const apps = appId ? [await getAppById(appId).catch(() => null)].filter(Boolean) : await getActiveApps().catch(() => []);
for (const app of apps.filter((candidate) => readWatches(candidate).length > 0)) {
const result = await sweepApp(app).catch((err) => {
console.error(`❌ Merge-gate watch sweep failed for ${app.name}: ${err.message}`);
return { checked: 0, recovered: 0, remaining: readWatches(app).length };
});
totals.checked += result.checked;
totals.recovered += result.recovered;
totals.remaining += result.remaining;
}
return totals;
})().finally(() => { sweepsInFlight.delete(scope); });
sweepsInFlight.set(scope, sweep);
return sweep;
}