Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): harden release packaging retries (#4502)

Andrew Mikofalvy997d886be0d2
diff --git a/.github/scripts/build-smoke-alert-payload.mjs b/.github/scripts/build-smoke-alert-payload.mjs
index e8f6221cc..006745c7b 100644
--- a/.github/scripts/build-smoke-alert-payload.mjs
+++ b/.github/scripts/build-smoke-alert-payload.mjs
@@ -64,6 +64,17 @@ function summaryLine(tag, verdict, blockedBy) {
return `🚨 ${ALERT_HEADLINE}: ${tag} ${subject} β€” nothing shipped`;
}
+function recoveryGuidance() {
+ return [
+ "*Recovery guidance:* Inspect the failed packaging job's final `outcome=`.",
+ 'With no newer stable release, `transient-exhausted`, `deadline`, or `signal` can re-fire the same tag; investigate a recurring deadline.',
+ '`unknown-exhausted` requires manual log review. `terminal` requires fixing the cause in `reason=`.',
+ '`reason=rule:download-integrity` is the terminal exception: re-fire once on a fresh runner, then stop and investigate upstream if the same artifact fails again.',
+ '`cleanup-failure` reports its runner/tooling cause in `cleanup=`; `spawn-failure`, `attempt-timeout`, and `child-signal` are also runner/tooling failures. Confirm the runner/tooling failure is absent before re-firing.',
+ 'Never re-fire a tag once a newer stable release has shipped; cut a new release from `main` instead.',
+ ].join('\n');
+}
+
function bodyLines({ tag, verdict, reason, runUrl, repo, blockedBy }) {
const { detail, smokePassed } = describeBlock({ verdict, blockedBy });
const other = String(blockedBy ?? '').trim();
@@ -76,7 +87,8 @@ function bodyLines({ tag, verdict, reason, runUrl, repo, blockedBy }) {
: `*Why:* ${other} failed (${reason}).`
: `*Why:* ${reason}`,
'*State:* the GitHub Release is still a DRAFT and npm `latest` has NOT moved.',
- `*Recovery (fix the cause above first β€” re-firing alone repairs nothing):*\n\`${recoveryCommand(tag, repo)}\``,
+ recoveryGuidance(),
+ `*Recovery command:*\n\`${recoveryCommand(tag, repo)}\``,
`*Run:* ${runUrl}`,
];
}
diff --git a/.github/scripts/build-smoke-alert-payload.test.mjs b/.github/scripts/build-smoke-alert-payload.test.mjs
index aeb2f18d7..e03d7e2d9 100644
--- a/.github/scripts/build-smoke-alert-payload.test.mjs
+++ b/.github/scripts/build-smoke-alert-payload.test.mjs
@@ -9,12 +9,14 @@ import {
parseArgs,
recoveryCommand,
} from './build-smoke-alert-payload.mjs';
+import { STOP_OUTCOMES } from './retry-transient.mjs';
const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const desktopRelease = readFileSync(
join(REPO_ROOT, '.github', 'workflows', 'desktop-release.yml'),
'utf8',
);
+const SHELL_DOLLAR = '$';
const alertStep = (source = desktopRelease) => {
const start = source.indexOf('- name: Alert on a blocked release');
@@ -62,10 +64,23 @@ describe('alert content', () => {
expect(recoveryCommand('v9.9.9', 'acme/fork')).toContain('repos/acme/fork/dispatches');
});
- test('recovery points at the cause instead of promising a repair-free retry', () => {
+ test('recovery maps every packaging outcome to an operator action', () => {
const body = buildSlackPayload(base).blocks[1].text.text;
expect(body).toContain('*Recovery');
- expect(body).toContain('fix the cause');
+ expect(body).toContain("Inspect the failed packaging job's final `outcome=`");
+ for (const outcome of STOP_OUTCOMES) {
+ expect(body).toContain(`\`${outcome}\``);
+ }
+ expect(body).toContain('no newer stable release');
+ expect(body).toContain('`transient-exhausted`, `deadline`, or `signal` can re-fire');
+ expect(body).toContain('`unknown-exhausted` requires manual log review');
+ expect(body).toContain('`terminal` requires fixing the cause in `reason=`');
+ expect(body).toContain('`reason=rule:download-integrity` is the terminal exception');
+ expect(body).toContain('stop and investigate upstream');
+ expect(body).toContain('`cleanup-failure` reports its runner/tooling cause in `cleanup=`');
+ expect(body).toContain('Confirm the runner/tooling failure is absent before re-firing');
+ expect(body).toContain('Never re-fire a tag once a newer stable release has shipped');
+ expect(body).not.toContain('re-firing alone repairs nothing');
expect(body).not.toContain('no manual repair needed');
});
});
@@ -252,16 +267,18 @@ describe('workflow wiring', () => {
test('an unset webhook is a notice-level skip, and a failed POST is a warning', () => {
const step = alertStep();
- expect(step).toContain('::notice::${label} webhook not set');
+ expect(step).toContain(`::notice::${SHELL_DOLLAR}{label} webhook not set`);
expect(step).toMatch(/if \[\[ -z "\$webhook" \]\]; then[\s\S]{0,200}?return 0/);
- expect(step).toContain('::warning::${label} smoke alert failed to POST');
- expect(step).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');
+ expect(step).toContain(`::warning::${SHELL_DOLLAR}{label} smoke alert failed to POST`);
+ expect(step).toContain(
+ `post "${SHELL_DOLLAR}{SLACK_RELEASES_WEBHOOK_URL:-${SHELL_DOLLAR}{SLACK_WEBHOOK_URL:-}}" Slack`,
+ );
});
test('a passing smoke verdict is not coerced into an error', () => {
const step = alertStep();
expect(step).not.toMatch(/if \[\[ "\$VERDICT" != "fail" \]\]/);
- expect(step).toContain('VERDICT="${SMOKE_VERDICT:-error}"');
+ expect(step).toContain(`VERDICT="${SHELL_DOLLAR}{SMOKE_VERDICT:-error}"`);
expect(step).toMatch(/skipped.*\]\]; then\s*\n\s*VERDICT="error"/);
});
@@ -302,7 +319,9 @@ describe('workflow wiring', () => {
test('the annotation is emitted in addition to the page, not instead of it', () => {
const step = alertStep();
- const slackAt = step.indexOf('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');
+ const slackAt = step.indexOf(
+ `post "${SHELL_DOLLAR}{SLACK_RELEASES_WEBHOOK_URL:-${SHELL_DOLLAR}{SLACK_WEBHOOK_URL:-}}" Slack`,
+ );
const annotationAt = step.indexOf('::error::RELEASE BLOCKED');
expect(slackAt).toBeGreaterThan(-1);
expect(annotationAt).toBeGreaterThan(slackAt);
diff --git a/.github/scripts/retry-transient.mjs b/.github/scripts/retry-transient.mjs
index 1332126af..b8cb7a003 100644
--- a/.github/scripts/retry-transient.mjs
+++ b/.github/scripts/retry-transient.mjs
@@ -1,138 +1,947 @@
#!/usr/bin/env node
import { spawn } from 'node:child_process';
+import { StringDecoder } from 'node:string_decoder';
import { pathToFileURL } from 'node:url';
export const DEFAULT_MAX_ATTEMPTS = 3;
+export const STOP_OUTCOMES = Object.freeze([
+ 'terminal',
+ 'transient-exhausted',
+ 'unknown-exhausted',
+ 'deadline',
+ 'signal',
+ 'attempt-timeout',
+ 'child-signal',
+ 'spawn-failure',
+ 'cleanup-failure',
+]);
-export const TRANSIENT_SIGNATURES = [
- 'A timestamp was expected but was not found',
- 'The timestamp service is not available',
- 'HTTPError\\(statusCode: nil',
- 'The request timed out',
- 'NSURLErrorDomain Code=-100[13459]',
- 'kCFErrorDomainCFNetwork',
- 'The network connection was lost',
- 'Could not connect to the server',
+const MAX_CAPTURE_BYTES = 16_384;
+const MIN_PROJECTED_ATTEMPT_MS = 1_000;
+const DEFAULT_ATTEMPT_TIMEOUT_MS = 15 * 60 * 1000;
+const DEFAULT_CLEANUP_GRACE_MS = 5_000;
+const DEFAULT_CLEANUP_RESERVE_MS = 15_000;
+const DEFAULT_POLL_INTERVAL_MS = 25;
+const STOP_OUTCOME_SET = new Set(STOP_OUTCOMES);
+const TRANSIENT_HTTP_STATUSES = new Set([408, 429, 500, 502, 503, 504, 521, 522, 524]);
+const TERMINAL_HTTP_STATUSES = new Set([400, 401, 404, 413, 422]);
+const NETWORK_CODES = [
+ 'EAI_AGAIN',
+ 'ENOTFOUND',
+ 'ECONNREFUSED',
+ 'ENETUNREACH',
+ 'EHOSTUNREACH',
+ 'ECONNABORTED',
'ECONNRESET',
'ETIMEDOUT',
'EPIPE',
- 'socket hang up',
- 'Client network socket disconnected',
- 'unexpected EOF',
- '502 Bad Gateway',
- '503 Service Unavailable',
- '504 Gateway Time-?out',
- 'You have exceeded a secondary rate limit',
- 'abuse detection mechanism',
- 'was submitted too quickly',
];
+// STOP: Order selects the operator-facing reason; keep root causes first and download-integrity last.
+const TERMINAL_RULES = [
+ ['disk-full', /\bENOSPC\b|no space left on device/i],
+ [
+ 'compiler',
+ /(?:\berror TS\d+:|\b(?:compilation|electron-vite build|vite build|Rolldown build) failed\b)/i,
+ ],
+ [
+ 'configuration',
+ /[\r\n]\s*(?:β¨―\s*|(?:Error|FATAL):?\s*)?(?:invalid configuration object|configuration (?:is invalid|validation failed)|cannot find configuration|unknown (?:configuration )?option|ERR_PNPM_OUTDATED_LOCKFILE)\b/i,
+ ],
+ [
+ 'credentials',
+ /\b(?:authentication failed|unauthorized credentials?|invalid credentials?|credentials? (?:are )?(?:missing|expired)|CSC_LINK is not set)\b/i,
+ ],
+ [
+ 'certificate',
+ /\b(?:certificate (?:has )?(?:expired|not trusted|revoked)|CSSMERR_TP_CERT_|no (?:valid )?signing identity|specified item could not be found in the keychain|SecKeychainUnlock)\b/i,
+ ],
+ [
+ 'notarization-invalid',
+ /(?:notari[sz]ation[^\r\n]*\bstatus["']?\s*[:=]\s*["']?Invalid\b|\bstatus["']?\s*[:=]\s*["']?Invalid\b[^\r\n]*notari[sz]|\bnotarytool\b[\s\S]{0,200}?\bstatus["']?\s*[:=]\s*["']?Invalid\b)/i,
+ ],
+ [
+ 'entitlement',
+ /(?:\bentitlements?\b[^\r\n]*(?:not permitted|mismatch|rejected|failed)|(?:invalid|rejected)[^\r\n]*\bentitlements?\b)/i,
+ ],
+ [
+ 'fuse',
+ /\bOK_PACKAGING_FUSE_FAILURE\b|\[afterSign\]\s+getCurrentFuseWire\s+failed\b|(?:\[after(?:Pack|Sign)\]\s+|\belectron\s+)fuse(?:s)?\b[^\r\n]*(?:\bmismatch(?:es|ed)?\b|\binvalid\b|\bfailed\b)/i,
+ ],
+ ['tls-trust', /\bNSURLErrorDomain\b[^\r\n]*\bCode=(?:-1201|-1202|-1203|-1204|-1205|-1206)\b/i],
+ [
+ 'integrity',
+ /\b(?:integrity check failed|code signature invalid|signature verification failed)\b/i,
+ ],
+ [
+ 'missing-executable',
+ /(?:\bspawn\s+\S+\s+ENOENT\b|[\r\n]\s*Error:\s*Cannot find module\b|\bERR_PNPM_RECURSIVE_EXEC_FIRST_FAIL\b[^\r\n]*\bCommand\b[^\r\n]*\bnot found\b|[\r\n]\s*\S{0,256}(?:ba)?sh(?:\.exe)?:[^\r\n]{0,512}:\s*command not found\b)/i,
+ ],
+ ['download-integrity', /\b(?:checksum|hash) mismatch,\s*expected\b/i],
+];
+const TERMINAL_RULE_IDS = Object.freeze(TERMINAL_RULES.map(([id]) => id));
+const TRANSIENT_RULES = [
+ ['timestamp-missing', /A timestamp was expected but was not found/i],
+ ['timestamp-service', /The timestamp service is not available/i],
+ ['http-status-nil', /HTTPError\s*\(\s*statusCode\s*:\s*nil/i],
+ ['request-timeout', /\bThe request timed out\b/i],
+ ['network-lost', /\bThe network connection was lost\b/i],
+ ['server-connect', /\bCould not connect to the server\b/i],
+ ['socket-hangup', /\bsocket hang up\b/i],
+ ['tls-disconnect', /\bClient network socket disconnected\b/i],
+ ['unexpected-eof', /\bunexpected EOF\b/i],
+ ['secondary-rate-limit', /\b(?:secondary rate limit|rate limit exceeded)\b/i],
+ ['rate-limit-header', /\bX-RateLimit-Remaining\s*:\s*0\b/i],
+ ['abuse-limit', /\babuse detection mechanism\b/i],
+ ['submitted-too-fast', /\bwas submitted too quickly\b/i],
+];
+
+function appendUtf8Tail(current, chunk, limit) {
+ const combined = current.length === 0 ? chunk : Buffer.concat([current, chunk]);
+ if (combined.length <= limit) return combined;
+ let start = combined.length - limit;
+ while (start < combined.length && (combined[start] & 0xc0) === 0x80) start += 1;
+ return combined.subarray(start);
+}
+
+function collectStructuredEvidence(evidence, text) {
+ const statusPattern =
+ /(?:\bHTTP(?:Error)?(?:\/[0-9.]+)?(?:\s+status)?\s*[:=]?\s*|\bresponse\s+(?:status|code)\s*[:=]?\s*|\bstatusCode["']?\s*[:=]\s*)(\d{3})\b/gi;
+ for (const match of text.matchAll(statusPattern)) {
+ evidence.httpStatuses.add(Number(match[1]));
+ }
+ for (const match of text.matchAll(
+ /[\r\n]\s*(?:β¨―\s*|Error:\s*)?(408|429|500|502|503|504|521|522|524)\s+(?:Request Timeout|Too Many Requests|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Time-?out|Web Server Is Down|Connection Timed Out|A Timeout Occurred)\b/gi,
+ )) {
+ evidence.httpStatuses.add(Number(match[1]));
+ }
+ const networkPattern = new RegExp(`\\b(${NETWORK_CODES.join('|')})\\b`, 'gi');
+ for (const match of text.matchAll(networkPattern)) {
+ evidence.errorCodes.add(match[1].toUpperCase());
+ }
+ for (const match of text.matchAll(/\b(UND_ERR_(?:[A-Z_]*TIMEOUT|SOCKET))\b/gi)) {
+ evidence.errorCodes.add(match[1].toUpperCase());
+ }
+ for (const match of text.matchAll(/\bcurl:\s*\((28|56)\)/gi)) {
+ evidence.errorCodes.add(`CURL_${match[1]}`);
+ }
+ for (const match of text.matchAll(
+ /\bNSURLErrorDomain\b[^\r\n]*\bCode=(-1001|-1003|-1004|-1005|-1006|-1008|-1009|-1011)\b/gi,
+ )) {
+ evidence.errorCodes.add(`NSURLError_${match[1]}`);
+ }
+}
+
+function collectRetryAfterEvidence(evidence, text) {
+ if (evidence.retryAfterValid) return;
+ const match = /^\s*(?:<\s*)?Retry-After\s*:\s*(.*?)\s*$/i.exec(text);
+ if (!match) return;
+ const raw = match[1].replace(/^["']|["']$/g, '');
+ const retryAt = /^\d+$/.test(raw) ? evidence.nowFn() + Number(raw) * 1000 : Date.parse(raw);
+ if (!Number.isFinite(retryAt)) return;
+ evidence.retryAfterValid = true;
+ evidence.retryAfterEpochMs = retryAt;
+}
+
+function collectDecodedEvidence(evidence, decoded, final) {
+ const joined = evidence.scanCarry + decoded;
+ const scanText = evidence.scanStarted ? joined : `\n${joined}`;
+ collectStructuredEvidence(evidence, scanText);
+ collectPhraseEvidence(evidence, scanText);
+ evidence.scanCarry = joined.slice(-2048);
+ evidence.scanStarted ||= joined.length > 0;
+
+ const retryText = evidence.retryAfterCarry + decoded;
+ const retryLines = retryText.split(/\r\n|\r|\n/);
+ evidence.retryAfterCarry = final ? '' : (retryLines.pop() ?? '').slice(-2048);
+ for (const line of retryLines) collectRetryAfterEvidence(evidence, line);
+}
+
+function collectPhraseEvidence(evidence, text) {
+ for (const [id, pattern] of TERMINAL_RULES) {
+ if (pattern.test(text)) evidence.terminalRules.add(id);
+ }
+ for (const [id, pattern] of TRANSIENT_RULES) {
+ if (pattern.test(text)) evidence.transientRules.add(id);
+ }
+}
+
+export class FailureEvidence {
+ constructor({ maxTailBytes = MAX_CAPTURE_BYTES, nowFn = Date.now } = {}) {
+ this.httpStatuses = new Set();
+ this.errorCodes = new Set();
+ this.terminalRules = new Set();
+ this.transientRules = new Set();
+ this.retryAfterEpochMs = undefined;
+ this.retryAfterValid = false;
+ this.tailBytes = Buffer.alloc(0);
+ this.tail = '';
+ this.maxTailBytes = maxTailBytes;
+ this.nowFn = nowFn;
+ this.decoder = new StringDecoder('utf8');
+ this.scanCarry = '';
+ this.scanStarted = false;
+ this.retryAfterCarry = '';
+ }
+
+ ingest(value) {
+ const chunk = Buffer.isBuffer(value) ? value : Buffer.from(String(value));
+ this.tailBytes = appendUtf8Tail(this.tailBytes, chunk, this.maxTailBytes);
+ collectDecodedEvidence(this, this.decoder.write(chunk), false);
+ }
+
+ finish() {
+ collectDecodedEvidence(this, this.decoder.end(), true);
+ this.tail = this.tailBytes.toString('utf8');
+ return this;
+ }
+}
+
+class AttemptResult {
+ constructor({
+ code = null,
+ closeSignal = null,
+ cancellationSignal = null,
+ cancelled = false,
+ timedOut = false,
+ deadlineExpired = false,
+ spawnError = null,
+ cleanup = { ok: true, reason: 'clean' },
+ evidence = new FailureEvidence(),
+ } = {}) {
+ this.code = code;
+ this.closeSignal = closeSignal;
+ this.cancellationSignal = cancellationSignal;
+ this.cancelled = cancelled;
+ this.timedOut = timedOut;
+ this.deadlineExpired = deadlineExpired;
+ this.spawnError = spawnError;
+ this.cleanup = cleanup;
+ this.evidence = evidence;
+ }
+}
+
+class RetryState {
+ constructor() {
+ this.attemptsStarted = 0;
+ this.unknownRetryUsed = false;
+ this.lastFailureClassification = undefined;
+ this.lastAttemptDurationMs = undefined;
+ }
+}
+
+export function classifyEvidence(evidence) {
+ if (evidence.terminalRules.size > 0) return 'terminal';
+ for (const status of evidence.httpStatuses) {
+ if (TERMINAL_HTTP_STATUSES.has(status)) return 'terminal';
+ }
+ const rateLimited =
+ evidence.transientRules.has('secondary-rate-limit') ||
+ evidence.transientRules.has('rate-limit-header') ||
+ evidence.transientRules.has('abuse-limit') ||
+ evidence.transientRules.has('submitted-too-fast');
+ if (evidence.httpStatuses.has(403) && !rateLimited) return 'terminal';
+ if (evidence.errorCodes.size > 0 || evidence.transientRules.size > 0) return 'transient';
+ for (const status of evidence.httpStatuses) {
+ if (TRANSIENT_HTTP_STATUSES.has(status)) return 'transient';
+ }
+ if (evidence.httpStatuses.has(403) && rateLimited) return 'transient';
+ return 'unknown';
+}
+
+function firstSorted(values) {
+ return [...values].sort()[0];
+}
+
+function firstTerminalRule(values) {
+ return TERMINAL_RULE_IDS.find((id) => values.has(id));
+}
+
+function matchedReason(evidence, classification) {
+ const statuses = [...evidence.httpStatuses].sort((left, right) => left - right);
+ if (classification === 'terminal') {
+ const terminalRule = firstTerminalRule(evidence.terminalRules);
+ if (terminalRule) return `rule:${terminalRule}`;
+ const status = statuses.find(
+ (candidate) => TERMINAL_HTTP_STATUSES.has(candidate) || candidate === 403,
+ );
+ return `http:${status}`;
+ }
+ if (classification === 'transient') {
+ const errorCode = firstSorted(evidence.errorCodes);
+ if (errorCode) return `code:${errorCode}`;
+ const transientRule = firstSorted(evidence.transientRules);
+ if (transientRule) return `rule:${transientRule}`;
+ const status = statuses.find((candidate) => TRANSIENT_HTTP_STATUSES.has(candidate));
+ return `http:${status}`;
+ }
+ return 'diagnostic:unknown';
+}
+
+function redactDiagnosticTail(tail) {
+ const withoutPem = tail.replace(
+ /(?:-----BEGIN [^-\r\n]+-----[\s\S]*?(?:-----END [^-\r\n]+-----|$)|(?:[A-Z0-9+/=]{20,}\r?\n)+-----END [^-\r\n]+-----)/gi,
+ '[REDACTED PEM]',
+ );
+ return withoutPem.replace(
+ /^(\s*["']?[A-Z0-9_.-]*(?:API[-_]?KEY|AUTHORIZATION|CREDENTIAL|PASSWORD|SECRET|TOKEN)[A-Z0-9_.-]*["']?\s*[:=]\s*)[^\r\n]*/gim,
+ '$1[REDACTED]',
+ );
+}
+
+function emitDiagnostic(log, label, evidence) {
+ log(`::group::${label} bounded failure diagnostic`);
+ const safe = redactDiagnosticTail(evidence.tail || '(no child diagnostic captured)');
+ for (const line of safe.split(/\r\n|\r|\n/)) log(`| ${line.replaceAll('##[', '# #[')}`);
+ log('::endgroup::');
+}
+
+function projectedAttemptMs(durationMs, attemptTimeoutMs) {
+ return Math.min(
+ attemptTimeoutMs,
+ Math.max(MIN_PROJECTED_ATTEMPT_MS, Math.ceil(durationMs * 1.25)),
+ );
+}
+
+function logDecision({
+ log,
+ level,
+ label,
+ action,
+ reason,
+ attempt,
+ maxAttempts,
+ result,
+ signal,
+ matched,
+ detail,
+ outcome,
+ classification,
+}) {
+ const annotation = level === 'warning' ? '::warning::' : '::error::';
+ const code = result?.code ?? 'none';
+ const signalFact = signal ?? result?.cancellationSignal ?? result?.closeSignal ?? 'none';
+ const matchedFact = matched ? ` matched=${matched}` : '';
+ const detailFact = detail ? ` ${detail}` : '';
+ const outcomeFact = outcome ? ` outcome=${outcome}` : '';
+ const classificationFact = classification ? ` classification=${classification}` : '';
+ const cleanupFact = result?.cleanup?.ok === false ? ` cleanup=${result.cleanup.reason}` : '';
+ log(
+ `${annotation}${label} decision=${action} reason=${reason}${outcomeFact}${classificationFact} attempt=${attempt}/${maxAttempts} code=${code} signal=${signalFact}${cleanupFact}${matchedFact}${detailFact}`,
+ );
+ if (action === 'stop' && !STOP_OUTCOME_SET.has(outcome)) {
+ if (result) emitDiagnostic(log, label, result.evidence);
+ throw new Error(`${label}: stop decision has unregistered outcome ${outcome ?? 'none'}`);
+ }
+}
-const TRANSIENT_RE = new RegExp(TRANSIENT_SIGNATURES.join('|'), 'i');
+function positiveJitterMs(minimumMs, randomFn) {
+ const ceiling = Math.max(1, Math.min(5_000, Math.ceil(minimumMs * 0.1)));
+ return Math.min(ceiling, Math.max(1, Math.floor(randomFn() * ceiling) + 1));
+}
+
+export function computeRetryDelayMs(
+ retryNumber,
+ evidence,
+ { randomFn = Math.random, nowMs = Date.now() } = {},
+) {
+ const baseMs = retryNumber === 1 ? 30_000 : 60_000;
+ const minimumMs = Math.max(baseMs, retryAfterDelayMs(evidence, nowMs) ?? 0);
+ return minimumMs + positiveJitterMs(minimumMs, randomFn);
+}
-export function isTransient(output) {
- return TRANSIENT_RE.test(String(output ?? ''));
+function retryAfterDelayMs(evidence, nowMs) {
+ if (!evidence.retryAfterValid) return undefined;
+ return Math.max(0, (evidence.retryAfterEpochMs ?? nowMs) - nowMs);
}
-export function backoffSeconds(attempt) {
- return attempt * 30;
+function delay(ms) {
+ return new Promise((resolve) => setTimeout(resolve, ms));
}
-const MAX_CAPTURE_BYTES = 2_000_000;
+async function waitUntil(predicate, timeoutMs, intervalMs, sleepFn) {
+ const end = Date.now() + timeoutMs;
+ while (!predicate()) {
+ if (Date.now() >= end) return false;
+ await sleepFn(Math.min(intervalMs, Math.max(1, end - Date.now())));
+ }
+ return true;
+}
+
+function processGroupExists(pid, killFn) {
+ try {
+ killFn(-pid, 0);
+ return true;
+ } catch (error) {
+ return error?.code !== 'ESRCH';
+ }
+}
-function appendCapped(buffer, chunk) {
- const next = buffer + chunk;
- return next.length > MAX_CAPTURE_BYTES ? next.slice(next.length - MAX_CAPTURE_BYTES) : next;
+function sendGroupSignal(pid, signal, killFn) {
+ try {
+ killFn(-pid, signal);
+ return true;
+ } catch (error) {
+ return error?.code === 'ESRCH';
+ }
}
-function runOnce(command, { shell = false, spawnFn = spawn } = {}) {
+function defaultTaskkill(args) {
return new Promise((resolve) => {
- const child = shell
- ? spawnFn('bash', ['-c', command[0]], { stdio: ['inherit', 'pipe', 'pipe'] })
- : spawnFn(command[0], command.slice(1), { stdio: ['inherit', 'pipe', 'pipe'] });
-
- let captured = '';
- child.stdout?.on('data', (d) => {
- const text = d.toString();
- captured = appendCapped(captured, text);
- process.stdout.write(text);
+ const child = spawn('taskkill', args, { stdio: 'ignore', windowsHide: true });
+ let settled = false;
+ child.once('error', () => {
+ if (!settled) resolve({ ok: false });
+ settled = true;
});
- child.stderr?.on('data', (d) => {
- const text = d.toString();
- captured = appendCapped(captured, text);
- process.stderr.write(text);
+ child.once('close', (code) => {
+ if (!settled) resolve({ ok: code === 0 });
+ settled = true;
});
+ });
+}
- child.on('error', (err) => {
- const text = `${err.message}\n`;
- captured = appendCapped(captured, text);
- process.stderr.write(text);
- resolve({ code: 1, output: captured });
+export function createOwnedTreeController({
+ platform = process.platform,
+ killFn = process.kill.bind(process),
+ taskkillFn = defaultTaskkill,
+ sleepFn = delay,
+ pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
+} = {}) {
+ if (platform === 'win32') {
+ return {
+ async cleanup(pid, { graceMs, waitForClose }) {
+ if (await waitForClose(0)) return { ok: true, reason: 'clean' };
+ const graceful = await taskkillFn(['/PID', String(pid), '/T']);
+ if (graceful.ok && (await waitForClose(graceMs))) {
+ return { ok: true, reason: 'clean' };
+ }
+ const forced = await taskkillFn(['/PID', String(pid), '/T', '/F']);
+ if (!forced.ok) return { ok: false, reason: 'taskkill-failure' };
+ if (!(await waitForClose(graceMs))) {
+ return { ok: false, reason: 'close-not-observed' };
+ }
+ return { ok: true, reason: 'clean' };
+ },
+ };
+ }
+ return {
+ async cleanup(pid, { graceMs, cleanupReserveMs, waitForClose }) {
+ if (!processGroupExists(pid, killFn)) {
+ return (await waitForClose(graceMs))
+ ? { ok: true, reason: 'clean' }
+ : { ok: false, reason: 'close-not-observed' };
+ }
+ if (!sendGroupSignal(pid, 'SIGTERM', killFn)) {
+ return { ok: false, reason: 'signal-send-failure' };
+ }
+ const goneAfterTerm = await waitUntil(
+ () => !processGroupExists(pid, killFn),
+ graceMs,
+ pollIntervalMs,
+ sleepFn,
+ );
+ if (!goneAfterTerm) {
+ if (!sendGroupSignal(pid, 'SIGKILL', killFn)) {
+ return { ok: false, reason: 'signal-send-failure' };
+ }
+ const goneAfterKill = await waitUntil(
+ () => !processGroupExists(pid, killFn),
+ cleanupReserveMs,
+ pollIntervalMs,
+ sleepFn,
+ );
+ if (!goneAfterKill) return { ok: false, reason: 'tree-survived-kill' };
+ }
+ return (await waitForClose(cleanupReserveMs))
+ ? { ok: true, reason: 'clean' }
+ : { ok: false, reason: 'close-not-observed' };
+ },
+ };
+}
+
+function waitForPromise(promise, timeoutMs) {
+ return new Promise((resolve) => {
+ const timeout = setTimeout(() => resolve(false), Math.max(0, timeoutMs));
+ promise.then(() => {
+ clearTimeout(timeout);
+ resolve(true);
+ });
+ });
+}
+
+function commandInvocation(command, shell, platform) {
+ if (!shell) return { executable: command[0], args: command.slice(1) };
+ if (platform === 'win32') {
+ return {
+ executable: 'bash',
+ args: ['-c', command[0]],
+ };
+ }
+ return { executable: '/bin/bash', args: ['-c', command[0]] };
+}
+
+async function runAttempt({
+ command,
+ shell,
+ spawnFn,
+ platform,
+ cancellationSignal,
+ deadlineEpochMs,
+ attemptTimeoutMs,
+ cleanupReserveMs,
+ cleanupGraceMs,
+ treeController,
+ nowFn,
+}) {
+ const evidence = new FailureEvidence({ nowFn });
+ const invocation = commandInvocation(command, shell, platform);
+ let child;
+ try {
+ child = spawnFn(invocation.executable, invocation.args, {
+ stdio: ['inherit', 'pipe', 'pipe'],
+ detached: platform !== 'win32',
+ windowsHide: platform === 'win32',
+ });
+ } catch (error) {
+ evidence.ingest(Buffer.from(`${error.message}\n`));
+ evidence.finish();
+ return new AttemptResult({
+ spawnError: error,
+ cleanup: { ok: true, reason: 'not-required' },
+ evidence,
+ });
+ }
+
+ child.stdout?.on('data', (chunk) => {
+ evidence.ingest(chunk);
+ process.stdout.write(chunk);
+ });
+ child.stderr?.on('data', (chunk) => {
+ evidence.ingest(chunk);
+ process.stderr.write(chunk);
+ });
+
+ let closeValue;
+ let resolveClose;
+ const closePromise = new Promise((resolve) => {
+ resolveClose = resolve;
+ });
+ let spawnError;
+ child.once('error', (error) => {
+ spawnError = error;
+ evidence.ingest(Buffer.from(`${error.message}\n`));
+ resolveClose({ kind: 'spawn-error', code: null, signal: null });
+ });
+ child.once('close', (code, signal) => {
+ closeValue = { kind: 'close', code, signal };
+ resolveClose(closeValue);
+ });
+
+ let timeoutId;
+ let deadlineId;
+ let cancelListener;
+ const timeoutPromise = new Promise((resolve) => {
+ timeoutId = setTimeout(() => resolve({ kind: 'attempt-timeout' }), attemptTimeoutMs);
+ });
+ const remainingToDeadline = Math.max(0, deadlineEpochMs - cleanupReserveMs - nowFn());
+ const deadlinePromise = new Promise((resolve) => {
+ deadlineId = setTimeout(() => resolve({ kind: 'deadline' }), remainingToDeadline);
+ });
+ const cancellationPromise = new Promise((resolve) => {
+ cancelListener = () =>
+ resolve({ kind: 'cancelled', signal: cancellationSignal.reason || 'SIGTERM' });
+ if (cancellationSignal.aborted) cancelListener();
+ else cancellationSignal.addEventListener('abort', cancelListener, { once: true });
+ });
+
+ const outcome = await Promise.race([
+ closePromise,
+ timeoutPromise,
+ deadlinePromise,
+ cancellationPromise,
+ ]);
+ clearTimeout(timeoutId);
+ clearTimeout(deadlineId);
+ cancellationSignal.removeEventListener('abort', cancelListener);
+
+ const waitForClose = (timeoutMs) => waitForPromise(closePromise, timeoutMs);
+ let cleanup = { ok: true, reason: 'clean' };
+ if (child.pid) {
+ cleanup = await treeController.cleanup(child.pid, {
+ graceMs: cleanupGraceMs,
+ cleanupReserveMs,
+ waitForClose,
});
- child.on('close', (code) => resolve({ code: code ?? 1, output: captured }));
+ } else if (outcome.kind !== 'close' && outcome.kind !== 'spawn-error') {
+ cleanup = { ok: false, reason: 'close-not-observed' };
+ }
+ if (!closeValue && outcome.kind !== 'spawn-error') {
+ await waitForClose(cleanupReserveMs);
+ }
+ evidence.finish();
+
+ return new AttemptResult({
+ code: closeValue?.code ?? outcome.code ?? null,
+ closeSignal: closeValue?.signal ?? null,
+ cancellationSignal: outcome.kind === 'cancelled' ? outcome.signal : null,
+ cancelled: outcome.kind === 'cancelled',
+ timedOut: outcome.kind === 'attempt-timeout',
+ deadlineExpired: outcome.kind === 'deadline',
+ spawnError,
+ cleanup,
+ evidence,
});
}
+function processStop(result, attempts) {
+ if (result.cancelled) {
+ return { ok: false, reason: 'signal', signal: result.cancellationSignal, attempts };
+ }
+ if (result.timedOut) return { ok: false, reason: 'attempt-timeout', attempts };
+ if (result.deadlineExpired) return { ok: false, reason: 'deadline', attempts };
+ if (result.spawnError) {
+ return { ok: false, reason: 'spawn-failure', attempts, error: result.spawnError.message };
+ }
+ if (!result.cleanup.ok) {
+ return {
+ ok: false,
+ reason: 'cleanup-failure',
+ cleanup: result.cleanup.reason,
+ attempts,
+ };
+ }
+ if (result.closeSignal) {
+ return { ok: false, reason: 'child-signal', signal: result.closeSignal, attempts };
+ }
+ return undefined;
+}
+
+function installSignalHandlers(signalSource, controller) {
+ const handlers = new Map();
+ for (const signal of ['SIGINT', 'SIGTERM']) {
+ const handler = () => {
+ if (!controller.signal.aborted) controller.abort(signal);
+ };
+ handlers.set(signal, handler);
+ signalSource.on(signal, handler);
+ }
+ return () => {
+ for (const [signal, handler] of handlers) signalSource.off(signal, handler);
+ };
+}
+
+async function waitForBackoff(ms, cancellationSignal, sleepFn) {
+ if (cancellationSignal.aborted) return false;
+ let abortListener;
+ const aborted = new Promise((resolve) => {
+ abortListener = () => resolve(false);
+ cancellationSignal.addEventListener('abort', abortListener, { once: true });
+ });
+ const slept = Promise.resolve(sleepFn(ms, cancellationSignal)).then(
+ () => !cancellationSignal.aborted,
+ );
+ const completed = await Promise.race([slept, aborted]);
+ cancellationSignal.removeEventListener('abort', abortListener);
+ return completed;
+}
+
export async function runWithRetry({
command,
label = 'command',
shell = false,
maxAttempts = DEFAULT_MAX_ATTEMPTS,
+ deadlineEpochMs = Date.now() + 60 * 60 * 1000,
+ attemptTimeoutMs = DEFAULT_ATTEMPT_TIMEOUT_MS,
+ cleanupReserveMs = DEFAULT_CLEANUP_RESERVE_MS,
+ cleanupGraceMs = DEFAULT_CLEANUP_GRACE_MS,
+ pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
spawnFn = spawn,
- sleepFn = (s) => new Promise((r) => setTimeout(r, s * 1000)),
+ sleepFn = (ms, signal) =>
+ new Promise((resolve) => {
+ const timer = setTimeout(resolve, ms);
+ signal?.addEventListener(
+ 'abort',
+ () => {
+ clearTimeout(timer);
+ resolve();
+ },
+ { once: true },
+ );
+ }),
+ randomFn = Math.random,
+ nowFn = Date.now,
log = console.log,
+ retryWarning = '',
+ signalSource = process,
+ platform = process.platform,
+ treeController,
+ attemptRunner = runAttempt,
} = {}) {
- for (let attempt = 1; ; attempt += 1) {
- log(`::group::${label} attempt ${attempt}/${maxAttempts}`);
- const { code, output } = await runOnce(command, { shell, spawnFn });
- log('::endgroup::');
-
- if (code === 0) {
- log(`${label} succeeded on attempt ${attempt}.`);
- return { ok: true, attempts: attempt };
- }
+ if (!Number.isInteger(maxAttempts) || maxAttempts < 1 || maxAttempts > DEFAULT_MAX_ATTEMPTS) {
+ throw new Error(`maxAttempts must be an integer from 1 to ${DEFAULT_MAX_ATTEMPTS}`);
+ }
+ const state = new RetryState();
+ const cancellation = new AbortController();
+ const removeSignalHandlers = installSignalHandlers(signalSource, cancellation);
+ const ownedTree =
+ treeController ??
+ createOwnedTreeController({
+ platform,
+ pollIntervalMs,
+ });
+ const recordDecision = (level, action, reason, result, facts = {}) =>
+ logDecision({
+ log,
+ level,
+ label,
+ action,
+ reason,
+ attempt: state.attemptsStarted,
+ maxAttempts,
+ result,
+ ...facts,
+ });
+ const finishFailure = (failure, result) => {
+ if (result) emitDiagnostic(log, label, result.evidence);
+ return failure;
+ };
- if (!isTransient(output)) {
- log(
- `::error::${label} failed on attempt ${attempt} with a NON-transient error (no known infra-flake signature matched). Not retrying β€” a real signing/notarize/cert/entitlements/build failure must fail loud, not be masked. See the log above.`,
- );
- return { ok: false, reason: 'non-transient', attempts: attempt, code };
- }
+ try {
+ while (state.attemptsStarted < maxAttempts) {
+ if (cancellation.signal.aborted) {
+ recordDecision('error', 'stop', 'control:signal', undefined, {
+ outcome: 'signal',
+ signal: cancellation.signal.reason,
+ });
+ return {
+ ok: false,
+ reason: 'signal',
+ signal: cancellation.signal.reason,
+ attempts: state.attemptsStarted,
+ };
+ }
+ const admissionMs = state.lastAttemptDurationMs
+ ? projectedAttemptMs(state.lastAttemptDurationMs, attemptTimeoutMs)
+ : MIN_PROJECTED_ATTEMPT_MS;
+ if (nowFn() + admissionMs + cleanupReserveMs > deadlineEpochMs) {
+ recordDecision('error', 'stop', 'control:deadline', undefined, {
+ outcome: 'deadline',
+ detail: 'phase=before-attempt',
+ });
+ return { ok: false, reason: 'deadline', attempts: state.attemptsStarted };
+ }
- if (attempt >= maxAttempts) {
- log(
- `::error::${label} failed after ${maxAttempts} attempts; the last failure matched a transient infra signature but retries are exhausted.`,
- );
- return { ok: false, reason: 'exhausted', attempts: attempt, code };
- }
+ state.attemptsStarted += 1;
+ log(`::group::${label} attempt ${state.attemptsStarted}/${maxAttempts}`);
+ const attemptStartedAt = nowFn();
+ const result = await attemptRunner({
+ command,
+ shell,
+ spawnFn,
+ platform,
+ cancellationSignal: cancellation.signal,
+ deadlineEpochMs,
+ attemptTimeoutMs,
+ cleanupReserveMs,
+ cleanupGraceMs,
+ treeController: ownedTree,
+ nowFn,
+ });
+ state.lastAttemptDurationMs = Math.max(0, nowFn() - attemptStartedAt);
+ log('::endgroup::');
+
+ const stopped = processStop(result, state.attemptsStarted);
+ if (stopped) {
+ recordDecision('error', 'stop', `control:${stopped.reason}`, result, {
+ outcome: stopped.reason,
+ signal: stopped.signal,
+ detail: stopped.reason === 'deadline' ? 'phase=mid-attempt' : undefined,
+ });
+ return finishFailure(stopped, result);
+ }
+ if (result.code === 0) {
+ log(`${label} succeeded on attempt ${state.attemptsStarted}.`);
+ return {
+ ok: true,
+ attempts: state.attemptsStarted,
+ recoveredFrom: state.attemptsStarted > 1 ? state.lastFailureClassification : undefined,
+ };
+ }
+
+ const classification = classifyEvidence(result.evidence);
+ const reasonId = matchedReason(result.evidence, classification);
+ state.lastFailureClassification = classification;
+ if (classification === 'terminal') {
+ recordDecision('error', 'stop', reasonId, result, { outcome: 'terminal' });
+ return finishFailure(
+ {
+ ok: false,
+ reason: 'terminal',
+ attempts: state.attemptsStarted,
+ code: result.code,
+ },
+ result,
+ );
+ }
+ if (classification === 'unknown') {
+ if (state.unknownRetryUsed) {
+ recordDecision('error', 'stop', reasonId, result, {
+ outcome: 'unknown-exhausted',
+ detail: 'allowance=already-used',
+ });
+ return finishFailure(
+ {
+ ok: false,
+ reason: 'unknown-exhausted',
+ attempts: state.attemptsStarted,
+ code: result.code,
+ },
+ result,
+ );
+ }
+ state.unknownRetryUsed = true;
+ }
+ if (state.attemptsStarted >= maxAttempts) {
+ const reason = classification === 'transient' ? 'transient-exhausted' : 'unknown-exhausted';
+ recordDecision('error', 'stop', reasonId, result, {
+ outcome: reason,
+ detail: `attempt-bound=${reason}`,
+ });
+ return finishFailure(
+ { ok: false, reason, attempts: state.attemptsStarted, code: result.code },
+ result,
+ );
+ }
- const backoff = backoffSeconds(attempt);
- log(
- `::warning::${label} attempt ${attempt} failed with a transient infra signature; retrying in ${backoff}s (bounded transient-only retry).`,
+ const decisionNow = nowFn();
+ const delayMs = computeRetryDelayMs(state.attemptsStarted, result.evidence, {
+ randomFn,
+ nowMs: decisionNow,
+ });
+ const nextAttemptMs = projectedAttemptMs(state.lastAttemptDurationMs, attemptTimeoutMs);
+ if (decisionNow + delayMs + nextAttemptMs + cleanupReserveMs > deadlineEpochMs) {
+ const retryAfterMs = retryAfterDelayMs(result.evidence, decisionNow);
+ recordDecision('error', 'stop', 'control:deadline', result, {
+ outcome: 'deadline',
+ matched: reasonId,
+ detail: [
+ 'phase=before-backoff',
+ retryAfterMs === undefined ? undefined : `retry-after-ms=${retryAfterMs}`,
+ `projected-attempt-ms=${nextAttemptMs}`,
+ ]
+ .filter(Boolean)
+ .join(' '),
+ });
+ return finishFailure(
+ { ok: false, reason: 'deadline', attempts: state.attemptsStarted },
+ result,
+ );
+ }
+ if (retryWarning) log(`::warning::${retryWarning}`);
+ recordDecision('warning', 'retry', reasonId, result, {
+ classification,
+ detail:
+ classification === 'unknown'
+ ? `UNKNOWN_CLASSIFICATION_RETRY allowance=invocation-wide-single-use delay-ms=${delayMs}`
+ : `delay-ms=${delayMs}`,
+ });
+ if (!(await waitForBackoff(delayMs, cancellation.signal, sleepFn))) {
+ recordDecision('error', 'stop', 'control:signal', result, {
+ outcome: 'signal',
+ signal: cancellation.signal.reason,
+ matched: reasonId,
+ detail: 'phase=backoff',
+ });
+ return finishFailure(
+ {
+ ok: false,
+ reason: 'signal',
+ signal: cancellation.signal.reason,
+ attempts: state.attemptsStarted,
+ },
+ result,
+ );
+ }
+ }
+ throw new Error(
+ `${label}: retry state exhausted unexpectedly after ${state.attemptsStarted}/${maxAttempts} attempts`,
);
- await sleepFn(backoff);
+ } finally {
+ removeSignalHandlers();
+ }
+}
+
+function parseDuration(value, flag) {
+ const match = /^(\d+)(ms|s|m)$/.exec(value ?? '');
+ if (!match) throw new Error(`${flag} must be a positive duration such as 30m`);
+ const multiplier = match[2] === 'm' ? 60_000 : match[2] === 's' ? 1_000 : 1;
+ const duration = Number(match[1]) * multiplier;
+ if (!Number.isSafeInteger(duration) || duration <= 0) {
+ throw new Error(`${flag} must be a positive duration`);
}
+ return duration;
+}
+
+function flagValue(flags, name, required = false) {
+ const index = flags.indexOf(name);
+ if (index === -1) {
+ if (required) throw new Error(`${name} is required`);
+ return undefined;
+ }
+ return flags[index + 1];
}
export function parseArgs(argv) {
const rest = argv.slice(2);
- const sep = rest.indexOf('--');
- if (sep === -1) throw new Error('a `--` separator followed by the command is required');
- const flags = rest.slice(0, sep);
- const command = rest.slice(sep + 1);
+ const separator = rest.indexOf('--');
+ if (separator === -1) throw new Error('a `--` separator followed by the command is required');
+ const flags = rest.slice(0, separator);
+ const command = rest.slice(separator + 1);
if (command.length === 0) throw new Error('no command given after `--`');
+ const allowedFlags = new Set([
+ '--label',
+ '--max-attempts',
+ '--deadline-epoch-ms',
+ '--attempt-timeout',
+ '--retry-warning',
+ '--shell',
+ ]);
+ for (let index = 0; index < flags.length; index += 1) {
+ const flag = flags[index];
+ if (!allowedFlags.has(flag)) throw new Error(`unknown flag: ${flag}`);
+ if (flag === '--shell') continue;
+ const value = flags[index + 1];
+ if (
+ value === undefined ||
+ value.startsWith('--') ||
+ (value === '' && flag !== '--retry-warning')
+ ) {
+ throw new Error(`${flag} requires a value`);
+ }
+ index += 1;
+ }
const shell = flags.includes('--shell');
- if (shell && command.length !== 1) {
- throw new Error('--shell takes exactly one command string');
+ if (shell && command.length !== 1) throw new Error('--shell takes exactly one command string');
+ const maxAttempts = Number(flagValue(flags, '--max-attempts') ?? DEFAULT_MAX_ATTEMPTS);
+ if (!Number.isInteger(maxAttempts) || maxAttempts < 1 || maxAttempts > DEFAULT_MAX_ATTEMPTS) {
+ throw new Error(`--max-attempts must be between 1 and ${DEFAULT_MAX_ATTEMPTS}`);
}
-
- const labelIdx = flags.indexOf('--label');
- const attemptsIdx = flags.indexOf('--max-attempts');
+ const deadlineEpochMs = Number(flagValue(flags, '--deadline-epoch-ms', true));
+ if (!Number.isSafeInteger(deadlineEpochMs) || deadlineEpochMs <= 0) {
+ throw new Error('--deadline-epoch-ms must be a positive integer');
+ }
+ const attemptTimeoutMs = parseDuration(
+ flagValue(flags, '--attempt-timeout', true),
+ '--attempt-timeout',
+ );
return {
- label: labelIdx === -1 ? 'command' : (flags[labelIdx + 1] ?? 'command'),
- maxAttempts:
- attemptsIdx === -1 ? DEFAULT_MAX_ATTEMPTS : Number(flags[attemptsIdx + 1]) || DEFAULT_MAX_ATTEMPTS,
+ label: flagValue(flags, '--label') ?? 'command',
+ maxAttempts,
+ deadlineEpochMs,
+ attemptTimeoutMs,
+ retryWarning: flagValue(flags, '--retry-warning') ?? '',
shell,
command,
};
@@ -141,9 +950,13 @@ export function parseArgs(argv) {
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
try {
const result = await runWithRetry(parseArgs(process.argv));
- process.exitCode = result.ok ? 0 : 1;
- } catch (err) {
- process.stderr.write(`${err.message}\n`);
+ if (result.reason === 'signal' && result.signal) {
+ process.kill(process.pid, result.signal);
+ } else {
+ process.exitCode = result.ok ? 0 : 1;
+ }
+ } catch (error) {
+ process.stderr.write(`${error.message}\n`);
process.exitCode = 1;
}
}
diff --git a/.github/scripts/retry-transient.test.mjs b/.github/scripts/retry-transient.test.mjs
index 19adc148d..3a3ee038e 100644
--- a/.github/scripts/retry-transient.test.mjs
+++ b/.github/scripts/retry-transient.test.mjs
@@ -1,271 +1,1454 @@
-import { spawnSync } from 'node:child_process';
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { spawn, spawnSync } from 'node:child_process';
+import { EventEmitter } from 'node:events';
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
+import { PassThrough } from 'node:stream';
import { fileURLToPath } from 'node:url';
-import { afterAll, beforeEach, describe, expect, test, vi } from 'vitest';
+import { afterAll, afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
import {
- backoffSeconds,
+ createFuseFailure,
+ FUSE_FAILURE_MARKER,
+} from '../../packages/desktop/scripts/packaging-diagnostics.mjs';
+import {
+ classifyEvidence,
+ computeRetryDelayMs,
+ createOwnedTreeController,
DEFAULT_MAX_ATTEMPTS,
- isTransient,
+ FailureEvidence,
parseArgs,
runWithRetry,
- TRANSIENT_SIGNATURES,
} from './retry-transient.mjs';
const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const SCRIPT = join(REPO_ROOT, '.github', 'scripts', 'retry-transient.mjs');
+const afterPackSource = readFileSync(
+ join(REPO_ROOT, 'packages', 'desktop', 'scripts', 'afterPack.mjs'),
+ 'utf8',
+);
+const afterSignSource = readFileSync(
+ join(REPO_ROOT, 'packages', 'desktop', 'scripts', 'afterSign.mjs'),
+ 'utf8',
+);
const desktopRelease = readFileSync(
join(REPO_ROOT, '.github', 'workflows', 'desktop-release.yml'),
'utf8',
);
-
+const WORKFLOW_JOBS = [
+ ['build-macos', '65', '30m', 'Build + sign + notarize DMG/ZIP'],
+ ['build-windows', '35', '15m', 'Package NSIS installers (x64 + arm64, signed)'],
+ ['build-linux', '35', '15m', 'Package $' + '{{ matrix.targets }}'],
+];
+const workflowJob = (name) => {
+ const start = desktopRelease.indexOf(`\n ${name}:`);
+ if (start === -1) throw new Error(`missing job ${name}`);
+ const rest = desktopRelease.slice(start + 1);
+ const next = rest.slice(1).search(/\n {2}[a-z][a-z0-9-]*:/);
+ return next === -1 ? rest : rest.slice(0, next + 1);
+};
+const workflowStep = (body, name) => {
+ const start = body.indexOf(`- name: ${name}`);
+ if (start === -1) throw new Error(`missing step ${name}`);
+ const rest = body.slice(start);
+ const ends = [rest.indexOf('\n - name: ', 1), rest.indexOf('\n - uses: ', 1)].filter(
+ (index) => index !== -1,
+ );
+ return ends.length === 0 ? rest : rest.slice(0, Math.min(...ends));
+};
+const workflowTiming = ([jobName, , , packageName]) => {
+ const body = workflowJob(jobName);
+ const budget = /PACKAGING_BUDGET_MINUTES: "(\d+)"/.exec(
+ workflowStep(body, 'Start packaging deadline'),
+ )?.[1];
+ const timeout = /--attempt-timeout "(\d+)m"/.exec(workflowStep(body, packageName))?.[1];
+ if (!budget || !timeout) throw new Error(`missing workflow timing for ${jobName}`);
+ return { budgetMs: Number(budget) * 60_000, attemptTimeoutMs: Number(timeout) * 60_000 };
+};
const scratch = mkdtempSync(join(tmpdir(), 'retry-transient-'));
-afterAll(() => rmSync(scratch, { recursive: true, force: true }));
+afterAll(() => rmSync(scratch, { recursive: true, force: true }));
+afterEach(() => vi.restoreAllMocks());
beforeEach(() => {
vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
- return () => vi.restoreAllMocks();
});
-const nodeCmd = (src) => ['node', '-e', src];
-
-const run = (overrides) => {
+const nodeCmd = (src) => [process.execPath, '-e', src];
+const sourceSection = (source, start, end) => {
+ const startIndex = source.indexOf(start);
+ const endIndex = source.indexOf(end, startIndex);
+ if (startIndex === -1 || endIndex === -1) throw new Error(`missing source section: ${start}`);
+ return source.slice(startIndex, endIndex);
+};
+const throwTargets = (source) =>
+ new Set(
+ [...source.matchAll(/\bthrow\s+((?:new\s+)?[A-Za-z_$][\w$]*)/g)].map((match) => match[1]),
+ );
+const run = (overrides = {}) => {
const lines = [];
+ const now = Date.now();
return runWithRetry({
+ command: nodeCmd('process.exit(0)'),
+ deadlineEpochMs: now + 120_000,
+ attemptTimeoutMs: 10_000,
+ cleanupReserveMs: 100,
+ cleanupGraceMs: 20,
+ pollIntervalMs: 2,
sleepFn: () => Promise.resolve(),
- log: (l) => lines.push(l),
+ randomFn: () => 0,
+ log: (line) => lines.push(line),
...overrides,
}).then((result) => ({ ...result, lines, log: lines.join('\n') }));
};
-describe('the retry decision', () => {
- test('a command that succeeds runs exactly once', async () => {
- const r = await run({ command: nodeCmd('console.log("built")') });
- expect(r.ok).toBe(true);
- expect(r.attempts).toBe(1);
+const inspect = (text, options) => {
+ const evidence = new FailureEvidence(options);
+ evidence.ingest(Buffer.from(text));
+ evidence.finish();
+ return { evidence, classification: classifyEvidence(evidence) };
+};
+
+describe('failure evidence classification', () => {
+ test.each([408, 429, 500, 502, 503, 504, 521, 522, 524])(
+ 'classifies structured HTTP %s as transient',
+ (status) => {
+ expect(inspect(`HTTPError: Response code ${status} (service response)`).classification).toBe(
+ 'transient',
+ );
+ },
+ );
+
+ test.each(['502 Bad Gateway', '503 Service Unavailable', '504 Gateway Time-out'])(
+ 'keeps structured status-line coverage: %s',
+ (text) => {
+ expect(inspect(text).classification).toBe('transient');
+ },
+ );
+
+ test.each([
+ 'getaddrinfo EAI_AGAIN github.com',
+ 'getaddrinfo ENOTFOUND github.com',
+ 'connect ECONNREFUSED 127.0.0.1:443',
+ 'connect ENETUNREACH 10.0.0.1:443',
+ 'connect EHOSTUNREACH 10.0.0.1:443',
+ 'read ECONNABORTED',
+ 'read ECONNRESET',
+ 'connect ETIMEDOUT',
+ 'write EPIPE',
+ 'cause: UND_ERR_CONNECT_TIMEOUT',
+ 'cause: UND_ERR_HEADERS_TIMEOUT',
+ 'cause: UND_ERR_BODY_TIMEOUT',
+ 'cause: UND_ERR_SOCKET',
+ 'curl: (28) Operation timed out',
+ 'curl: (56) Recv failure: Connection reset by peer',
+ ])('classifies scoped network evidence as transient: %s', (text) => {
+ expect(inspect(text).classification).toBe('transient');
+ });
+
+ test.each([-1001, -1003, -1004, -1005, -1006, -1008, -1009, -1011])(
+ 'classifies NSURLErrorDomain Code=%s as transient',
+ (code) => {
+ expect(inspect(`Error Domain=NSURLErrorDomain Code=${code}`).classification).toBe(
+ 'transient',
+ );
+ },
+ );
+
+ test.each([-1201, -1202, -1203, -1204, -1205, -1206])(
+ 'classifies bare NSURLErrorDomain TLS trust Code=%s as terminal',
+ (code) => {
+ expect(inspect(`Error Domain=NSURLErrorDomain Code=${code}`).classification).toBe('terminal');
+ },
+ );
+
+ test('leaves generic NSURLErrorDomain secure-connection failure unknown', () => {
+ expect(inspect('Error Domain=NSURLErrorDomain Code=-1200').classification).toBe('unknown');
+ });
+
+ test.each([
+ 'A timestamp was expected but was not found',
+ 'The timestamp service is not available',
+ 'HTTPError(statusCode: nil)',
+ 'The request timed out',
+ 'The network connection was lost',
+ 'Could not connect to the server',
+ 'socket hang up',
+ 'Client network socket disconnected before secure TLS connection was established',
+ 'unexpected EOF',
+ 'You have exceeded a secondary rate limit',
+ 'abuse detection mechanism',
+ 'was submitted too quickly',
+ ])('keeps narrow transient phrase coverage: %s', (text) => {
+ expect(inspect(text).classification).toBe('transient');
+ });
+
+ test.each([
+ 'HTTP 401 Unauthorized',
+ 'HTTP 400 Bad Request',
+ 'HTTPError: Response code 422 (Unprocessable Entity)',
+ 'response status: 404',
+ 'statusCode=413',
+ 'error TS2345: Argument of type string is not assignable',
+ 'electron-vite build failed',
+ 'Invalid configuration object',
+ 'configuration is invalid',
+ 'β¨― Invalid configuration object. electron-builder 26.0.1 has been initialized using a configuration object that does not match the API schema.',
+ 'Error: unknown option "--foo"',
+ ' ERR_PNPM_OUTDATED_LOCKFILE Cannot install with "frozen-lockfile" because pnpm-lock.yaml is not up to date',
+ 'authentication failed for signing service',
+ 'invalid credentials supplied to Azure Trusted Signing',
+ 'The specified item could not be found in the keychain',
+ 'certificate has expired',
+ 'certificate not trusted',
+ 'notarization failed with status: Invalid',
+ 'notarytool submission completed\nstatus: Invalid',
+ 'notarytool submission completed\n{"status":"Invalid"}',
+ 'entitlement com.apple.security.foo is not permitted',
+ 'Electron fuse verification failed',
+ '[afterSign] fuse verification failed (D17 paranoid check):\n RunAsNode: expected ENABLE (target=true), got DISABLE [OK_PACKAGING_FUSE_FAILURE]',
+ '[afterSign] fuse verification read failed on /tmp/OpenKnowledge: EACCES [OK_PACKAGING_FUSE_FAILURE]',
+ '[afterPack] fuse flip failed on /tmp/OpenKnowledge: Could not find sentinel [OK_PACKAGING_FUSE_FAILURE]',
+ 'Electron fuse mismatch',
+ 'Electron fuse mismatches target',
+ 'Electron fuse mismatched target',
+ 'integrity check failed: checksum mismatch',
+ 'sha512 checksum mismatch, expected AAA, got BBB',
+ 'sha512 hash mismatch, expected AAA, got BBB',
+ 'Error: Cannot find module ./missing',
+ 'bash: pnpm: command not found',
+ 'ERR_PNPM_RECURSIVE_EXEC_FIRST_FAIL Command "electron-builder" not found',
+ 'spawn electron-builder ENOENT',
+ 'ENOSPC: no space left on device',
+ ])('classifies explicit terminal evidence as terminal: %s', (text) => {
+ expect(inspect(text).classification).toBe('terminal');
+ });
+
+ test('classifies errors from the owned fuse-failure factory', () => {
+ const error = createFuseFailure('fuse detail changed');
+ expect(error.message).toBe(`fuse detail changed [${FUSE_FAILURE_MARKER}]`);
+ expect(inspect(error.message).classification).toBe('terminal');
+ });
+
+ test('routes every throw in the owned fuse functions through the shared factory', () => {
+ const afterPackFuseFunction = sourceSection(
+ afterPackSource,
+ 'async function flipElectronFuses',
+ '\n\nexport default async function afterPack',
+ );
+ const afterSignFuseFunction = sourceSection(
+ afterSignSource,
+ 'async function verifyFuses',
+ '\n\nexport default async function afterSign',
+ );
+ expect(throwTargets(afterPackFuseFunction)).toEqual(new Set(['createFuseFailure']));
+ expect(throwTargets(afterSignFuseFunction)).toEqual(new Set(['createFuseFailure']));
+ });
+
+ test.each([
+ '[afterSign] getCurrentFuseWire failed on /tmp/OpenKnowledge: EACCES',
+ '[afterSign] Fuse verification failed (D17 paranoid check):\n RunAsNode: expected ENABLE (target=true), got DISABLE',
+ '[afterPack] fuse flip failed on /tmp/OpenKnowledge: Could not find sentinel',
+ ])('classifies old-tag fuse failures without the owned marker: %s', (text) => {
+ expect(inspect(text).classification).toBe('terminal');
+ });
+
+ test.each([
+ 'warning: optional tool: command not found\nsocket hang up',
+ 'download retry noted checksum mismatch\nECONNRESET',
+ 'warning: unknown option "--foo"\nsocket hang up',
+ ])('does not let incidental terminal-like prose outrank network evidence: %s', (text) => {
+ expect(inspect(text).classification).toBe('transient');
+ });
+
+ test.each([
+ '[afterSign] fuse verification passed β€” all 6 fuses match targetFuses',
+ '[afterSign] fuse verification done; no notarize step on platform "linux"',
+ '[afterSign] signed + notarized + stapled + fuse-verified successfully',
+ '[afterPack] skipping per-arch temp "/tmp/app-temp" β€” fuses flip on the merged universal app',
+ '[afterPack] flipping fuses on /tmp/OpenKnowledge',
+ '[afterPack] fuses flipped successfully; electron-builder will re-sign next',
+ '[afterPack] fuses done; skipping darwin-only helper-bundle + node-pty steps on "linux"',
+ 'fuse: failed to exec fusermount: No such file or directory',
+ ])('does not classify excluded fuse output as terminal: %s', (text) => {
+ expect(inspect(`${text}\nHTTP 500`).classification).toBe('transient');
+ });
+
+ test.each(['unknown option "--foo"', 'bash: pnpm: command not found'])(
+ 'does not treat a chunk-carry boundary as a line boundary: %s',
+ (phrase) => {
+ const evidence = new FailureEvidence();
+ evidence.ingest(
+ Buffer.from(`${'x'.repeat(2048)}${phrase}${'y'.repeat(2048 - phrase.length)}`),
+ );
+ evidence.ingest(Buffer.from('\nsocket hang up'));
+ evidence.finish();
+ expect(classifyEvidence(evidence)).toBe('transient');
+ },
+ );
+
+ test('does not treat a chunk-carry boundary as a module-error line start', () => {
+ const phrase = 'Error: Cannot find module ./missing';
+ const evidence = new FailureEvidence();
+ evidence.ingest(Buffer.from(`${'x'.repeat(2048)}${phrase}${'y'.repeat(2048 - phrase.length)}`));
+ evidence.ingest(Buffer.from('\nsocket hang up'));
+ evidence.finish();
+ expect(classifyEvidence(evidence)).toBe('transient');
+ });
+
+ test('does not treat a chunk-carry boundary as a decorated-status line start', () => {
+ const phrase = 'β¨― 502 Bad Gateway';
+ const evidence = new FailureEvidence();
+ evidence.ingest(Buffer.from(`${'x'.repeat(2048)}${phrase}${'y'.repeat(2048 - phrase.length)}`));
+ evidence.finish();
+ expect(classifyEvidence(evidence)).toBe('unknown');
+ });
+
+ test.each([
+ ['Error: unknown option "--foo"', 'terminal'],
+ ['Error: Cannot find module ./missing', 'terminal'],
+ ['bash: pnpm: command not found', 'terminal'],
+ ['β¨― 502 Bad Gateway', 'transient'],
+ ])('retains a real line boundary across chunks: %s', (line, classification) => {
+ const evidence = new FailureEvidence();
+ evidence.ingest(Buffer.from('progress without a newline'));
+ evidence.ingest(Buffer.from(`\n${line}`));
+ evidence.finish();
+ expect(classifyEvidence(evidence)).toBe(classification);
+ });
+
+ test('distinguishes permission-denied and rate-limited HTTP 403', () => {
+ expect(inspect('HTTP 403 Forbidden').classification).toBe('terminal');
+ expect(inspect('HTTP 403 Forbidden\nsecondary rate limit exceeded').classification).toBe(
+ 'transient',
+ );
+ expect(inspect('HTTP 403 Forbidden\nRetry-After: 45').classification).toBe('terminal');
+ expect(inspect('HTTP 403 Forbidden\nX-RateLimit-Remaining: 0').classification).toBe(
+ 'transient',
+ );
+ expect(inspect('HTTP 403 Forbidden\nRetry-After: tomorrow').classification).toBe('terminal');
+ });
+
+ test.each(['β¨― 502 Bad Gateway', 'Error: 503 Service Unavailable'])(
+ 'accepts explicit decorated status lines: %s',
+ (text) => expect(inspect(text).classification).toBe('transient'),
+ );
+
+ test('rejects arbitrary decorated status lines', () => {
+ expect(inspect('download failed: 502 Bad Gateway').classification).toBe('unknown');
+ });
+
+ test('matched reasons follow classifier precedence for mixed evidence', async () => {
+ const terminal = await run({
+ command: nodeCmd('console.error("certificate has expired\\nHTTP 404");process.exit(1)'),
+ });
+ expect(terminal.log).toContain('reason=rule:certificate outcome=terminal');
+ expect(terminal.log).not.toContain('reason=http:404');
+ const downloadIntegrity = await run({
+ command: nodeCmd(
+ 'console.error("sha512 checksum mismatch, expected AAA, got BBB");process.exit(1)',
+ ),
+ });
+ expect(downloadIntegrity.log).toContain('reason=rule:download-integrity outcome=terminal');
+ const maskedIntegrity = await run({
+ command: nodeCmd(
+ 'console.error("sha512 checksum mismatch, expected AAA, got BBB\\nrejected entitlements for app");process.exit(1)',
+ ),
+ });
+ expect(maskedIntegrity.log).toContain('reason=rule:entitlement outcome=terminal');
+ const maskedTlsTrust = await run({
+ command: nodeCmd(
+ 'console.error("sha512 checksum mismatch, expected AAA, got BBB\\nError Domain=NSURLErrorDomain Code=-1202");process.exit(1)',
+ ),
+ });
+ expect(maskedTlsTrust.log).toContain('reason=rule:tls-trust outcome=terminal');
+ expect(maskedTlsTrust.log).not.toContain('rule:download-integrity');
+ const tlsTrustWithSymptom = await run({
+ command: nodeCmd(
+ 'console.error("Error Domain=NSURLErrorDomain Code=-1202\\nError: Exit code: ENOENT. spawn /Users/runner/Library/Caches/electron-builder/app-builder/app-builder ENOENT");process.exit(1)',
+ ),
+ });
+ expect(tlsTrustWithSymptom.log).toContain('reason=rule:tls-trust outcome=terminal');
+ const tlsTrustWithIntegrity = await run({
+ command: nodeCmd(
+ 'console.error("Error Domain=NSURLErrorDomain Code=-1202\\nintegrity check failed");process.exit(1)',
+ ),
+ });
+ expect(tlsTrustWithIntegrity.log).toContain('reason=rule:tls-trust outcome=terminal');
+ const notarizationInvalidWithTlsNoise = await run({
+ command: nodeCmd(
+ 'console.error("Error Domain=NSURLErrorDomain Code=-1202");console.error(`Failed to notarize via notarytool\\n{"id":"abc","status":"Invalid","message":"Processing complete"}`);process.exit(1)',
+ ),
+ });
+ expect(notarizationInvalidWithTlsNoise.log).toContain(
+ 'reason=rule:notarization-invalid outcome=terminal',
+ );
+ const fuseFailure = await run({
+ command: nodeCmd(
+ 'console.error("[afterPack] fuse flip failed on /tmp/OpenKnowledge: Could not find sentinel [OK_PACKAGING_FUSE_FAILURE]");process.exit(1)',
+ ),
+ });
+ expect(fuseFailure.log).toContain('reason=rule:fuse outcome=terminal');
+ const d17FuseFailure = await run({
+ command: nodeCmd(
+ 'console.error("[afterSign] fuse verification failed (D17 paranoid check):\\n RunAsNode: expected ENABLE (target=true), got DISABLE [OK_PACKAGING_FUSE_FAILURE]");process.exit(1)',
+ ),
+ });
+ expect(d17FuseFailure.log).toContain('reason=rule:fuse outcome=terminal');
+ const diskFull = await run({
+ command: nodeCmd(
+ 'console.error("ENOSPC: no space left on device\\nintegrity check failed");process.exit(1)',
+ ),
+ });
+ expect(diskFull.log).toContain('reason=rule:disk-full outcome=terminal');
+ const diskFullWithCertificate = await run({
+ command: nodeCmd(
+ 'console.error("certificate has expired\\nENOSPC: no space left on device");process.exit(1)',
+ ),
+ });
+ expect(diskFullWithCertificate.log).toContain('reason=rule:disk-full outcome=terminal');
+ const signedIntegrity = await run({
+ command: nodeCmd(
+ 'console.error("integrity check failed: code signature invalid");process.exit(1)',
+ ),
+ });
+ expect(signedIntegrity.log).toContain('reason=rule:integrity outcome=terminal');
+ expect(signedIntegrity.log).not.toContain('download-integrity');
+ const transient = await run({
+ command: nodeCmd('console.error("ECONNRESET\\nHTTP 503");process.exit(1)'),
+ maxAttempts: 1,
+ });
+ expect(transient.log).toContain('reason=code:ECONNRESET');
+ expect(transient.log).not.toContain('reason=http:503');
+ });
+
+ test.each([
+ 'earlier warning: ECONNRESET\nnotarization failed with status: Invalid',
+ 'certificate has expired\nlater warning: socket hang up',
+ ])('terminal evidence dominates transient evidence in either order', (text) => {
+ expect(inspect(text).classification).toBe('terminal');
+ });
+
+ test('deletes broad CFNetwork matching and retains byte-capped early evidence', () => {
+ expect(
+ inspect('Error Domain=kCFErrorDomainCFNetwork Code=-1202 certificate not trusted')
+ .classification,
+ ).toBe('terminal');
+ expect(inspect('Error Domain=NSURLErrorDomain Code=-1202').classification).toBe('terminal');
+ const evidence = new FailureEvidence({ maxTailBytes: 19 });
+ evidence.ingest(Buffer.from('certificate has expired\n'));
+ evidence.ingest(Buffer.from('πŸ™‚'.repeat(100)));
+ evidence.finish();
+ expect(classifyEvidence(evidence)).toBe('terminal');
+ expect(Buffer.byteLength(evidence.tail, 'utf8')).toBeLessThanOrEqual(19);
+ expect(evidence.tail).not.toContain('certificate');
+ });
+
+ test('does not consume the first-line anchor on an empty chunk', () => {
+ const evidence = new FailureEvidence();
+ evidence.ingest(Buffer.alloc(0));
+ evidence.ingest(Buffer.from('Error: unknown option "--publish"'));
+ evidence.finish();
+ expect(classifyEvidence(evidence)).toBe('terminal');
+ });
+
+ test('leaves unrelated failures unknown', () => {
+ expect(inspect('packager exited without a diagnostic').classification).toBe('unknown');
});
+});
- test('a NON-transient failure fails loud on the first attempt', async () => {
- const r = await run({
- command: nodeCmd('console.error("notarization status: Invalid");process.exit(1)'),
+describe('retry state machine', () => {
+ test('success runs once and terminal evidence never retries', async () => {
+ expect(await run({ command: nodeCmd('console.log("built")') })).toMatchObject({
+ ok: true,
+ attempts: 1,
});
- expect(r.ok).toBe(false);
- expect(r.reason).toBe('non-transient');
- expect(r.attempts).toBe(1);
- expect(r.log).toContain('NON-transient');
+ const terminal = await run({
+ command: nodeCmd(
+ 'console.error("socket hang up");console.error("notarization failed with status: Invalid");process.exit(1)',
+ ),
+ });
+ expect(terminal).toMatchObject({ ok: false, reason: 'terminal', attempts: 1 });
+ expect(terminal.log).toContain(
+ 'decision=stop reason=rule:notarization-invalid outcome=terminal attempt=1/3 code=1 signal=none',
+ );
});
- test('a transient failure is retried and can succeed', async () => {
- const counter = join(scratch, 'flaky-count');
- writeFileSync(counter, '0');
- const r = await run({
+ test('shell mode executes the workflow command with Bash', async () => {
+ expect(await run({ command: ['true && printf composed'], shell: true })).toMatchObject({
+ ok: true,
+ attempts: 1,
+ });
+ });
+
+ test('transient failures retry within three total attempts and can recover', async () => {
+ const count = join(scratch, 'transient-recovery-count');
+ writeFileSync(count, '0');
+ const result = await run({
command: nodeCmd(
- `const fs=require('fs');const p=${JSON.stringify(counter)};` +
- `const n=Number(fs.readFileSync(p,'utf8'))+1;fs.writeFileSync(p,String(n));` +
- `if(n<2){console.error('β¨― socket hang up failedTask=build');process.exit(1)}` +
- `console.log('packaged')`,
+ `const fs=require('fs');const p=${JSON.stringify(count)};const n=+fs.readFileSync(p,'utf8')+1;fs.writeFileSync(p,String(n));if(n<3){console.error('HTTPError: Response code 500 (Internal Server Error)');process.exit(1)}`,
),
});
- expect(r.ok).toBe(true);
- expect(r.attempts).toBe(2);
- expect(readFileSync(counter, 'utf8')).toBe('2');
+ expect(result).toMatchObject({ ok: true, attempts: 3 });
+ expect(readFileSync(count, 'utf8')).toBe('3');
+ expect(result.log).toContain(
+ 'decision=retry reason=http:500 classification=transient attempt=1/3 code=1 signal=none',
+ );
});
- test('a permanently transient failure stops at the attempt bound', async () => {
- const r = await run({
+ test('persistent transient failure stops at the total-attempt bound', async () => {
+ const result = await run({
command: nodeCmd('console.error("socket hang up");process.exit(1)'),
- maxAttempts: 3,
});
- expect(r.ok).toBe(false);
- expect(r.reason).toBe('exhausted');
- expect(r.attempts).toBe(3);
- expect(r.log).toContain('retries are exhausted');
+ expect(result).toMatchObject({
+ ok: false,
+ reason: 'transient-exhausted',
+ attempts: DEFAULT_MAX_ATTEMPTS,
+ });
+ expect(result.log).toContain('outcome=transient-exhausted');
+ });
+
+ test('one unknown can recover but a second unknown stops', async () => {
+ const count = join(scratch, 'unknown-recovery-count');
+ writeFileSync(count, '0');
+ const recovered = await run({
+ command: nodeCmd(
+ `const fs=require('fs');const p=${JSON.stringify(count)};const n=+fs.readFileSync(p,'utf8')+1;fs.writeFileSync(p,String(n));if(n===1){console.error('unrecognized packager failure');process.exit(1)}`,
+ ),
+ });
+ expect(recovered).toMatchObject({ ok: true, attempts: 2, recoveredFrom: 'unknown' });
+ expect(recovered.log).toContain(
+ 'decision=retry reason=diagnostic:unknown classification=unknown attempt=1/3 code=1 signal=none',
+ );
+ expect(recovered.log).toContain(
+ 'UNKNOWN_CLASSIFICATION_RETRY allowance=invocation-wide-single-use',
+ );
+ const exhausted = await run({
+ command: nodeCmd('console.error("unrecognized packager failure");process.exit(1)'),
+ });
+ expect(exhausted).toMatchObject({ ok: false, reason: 'unknown-exhausted', attempts: 2 });
+ expect(exhausted.log).toContain('outcome=unknown-exhausted');
+ const bounded = await run({
+ command: nodeCmd('console.error("unrecognized packager failure");process.exit(1)'),
+ maxAttempts: 1,
+ });
+ expect(bounded).toMatchObject({ ok: false, reason: 'unknown-exhausted', attempts: 1 });
+ expect(bounded.log).toContain('outcome=unknown-exhausted');
});
- test('a command that cannot be spawned at all is not retried forever', async () => {
- const r = await run({ command: ['definitely-not-a-real-binary-xyz'], maxAttempts: 2 });
- expect(r.ok).toBe(false);
- expect(r.reason).toBe('non-transient');
+ test('the unknown allowance is shared across a mixed sequence', async () => {
+ const count = join(scratch, 'mixed-count');
+ writeFileSync(count, '0');
+ const result = await run({
+ command: nodeCmd(
+ `const fs=require('fs');const p=${JSON.stringify(count)};const n=+fs.readFileSync(p,'utf8')+1;fs.writeFileSync(p,String(n));console.error(n===2?'socket hang up':'unrecognized packager failure');process.exit(1)`,
+ ),
+ });
+ expect(result).toMatchObject({ ok: false, reason: 'unknown-exhausted', attempts: 3 });
+ expect(result.log).toContain('outcome=unknown-exhausted');
});
- test('--shell mode runs a compound command', async () => {
- const r = await run({ command: ['true && echo composed'], shell: true });
- expect(r.ok).toBe(true);
+ test('spawn failure and child signals are process-control stops', async () => {
+ const spawnFailed = await run({ command: ['definitely-not-a-real-binary-xyz'] });
+ expect(spawnFailed).toMatchObject({
+ ok: false,
+ reason: 'spawn-failure',
+ attempts: 1,
+ });
+ expect(spawnFailed.log).toContain('outcome=spawn-failure');
+ expect(spawnFailed.log).not.toContain('cleanup=');
+ const signalled = await run({ command: nodeCmd('process.kill(process.pid, "SIGTERM")') });
+ expect(signalled).toMatchObject({
+ ok: false,
+ reason: 'child-signal',
+ attempts: 1,
+ signal: 'SIGTERM',
+ });
+ expect(signalled.log).toContain(
+ 'decision=stop reason=control:child-signal outcome=child-signal attempt=1/3 code=none signal=SIGTERM',
+ );
});
-});
-describe('the transient allowlist', () => {
- test('matches the signature that blocked the 0.52.2 and 0.52.3 cuts', () => {
- expect(isTransient('β¨― socket hang up failedTask=build stackTrace=RequestError: socket hang up')).toBe(
- true,
+ test('the re-emitted bounded tail is redacted and workflow-command safe', async () => {
+ const secret = 'super-secret-output-value';
+ const sensitive = [
+ `CSC_KEY_PASSWORD=${secret}`,
+ `Authorization: Bearer ${secret}`,
+ `"api_key": "${secret}"`,
+ '-----BEGIN PRIVATE KEY-----',
+ 'private-body',
+ '-----END PRIVATE KEY-----',
+ '::error::injected',
+ '##[error]legacy-injected',
+ ].join('\n');
+ const result = await run({
+ command: nodeCmd(`console.error(${JSON.stringify(sensitive)});process.exit(1)`),
+ maxAttempts: 1,
+ });
+ expect(result.log).toContain(
+ 'reason=diagnostic:unknown outcome=unknown-exhausted attempt=1/1 code=1 signal=none',
);
+ expect(result.log).toContain('::group::command bounded failure diagnostic');
+ expect(result.log).toContain('| CSC_KEY_PASSWORD=[REDACTED]');
+ expect(result.log).toContain('| Authorization: [REDACTED]');
+ expect(result.log).toContain('| "api_key": [REDACTED]');
+ expect(result.log).toContain('| [REDACTED PEM]');
+ expect(result.log).toContain('| ::error::injected');
+ expect(result.log).not.toContain('\n::error::injected');
+ expect(result.log).toContain('| # #[error]legacy-injected');
+ expect(result.log).not.toContain('##[');
+ expect(result.log).not.toContain(secret);
+ expect(result.log).not.toContain('private-body');
+ expect(result).not.toHaveProperty('diagnosticTail');
});
- test('matches case-insensitively and mid-line', () => {
- expect(isTransient('...Socket Hang Up...')).toBe(true);
- expect(isTransient('fetch failed: ECONNRESET')).toBe(true);
+ test('the live child stream remains verbatim before the bounded duplicate', async () => {
+ const marker = 'PUBLIC_TRANSCRIPT_MARKER';
+ const result = await run({
+ command: nodeCmd(`console.error(${JSON.stringify(marker)});process.exit(1)`),
+ maxAttempts: 1,
+ });
+ const liveStderr = process.stderr.write.mock.calls.map(([chunk]) => String(chunk)).join('');
+ expect(liveStderr).toContain(marker);
+ expect(result.log).toContain(`| ${marker}`);
});
- test('does NOT match real build, cert, or notarization failures', () => {
- for (const real of [
- 'error TS2345: Argument of type string is not assignable',
- 'The specified item could not be found in the keychain',
- 'certificate has expired',
- 'notarization failed with status: Invalid',
- 'Command failed with exit code 1: tsc --noEmit',
- 'Error: Cannot find module ./missing',
- 'ENOSPC: no space left on device',
- ]) {
- expect(isTransient(real), `${real} must not be treated as transient`).toBe(false);
- }
+ test('production diagnostic output remains byte bounded', async () => {
+ const result = await run({
+ command: nodeCmd(
+ `console.error(${JSON.stringify(`rolled-off-marker\n${'x'.repeat(40_000)}\nfinal-marker`)});process.exit(1)`,
+ ),
+ maxAttempts: 1,
+ });
+ expect(result.log).not.toContain('rolled-off-marker');
+ expect(result.log).toContain('final-marker');
+ expect(Buffer.byteLength(result.log, 'utf8')).toBeLessThan(18_000);
});
- test('handles empty and nullish output without throwing', () => {
- expect(isTransient('')).toBe(false);
- expect(isTransient(undefined)).toBe(false);
- expect(isTransient(null)).toBe(false);
+ test('unproven cleanup stops before diagnostic classification', async () => {
+ const result = await run({
+ command: nodeCmd('console.error("socket hang up");process.exit(1)'),
+ treeController: { cleanup: async () => ({ ok: false, reason: 'tree-survived-kill' }) },
+ });
+ expect(result).toMatchObject({
+ ok: false,
+ reason: 'cleanup-failure',
+ cleanup: 'tree-survived-kill',
+ attempts: 1,
+ });
+ expect(result.log).toContain('cleanup=tree-survived-kill');
});
- test('every signature is a real regex and none is so broad it matches bare text', () => {
- for (const sig of TRANSIENT_SIGNATURES) {
- expect(() => new RegExp(sig)).not.toThrow();
- expect(new RegExp(sig, 'i').test('Building desktop app')).toBe(false);
- }
+ test('exit zero with unproven cleanup fails closed', async () => {
+ const result = await run({
+ command: nodeCmd('process.exit(0)'),
+ treeController: { cleanup: async () => ({ ok: false, reason: 'close-not-observed' }) },
+ });
+ expect(result).toMatchObject({ ok: false, reason: 'cleanup-failure', attempts: 1 });
+ expect(result.log).toContain('reason=control:cleanup-failure outcome=cleanup-failure');
+ expect(result.log).toContain('cleanup=close-not-observed');
+ });
+
+ test('retry warning emits only when a retry is actually scheduled', async () => {
+ const warning = 'duplicate Apple notarization submission risk';
+ const success = await run({ retryWarning: warning });
+ expect(success.log).not.toContain(warning);
+ expect(success.log).not.toContain('bounded failure diagnostic');
+ const terminal = await run({
+ command: nodeCmd('console.error("certificate has expired");process.exit(1)'),
+ retryWarning: warning,
+ });
+ expect(terminal.log).not.toContain(warning);
+ const transient = await run({
+ command: nodeCmd('console.error("socket hang up");process.exit(1)'),
+ retryWarning: warning,
+ });
+ expect(transient.log.match(new RegExp(warning, 'g'))).toHaveLength(2);
+ });
+
+ test('the state machine rejects an invalid attempt bound before installing handlers', async () => {
+ const signals = new EventEmitter();
+ await expect(run({ maxAttempts: 0, signalSource: signals })).rejects.toThrow(
+ /maxAttempts must be an integer from 1 to 3/,
+ );
+ expect(signals.listenerCount('SIGINT')).toBe(0);
+ expect(signals.listenerCount('SIGTERM')).toBe(0);
});
});
-describe('backoff', () => {
- test('grows between attempts and stays bounded for the default attempt count', () => {
- expect(backoffSeconds(1)).toBe(30);
- expect(backoffSeconds(2)).toBeGreaterThan(backoffSeconds(1));
- const total = [...Array(DEFAULT_MAX_ATTEMPTS - 1).keys()]
- .map((i) => backoffSeconds(i + 1))
- .reduce((a, b) => a + b, 0);
- expect(total).toBeLessThan(10 * 60);
+describe('retry delay', () => {
+ test('uses jittered 30s and 60s bases', () => {
+ const evidence = inspect('socket hang up').evidence;
+ expect(computeRetryDelayMs(1, evidence, { randomFn: () => 0, nowMs: 0 })).toBe(30_001);
+ expect(computeRetryDelayMs(2, evidence, { randomFn: () => 0, nowMs: 0 })).toBe(60_001);
+ expect(computeRetryDelayMs(2, evidence, { randomFn: () => 1, nowMs: 0 })).toBeLessThanOrEqual(
+ 65_000,
+ );
+ });
+
+ test('honors both Retry-After forms as a minimum plus positive jitter', () => {
+ const delta = inspect('HTTP 429\nRetry-After: 120', { nowFn: () => 0 }).evidence;
+ expect(computeRetryDelayMs(1, delta, { randomFn: () => 0, nowMs: 0 })).toBe(120_001);
+ const nowMs = Date.parse('2026-09-11T20:00:00Z');
+ const retryAt = new Date(nowMs + 90_000).toUTCString();
+ const date = inspect(`HTTP 429\nRetry-After: ${retryAt}`, { nowFn: () => nowMs }).evidence;
+ expect(computeRetryDelayMs(1, date, { randomFn: () => 0, nowMs })).toBe(90_001);
+ });
+
+ test('does not reinterpret a carried delta Retry-After against a later clock', () => {
+ let nowMs = 0;
+ const evidence = new FailureEvidence({ nowFn: () => nowMs });
+ evidence.ingest(Buffer.from('HTTP 429\nRetry-After: 120\n'));
+ nowMs = 10_000;
+ evidence.ingest(Buffer.from('download cleanup\n'));
+ nowMs = 20_000;
+ evidence.ingest(Buffer.from('another chunk\n'));
+ evidence.finish();
+ expect(evidence.retryAfterEpochMs).toBe(120_000);
+ expect(computeRetryDelayMs(1, evidence, { randomFn: () => 0, nowMs })).toBe(100_001);
+ });
+
+ test('accepts only header-shaped lines and keeps the first valid Retry-After', () => {
+ const evidence = new FailureEvidence({ nowFn: () => 0 });
+ evidence.ingest(
+ Buffer.from('noise Retry-After: 900\nError: Retry-After: 800\n* Retry-After: 700\n'),
+ );
+ evidence.ingest(Buffer.from('< Retry-After: 45\nRetry-After: 120\n'));
+ evidence.finish();
+ expect(evidence.retryAfterEpochMs).toBe(45_000);
});
});
-describe('parseArgs', () => {
- const argv = (...rest) => ['node', 'x', ...rest];
+describe('deadlines and cancellation', () => {
+ const completedAttempt = (text, code = 1, options) => ({
+ code,
+ closeSignal: null,
+ cancellationSignal: null,
+ cancelled: false,
+ timedOut: false,
+ deadlineExpired: false,
+ spawnError: null,
+ cleanup: { ok: true, reason: 'clean' },
+ evidence: inspect(text, options).evidence,
+ });
+ const assertPidDead = (path) => {
+ const pid = Number(readFileSync(path, 'utf8'));
+ expect(() => process.kill(pid, 0)).toThrow();
+ };
- test('splits flags from the command at `--`', () => {
- const p = parseArgs(argv('--label', 'pkg (linux)', '--', 'pnpm', 'exec', 'electron-builder'));
- expect(p.label).toBe('pkg (linux)');
- expect(p.command).toEqual(['pnpm', 'exec', 'electron-builder']);
- expect(p.shell).toBe(false);
- expect(p.maxAttempts).toBe(DEFAULT_MAX_ATTEMPTS);
+ test('real macOS budget arithmetic admits a quick incident-style retry', async () => {
+ const minute = 60_000;
+ const timing = workflowTiming(WORKFLOW_JOBS[0]);
+ let clock = 15 * minute;
+ let attempt = 0;
+ const result = await run({
+ deadlineEpochMs: timing.budgetMs,
+ attemptTimeoutMs: timing.attemptTimeoutMs,
+ cleanupReserveMs: 15_000,
+ nowFn: () => clock,
+ sleepFn: async (ms) => {
+ clock += ms;
+ },
+ attemptRunner: async () => {
+ attempt += 1;
+ clock += minute;
+ return attempt === 1
+ ? completedAttempt('HTTPError: Response code 500 (Internal Server Error)')
+ : completedAttempt('', 0);
+ },
+ });
+ expect(result).toMatchObject({ ok: true, attempts: 2 });
});
- test('a flag-looking token after `--` belongs to the command', () => {
- const p = parseArgs(argv('--', 'pnpm', 'exec', 'electron-builder', '--linux', '--publish', 'never'));
- expect(p.command).toContain('--linux');
- expect(p.command).toContain('never');
+ test('real Windows and Linux budget arithmetic rejects a doomed retry projection', async () => {
+ const minute = 60_000;
+ const windowsTiming = workflowTiming(WORKFLOW_JOBS[1]);
+ const linuxTiming = workflowTiming(WORKFLOW_JOBS[2]);
+ expect(linuxTiming).toEqual(windowsTiming);
+ const execute = async (evidenceText) => {
+ let clock = 19 * minute;
+ return run({
+ deadlineEpochMs: windowsTiming.budgetMs,
+ attemptTimeoutMs: windowsTiming.attemptTimeoutMs,
+ cleanupReserveMs: 15_000,
+ nowFn: () => clock,
+ attemptRunner: async () => {
+ clock += 12 * minute;
+ return completedAttempt(evidenceText, 1, { nowFn: () => clock });
+ },
+ });
+ };
+ const baseline = await execute('socket hang up');
+ const withHeader = await execute('HTTP 429\nRetry-After: 31');
+ for (const result of [baseline, withHeader]) {
+ expect(result).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });
+ expect(result.log).toContain('reason=control:deadline outcome=deadline');
+ expect(result.log).toContain('phase=before-backoff');
+ expect(result.log).toContain('projected-attempt-ms=900000');
+ }
+ expect(baseline.log).not.toContain('retry-after-ms=');
+ expect(baseline.log).toContain('matched=rule:socket-hangup');
+ expect(withHeader.log).toContain('retry-after-ms=31000');
+ expect(withHeader.log).toContain('matched=http:429');
});
- test('requires a separator and a command', () => {
- expect(() => parseArgs(argv('--label', 'x'))).toThrow(/--/);
- expect(() => parseArgs(argv('--label', 'x', '--'))).toThrow(/no command/);
+ test('a genuine Retry-After beyond the wall-clock budget stops explicitly', async () => {
+ const timing = workflowTiming(WORKFLOW_JOBS[1]);
+ let clock = 0;
+ const result = await run({
+ deadlineEpochMs: timing.budgetMs,
+ attemptTimeoutMs: timing.attemptTimeoutMs,
+ nowFn: () => clock,
+ attemptRunner: async () => {
+ clock += 1_000;
+ return completedAttempt('HTTP 429\nRetry-After: 3600', 1, { nowFn: () => clock });
+ },
+ });
+ expect(result).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });
+ expect(result.log).toContain('reason=control:deadline outcome=deadline');
+ expect(result.log).toContain('retry-after-ms=3600000');
+ expect(result.log).toContain('projected-attempt-ms=1250');
});
- test('--shell demands exactly one command string', () => {
- expect(() => parseArgs(argv('--shell', '--', 'a', 'b'))).toThrow(/exactly one/);
- expect(parseArgs(argv('--shell', '--', 'a && b')).shell).toBe(true);
+ test('attempt timeout stops and cleans the child without retrying', async () => {
+ const pidFile = join(scratch, 'timeout-child-pid');
+ const result = await run({
+ command: nodeCmd(
+ `require('fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));setInterval(() => {},1000)`,
+ ),
+ attemptTimeoutMs: 40,
+ deadlineEpochMs: Date.now() + 5_000,
+ });
+ expect(result).toMatchObject({ ok: false, reason: 'attempt-timeout', attempts: 1 });
+ expect(result.log).toContain(
+ 'decision=stop reason=control:attempt-timeout outcome=attempt-timeout attempt=1/3 code=none signal=SIGTERM',
+ );
+ expect(result.log).not.toContain('phase=');
+ assertPidDead(pidFile);
+ });
+
+ test('absolute deadline stops and cleans a live child', async () => {
+ const pidFile = join(scratch, 'deadline-child-pid');
+ const result = await run({
+ command: nodeCmd(
+ `require('fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));setInterval(() => {},1000)`,
+ ),
+ attemptTimeoutMs: 10_000,
+ cleanupReserveMs: 10,
+ deadlineEpochMs: Date.now() + 1_100,
+ });
+ expect(result).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });
+ expect(result.log).toContain('reason=control:deadline outcome=deadline');
+ expect(result.log).toContain('phase=mid-attempt');
+ assertPidDead(pidFile);
+ });
+
+ test('does not start an attempt or backoff that cannot fit the deadline', async () => {
+ const spawnFn = vi.fn();
+ const nowMs = 10_000;
+ const beforeAttempt = await run({
+ spawnFn,
+ nowFn: () => nowMs,
+ deadlineEpochMs: nowMs + 1_099,
+ attemptTimeoutMs: 1_000,
+ cleanupReserveMs: 100,
+ });
+ expect(beforeAttempt).toMatchObject({ ok: false, reason: 'deadline', attempts: 0 });
+ expect(beforeAttempt.log).toContain('reason=control:deadline outcome=deadline');
+ expect(beforeAttempt.log).toContain('phase=before-attempt');
+ expect(spawnFn).not.toHaveBeenCalled();
+ expect(
+ await run({
+ command: nodeCmd('console.error("socket hang up");process.exit(1)'),
+ nowFn: () => nowMs,
+ deadlineEpochMs: nowMs + 31_000,
+ attemptTimeoutMs: 1_000,
+ cleanupReserveMs: 50,
+ }),
+ ).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });
+ });
+
+ test('global deadline exhaustion aborts an active attempt', async () => {
+ let reads = 0;
+ const result = await run({
+ command: nodeCmd('setInterval(() => {}, 1000)'),
+ nowFn: () => {
+ reads += 1;
+ return reads === 1 ? 0 : 1_401;
+ },
+ deadlineEpochMs: 1_500,
+ attemptTimeoutMs: 1_000,
+ cleanupReserveMs: 100,
+ });
+ expect(result).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });
+ expect(result.log).toContain('reason=control:deadline outcome=deadline');
+ });
+
+ test('parent cancellation during a child stops the tree and removes handlers', async () => {
+ const signals = new EventEmitter();
+ const pidFile = join(scratch, 'cancelled-child-pid');
+ const promise = run({
+ command: nodeCmd(
+ `require('fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));console.log('ready');setInterval(() => {},1000)`,
+ ),
+ signalSource: signals,
+ });
+ while (!existsSync(pidFile)) await new Promise((resolve) => setTimeout(resolve, 5));
+ signals.emit('SIGINT');
+ const result = await promise;
+ expect(result).toMatchObject({ ok: false, reason: 'signal', signal: 'SIGINT', attempts: 1 });
+ expect(result.log).toContain('reason=control:signal outcome=signal');
+ expect(signals.listenerCount('SIGINT')).toBe(0);
+ expect(signals.listenerCount('SIGTERM')).toBe(0);
+ assertPidDead(pidFile);
+ });
+
+ test('parent cancellation during backoff prevents another attempt', async () => {
+ const signals = new EventEmitter();
+ const count = join(scratch, 'backoff-cancel-count');
+ let enterBackoff;
+ const backoffStarted = new Promise((resolve) => {
+ enterBackoff = resolve;
+ });
+ writeFileSync(count, '0');
+ const promise = run({
+ command: nodeCmd(
+ `const fs=require('fs');const p=${JSON.stringify(count)};fs.writeFileSync(p,String(+fs.readFileSync(p,'utf8')+1));console.error('socket hang up');process.exit(1)`,
+ ),
+ signalSource: signals,
+ sleepFn: (_ms, signal) =>
+ new Promise((resolve) => {
+ enterBackoff();
+ signal.addEventListener('abort', resolve, { once: true });
+ }),
+ });
+ await backoffStarted;
+ signals.emit('SIGTERM');
+ const result = await promise;
+ expect(result).toMatchObject({
+ ok: false,
+ reason: 'signal',
+ signal: 'SIGTERM',
+ attempts: 1,
+ });
+ expect(result.log).toContain('reason=control:signal outcome=signal');
+ expect(readFileSync(count, 'utf8')).toBe('1');
});
+
+ test.runIf(process.platform !== 'win32')(
+ 'reaps a real grandchild before the next attempt starts',
+ async () => {
+ const helper = join(scratch, 'tree-helper.mjs');
+ const count = join(scratch, 'tree-count');
+ const pidFile = join(scratch, 'grandchild-pid');
+ const overlap = join(scratch, 'tree-overlap');
+ writeFileSync(count, '0');
+ writeFileSync(
+ helper,
+ `import { spawn } from 'node:child_process';import { readFileSync,writeFileSync } from 'node:fs';const [count,pidFile,overlap]=process.argv.slice(2);const n=+readFileSync(count,'utf8')+1;writeFileSync(count,String(n));if(n===1){const child=spawn(process.execPath,['-e','process.on("SIGTERM",()=>{});setInterval(()=>{},1000)'],{stdio:'ignore'});writeFileSync(pidFile,String(child.pid));console.error('socket hang up');process.exit(1)}const pid=+readFileSync(pidFile,'utf8');try{process.kill(pid,0);writeFileSync(overlap,'alive');process.exit(1)}catch{process.exit(0)}`,
+ );
+ const result = await run({
+ command: [process.execPath, helper, count, pidFile, overlap],
+ cleanupGraceMs: 30,
+ });
+ expect(result).toMatchObject({ ok: true, attempts: 2 });
+ expect(existsSync(overlap)).toBe(false);
+ expect(() => process.kill(Number(readFileSync(pidFile, 'utf8')), 0)).toThrow();
+ },
+ 5_000,
+ );
+
+ test.runIf(process.platform !== 'win32')(
+ 'the CLI exits with the same parent signal after cleanup',
+ async () => {
+ const child = spawn(
+ process.execPath,
+ [
+ SCRIPT,
+ '--deadline-epoch-ms',
+ String(Date.now() + 60_000),
+ '--attempt-timeout',
+ '30s',
+ '--',
+ process.execPath,
+ '-e',
+ 'console.log("ready");setInterval(()=>{},1000)',
+ ],
+ { stdio: ['ignore', 'pipe', 'pipe'] },
+ );
+ await new Promise((resolve, reject) => {
+ child.once('error', reject);
+ child.stdout.once('data', resolve);
+ });
+ child.kill('SIGTERM');
+ const closed = await new Promise((resolve) => {
+ child.once('close', (code, signal) => resolve({ code, signal }));
+ });
+ expect(closed).toEqual({ code: null, signal: 'SIGTERM' });
+ },
+ 5_000,
+ );
});
-describe('workflow wiring', () => {
- const step = (needle) => {
- const start = desktopRelease.indexOf(`- name: ${needle}`);
- if (start === -1) throw new Error(`desktop-release.yml has no step named ${needle}`);
- const rest = desktopRelease.slice(start);
- const end = rest.indexOf('\n - name: ');
- return end === -1 ? rest : rest.slice(0, end);
- };
+describe('POSIX owned-tree cleanup reasons', () => {
+ const options = { graceMs: 1, cleanupReserveMs: 1, waitForClose: async () => false };
+
+ test('distinguishes signal-send failure', async () => {
+ const controller = createOwnedTreeController({
+ platform: 'darwin',
+ killFn: (_pid, signal) => {
+ if (signal !== 0) throw Object.assign(new Error('denied'), { code: 'EPERM' });
+ },
+ });
+ expect(await controller.cleanup(4321, options)).toEqual({
+ ok: false,
+ reason: 'signal-send-failure',
+ });
+ });
- test('step() throws on a missing name instead of returning a degenerate slice', () => {
- expect(() => step('This step does not exist')).toThrow(/has no step named/);
+ test('distinguishes a tree surviving SIGKILL', async () => {
+ const controller = createOwnedTreeController({
+ platform: 'linux',
+ killFn: () => {},
+ sleepFn: async () => {},
+ pollIntervalMs: 1,
+ });
+ expect(await controller.cleanup(4321, options)).toEqual({
+ ok: false,
+ reason: 'tree-survived-kill',
+ });
});
- const PACKAGING_STEPS = [
- 'Build + sign + notarize DMG/ZIP',
- 'Package NSIS installers',
- 'Package ${{ matrix.targets }}',
- ];
+ test('distinguishes a missing close observation', async () => {
+ const controller = createOwnedTreeController({
+ platform: 'linux',
+ killFn: () => {
+ throw Object.assign(new Error('gone'), { code: 'ESRCH' });
+ },
+ });
+ expect(await controller.cleanup(4321, options)).toEqual({
+ ok: false,
+ reason: 'close-not-observed',
+ });
+ });
+});
- test.each(PACKAGING_STEPS)('%s routes through the shared retry wrapper', (name) => {
- expect(step(name)).toContain('.github/scripts/retry-transient.mjs');
+describe('Windows owned-tree adapter', () => {
+ test('shell mode uses Bash through the injected Windows spawn boundary', async () => {
+ const calls = [];
+ const spawnFn = (executable, args, options) => {
+ calls.push({ executable, args, options });
+ const child = new EventEmitter();
+ child.stdout = new PassThrough();
+ child.stderr = new PassThrough();
+ process.nextTick(() => child.emit('close', 0, null));
+ return child;
+ };
+ const result = await run({
+ command: ['printf windows-shell'],
+ shell: true,
+ platform: 'win32',
+ spawnFn,
+ });
+ expect(result).toMatchObject({ ok: true, attempts: 1 });
+ expect(calls).toEqual([
+ {
+ executable: 'bash',
+ args: ['-c', 'printf windows-shell'],
+ options: { stdio: ['inherit', 'pipe', 'pipe'], detached: false, windowsHide: true },
+ },
+ ]);
});
- test.each(PACKAGING_STEPS)('%s defines its packaging command exactly once', (name) => {
- const invocations = step(name)
- .split('\n')
- .filter((l) => !/^\s*#/.test(l))
- .filter((l) => l.includes('pnpm exec electron-builder'));
- expect(invocations).toHaveLength(1);
- expect(invocations[0]).toMatch(/^\s*PKG_CMD=/);
+ test('uses taskkill tree mode then forced tree mode after grace', async () => {
+ const calls = [];
+ const controller = createOwnedTreeController({
+ platform: 'win32',
+ taskkillFn: async (args) => {
+ calls.push(args);
+ return { ok: true };
+ },
+ sleepFn: () => Promise.resolve(),
+ });
+ const cleaned = await controller.cleanup(4321, {
+ graceMs: 50,
+ waitForClose: async () => calls.length > 1,
+ });
+ expect(cleaned).toEqual({ ok: true, reason: 'clean' });
+ expect(calls).toEqual([
+ ['/PID', '4321', '/T'],
+ ['/PID', '4321', '/T', '/F'],
+ ]);
});
- test.each(PACKAGING_STEPS)('%s runs that command through the wrapper', (name) => {
- const body = step(name);
- expect(body).toContain('node "$WRAPPER"');
- expect(body).toContain('--shell -- "$PKG_CMD"');
+ test('fails closed when forced taskkill cannot prove cleanup', async () => {
+ const controller = createOwnedTreeController({
+ platform: 'win32',
+ taskkillFn: async () => ({ ok: false }),
+ });
+ expect(
+ await controller.cleanup(4321, {
+ graceMs: 1,
+ waitForClose: async () => false,
+ }),
+ ).toEqual({ ok: false, reason: 'taskkill-failure' });
});
- test.each(PACKAGING_STEPS)('%s still packages when the wrapper is absent', (name) => {
- const body = step(name);
- expect(body).toContain('git -C "${GITHUB_WORKSPACE}" fetch --depth=1 origin "$GITHUB_SHA"');
- expect(body).toContain('bash -c "$PKG_CMD"');
- expect(body).toContain('::warning::Retry wrapper absent');
+ test('distinguishes successful taskkill without a close observation', async () => {
+ const controller = createOwnedTreeController({
+ platform: 'win32',
+ taskkillFn: async () => ({ ok: true }),
+ });
+ expect(
+ await controller.cleanup(4321, {
+ graceMs: 1,
+ waitForClose: async () => false,
+ }),
+ ).toEqual({ ok: false, reason: 'close-not-observed' });
});
+});
+
+describe('parseArgs', () => {
+ const argv = (...rest) => ['node', 'x', ...rest];
- test('the retry allowlist is not duplicated back into the workflow', () => {
- expect(desktopRelease).not.toContain('transient_signatures=');
+ test('parses the bounded caller contract', () => {
+ expect(
+ parseArgs(
+ argv(
+ '--label',
+ 'pkg (linux)',
+ '--max-attempts',
+ '3',
+ '--deadline-epoch-ms',
+ '2000000000000',
+ '--attempt-timeout',
+ '15m',
+ '--retry-warning',
+ 'retry side effect',
+ '--shell',
+ '--',
+ 'pnpm exec electron-builder',
+ ),
+ ),
+ ).toEqual({
+ label: 'pkg (linux)',
+ maxAttempts: 3,
+ deadlineEpochMs: 2_000_000_000_000,
+ attemptTimeoutMs: 900_000,
+ retryWarning: 'retry side effect',
+ shell: true,
+ command: ['pnpm exec electron-builder'],
+ });
});
- const preamble = (name) => {
- const body = step(name);
- const start = body.indexOf('WRAPPER="');
- const end = body.indexOf('PKG_CMD=');
- expect(start, `${name}: WRAPPER= not found`).toBeGreaterThan(-1);
- expect(end, `${name}: PKG_CMD= not found`).toBeGreaterThan(start);
- return body.slice(start, end);
- };
+ test('rejects missing budgets, excess attempts, and malformed shell commands', () => {
+ expect(() => parseArgs(argv('--', 'true'))).toThrow(/deadline/);
+ expect(() => parseArgs(argv('--deadline-epoch-ms', '2000000000000', '--', 'true'))).toThrow(
+ /attempt-timeout/,
+ );
+ expect(() =>
+ parseArgs(
+ argv(
+ '--max-attempts',
+ '4',
+ '--deadline-epoch-ms',
+ '2000000000000',
+ '--attempt-timeout',
+ '1m',
+ '--',
+ 'true',
+ ),
+ ),
+ ).toThrow(/max-attempts/);
+ expect(() =>
+ parseArgs(
+ argv(
+ '--shell',
+ '--deadline-epoch-ms',
+ '2000000000000',
+ '--attempt-timeout',
+ '1m',
+ '--',
+ 'a',
+ 'b',
+ ),
+ ),
+ ).toThrow(/exactly one/);
+ expect(() =>
+ parseArgs(
+ argv(
+ '--unknown',
+ 'value',
+ '--deadline-epoch-ms',
+ '2000000000000',
+ '--attempt-timeout',
+ '1m',
+ '--',
+ 'true',
+ ),
+ ),
+ ).toThrow(/unknown flag/);
+ expect(() =>
+ parseArgs(
+ argv(
+ '--label',
+ '--deadline-epoch-ms',
+ '2000000000000',
+ '--attempt-timeout',
+ '1m',
+ '--',
+ 'true',
+ ),
+ ),
+ ).toThrow(/--label requires a value/);
+ });
+
+ test('accepts an explicit empty retry warning', () => {
+ const parsed = parseArgs(
+ argv(
+ '--deadline-epoch-ms',
+ '2000000000000',
+ '--attempt-timeout',
+ '1m',
+ '--retry-warning',
+ '',
+ '--',
+ 'true',
+ ),
+ );
+ expect(parsed.retryWarning).toBe('');
+ });
+
+ test('rejects an empty label while permitting only the optional warning to be empty', () => {
+ expect(() =>
+ parseArgs(
+ argv(
+ '--label',
+ '',
+ '--deadline-epoch-ms',
+ '2000000000000',
+ '--attempt-timeout',
+ '1m',
+ '--',
+ 'true',
+ ),
+ ),
+ ).toThrow(/--label requires a value/);
+ });
+});
- let snippetSeq = 0;
- const runUnderErrexit = (snippet) => {
- const file = join(scratch, `preamble-${(snippetSeq += 1)}.sh`);
+describe('workflow wiring', () => {
+ const job = workflowJob;
+ const step = workflowStep;
+ const jobs = WORKFLOW_JOBS;
+
+ test('download-integrity recovery depends on uncached packaging downloads', () => {
+ expect(desktopRelease).not.toMatch(
+ /ELECTRON_BUILDER_CACHE|ELECTRON_CACHE|electron_config_cache|electronDownload|[Cc]aches?[\\/]electron|electron(?:-builder)?[\\/][Cc]ache/,
+ );
+ });
+ const materializationBlock = (packageStep) => {
+ const start = packageStep.indexOf('WRAPPER="');
+ const notice = packageStep.indexOf(
+ 'echo "::notice::Retry wrapper materialized from workflow SHA $' + '{GITHUB_WORKFLOW_SHA}."',
+ );
+ if (start === -1 || notice === -1) throw new Error('materialization block not found');
+ const end = packageStep.indexOf('\n', notice);
+ return packageStep.slice(start, end);
+ };
+ const packageSteps = () => jobs.map((row) => step(job(row[0]), row[3]));
+ const runMaterialization = ({
+ fetchFailures = 0,
+ showFailure = false,
+ emptyShow = false,
+ mvFailure = false,
+ mutate,
+ } = {}) => {
+ const dir = mkdtempSync(join(scratch, 'materialize-'));
+ let block = materializationBlock(packageSteps()[0]);
+ if (mutate) block = mutate(block);
+ const script = join(dir, 'run.sh');
writeFileSync(
- file,
+ script,
[
- 'git() { case "$*" in *checkout*) return 1 ;; *) return 0 ;; esac; }',
- 'GITHUB_WORKSPACE=/nonexistent-workspace',
- 'GITHUB_SHA=deadbeefdeadbeefdeadbeefdeadbeefdeadbeef',
- snippet,
- 'echo REACHED_FALLBACK',
+ 'set -euo pipefail',
+ 'git() {',
+ ' if [[ " $* " == *" fetch "* ]]; then',
+ ' n=$(cat "$RUNNER_TEMP/fetch-count")',
+ ' n=$((n + 1))',
+ ' printf %s "$n" > "$RUNNER_TEMP/fetch-count"',
+ ' [[ "$n" -gt "$FETCH_FAILURES" ]]',
+ ' return',
+ ' fi',
+ ' if [[ "$SHOW_FAILURE" == true ]]; then printf partial; return 1; fi',
+ ' if [[ "$EMPTY_SHOW" == true ]]; then return 0; fi',
+ " printf '%s\\n' '#!/usr/bin/env node'",
+ '}',
+ 'sleep() { echo "SLEEP:$1"; }',
+ ...(mvFailure ? ['mv() { return 1; }'] : []),
+ block,
+ 'echo REACHED',
].join('\n'),
);
- return spawnSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', file], {
+ writeFileSync(join(dir, 'fetch-count'), '0');
+ const result = spawnSync('bash', ['--noprofile', '--norc', script], {
encoding: 'utf8',
+ env: {
+ ...process.env,
+ RUNNER_TEMP: dir,
+ GITHUB_WORKSPACE: dir,
+ GITHUB_WORKFLOW_SHA: 'deadbeef',
+ FETCH_FAILURES: String(fetchFailures),
+ SHOW_FAILURE: String(showFailure),
+ EMPTY_SHOW: String(emptyShow),
+ },
});
+ return { ...result, dir, files: readdirSync(dir) };
};
- test.each(PACKAGING_STEPS)('%s reaches its fallback when sourcing fails', (name) => {
- const r = runUnderErrexit(preamble(name));
- expect(r.status, `sourcing failure must not abort the step:\n${r.stderr}`).toBe(0);
- expect(r.stdout).toContain('REACHED_FALLBACK');
+ test.each(jobs)('%s starts an absolute %sm clock before checkout', (name, budget) => {
+ const body = job(name);
+ expect(body.indexOf('- name: Start packaging deadline')).toBeLessThan(
+ body.indexOf('- uses: actions/checkout'),
+ );
+ expect(step(body, 'Start packaging deadline')).toContain(
+ `PACKAGING_BUDGET_MINUTES: "${budget}"`,
+ );
+ });
+
+ test.each(jobs)(
+ '%s always materializes workflow-SHA tooling into RUNNER_TEMP',
+ (name, _budget, _timeout, packageName) => {
+ const packageStep = step(job(name), packageName);
+ expect(packageStep).toContain('GITHUB_WORKFLOW_SHA: $' + '{{ github.workflow_sha }}');
+ expect(packageStep).toContain('WRAPPER="$' + '{RUNNER_TEMP}/retry-transient.mjs"');
+ expect(packageStep).toContain('fetch --no-tags --depth=1 origin "$GITHUB_WORKFLOW_SHA"');
+ expect(packageStep).toContain(
+ 'show "$GITHUB_WORKFLOW_SHA:.github/scripts/retry-transient.mjs"',
+ );
+ expect(packageStep).not.toContain('GITHUB_WORKSPACE}/.github/scripts/retry-transient.mjs');
+ expect(packageStep).not.toMatch(/if \[\[ ! -f "\$WRAPPER"/);
+ expect(packageStep).not.toContain('packaging without transient retry');
+ },
+ );
+
+ test('all three workflow-SHA materialization blocks are byte-equivalent', () => {
+ const blocks = packageSteps().map(materializationBlock);
+ expect(new Set(blocks).size).toBe(1);
+ expect(blocks[0]).not.toMatch(/do\s+rm -f "\$WRAPPER_TMP"/);
+ expect(blocks[0]).toMatch(
+ /\[\[ -s "\$WRAPPER_TMP" \]\] &&\s+mv "\$WRAPPER_TMP" "\$WRAPPER"; then/,
+ );
+ });
+
+ test('materialization retries transient fetch failure with bounded delays', () => {
+ const result = runMaterialization({ fetchFailures: 1 });
+ expect(result.status).toBe(0);
+ expect(result.stdout).toContain('SLEEP:5');
+ expect(result.stdout).not.toContain('SLEEP:10');
+ expect(readFileSync(join(result.dir, 'retry-transient.mjs'), 'utf8')).toContain(
+ '#!/usr/bin/env node',
+ );
});
- test('the errexit harness actually discriminates (mutation control)', () => {
- const unExempt = [
- 'WRAPPER="/nonexistent/retry-transient.mjs"',
- 'if [[ ! -f "$WRAPPER" ]]; then',
- ' git fetch --depth=1 origin "$GITHUB_SHA" >/dev/null 2>&1 &&',
- ' git checkout "$GITHUB_SHA" -- .github/scripts/retry-transient.mjs 2>/dev/null',
- 'fi',
- ].join('\n');
- const r = runUnderErrexit(unExempt);
- expect(r.status).not.toBe(0);
- expect(r.stdout).not.toContain('REACHED_FALLBACK');
+ test('materialization fails before mv and cleans temporary output', () => {
+ const terminal = runMaterialization({ showFailure: true });
+ expect(terminal.status).toBe(1);
+ expect(terminal.stdout).toContain('SLEEP:5');
+ expect(terminal.stdout).toContain('SLEEP:10');
+ expect(terminal.stdout).toContain('::error::Failed to materialize retry wrapper');
+ expect(terminal.stdout).not.toContain('REACHED');
+ expect(terminal.files.some((file) => file.startsWith('retry-transient.mjs'))).toBe(false);
+
+ const trapped = runMaterialization({ mvFailure: true });
+ expect(trapped.status).toBe(1);
+ expect(trapped.stdout).toContain('SLEEP:5');
+ expect(trapped.stdout).toContain('SLEEP:10');
+ expect(trapped.stdout).toContain('::error::Failed to materialize retry wrapper');
+ expect(trapped.files.some((file) => file.includes('.tmp.'))).toBe(false);
+ });
+
+ test('zero-byte successful show reaches the bounded final error', () => {
+ const result = runMaterialization({ emptyShow: true });
+ expect(result.status).toBe(1);
+ expect(result.stdout).toContain('::error::Failed to materialize retry wrapper');
+ expect(result.stdout).not.toContain('REACHED');
+ expect(result.files.some((file) => file.startsWith('retry-transient.mjs'))).toBe(false);
+ });
+
+ test('the materialization harness discriminates a missing terminal exit', () => {
+ const result = runMaterialization({
+ fetchFailures: 99,
+ mutate: (block) => block.replace(' exit 1', ' true'),
+ });
+ expect(result.status).toBe(0);
+ expect(result.stdout).toContain('REACHED');
+ });
+
+ test.each(jobs)(
+ '%s compiles once outside retry and passes exact budgets',
+ (name, _budget, timeout, packageName) => {
+ const body = job(name);
+ expect(body.indexOf('- name: Build desktop main/preload/renderer')).toBeLessThan(
+ body.indexOf(`- name: ${packageName}`),
+ );
+ const packageStep = step(body, packageName);
+ expect(packageStep).not.toContain('pnpm run build:desktop');
+ expect(packageStep).toContain(`--attempt-timeout "${timeout}"`);
+ expect(packageStep).toContain('--deadline-epoch-ms "$PACKAGING_DEADLINE_EPOCH_MS"');
+ const command = packageStep.split('\n').find((line) => /^\s*PKG_CMD:/.test(line));
+ expect(command).toContain("PKG_CMD: 'rm -rf dist-desktop && pnpm exec electron-builder");
+ expect(command).toContain('--publish never');
+ },
+ );
+
+ test('logs duplicate Apple submissions and keeps strict downstream gates outside retry', () => {
+ const macPackage = step(job('build-macos'), 'Build + sign + notarize DMG/ZIP');
+ expect(macPackage).toContain('duplicate Apple notarization submission');
+ expect(macPackage).toContain('at most 2');
+ expect(macPackage).toContain('--retry-warning');
+ expect(macPackage).not.toMatch(/echo .*duplicate Apple notarization submission/);
+ for (const [body, packageName, downstream] of [
+ [
+ job('build-macos'),
+ 'Build + sign + notarize DMG/ZIP',
+ [
+ 'Attest the signed macOS app',
+ 'Smoke the packaged DMG (FR5b)',
+ 'Upload macOS release assets for the fan-in publisher',
+ ],
+ ],
+ [
+ job('build-windows'),
+ 'Package NSIS installers (x64 + arm64, signed)',
+ [
+ 'Attest signed Windows packages',
+ 'Assert the packaged asar carries its dependencies',
+ 'Upload Windows release assets for the fan-in publisher',
+ ],
+ ],
+ [
+ job('build-linux'),
+ 'Package $' + '{{ matrix.targets }}',
+ [
+ 'Assert the packaged asar carries its dependencies',
+ 'Assert app-update.yml + package-type are present and channel-correct',
+ 'Upload Linux $' + '{{ matrix.arch }} release assets for the fan-in publisher',
+ ],
+ ],
+ ]) {
+ const packageAt = body.indexOf(`- name: ${packageName}`);
+ const packageStep = step(body, packageName);
+ for (const name of downstream) {
+ expect(body.indexOf(`- name: ${name}`)).toBeGreaterThan(packageAt);
+ expect(packageStep).not.toContain(name);
+ }
+ }
+ for (const gate of [
+ 'Assert the complete cross-platform inventory',
+ 'Verify the Release carries the full inventory',
+ 'Promote draft release to published',
+ ]) {
+ expect(desktopRelease).toContain(`- name: ${gate}`);
+ }
});
});
diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml
index 5cc22c9a2..0068d5575 100644
--- a/.github/workflows/desktop-release.yml
+++ b/.github/workflows/desktop-release.yml
@@ -497,11 +497,6 @@ jobs:
# electron-builder compiles `build/okglass.icon` β†’ `Assets.car` via
# `actool`, which only ships with Xcode 26+.
runs-on: macos-26
- # 75 min budget. A signed build + notary round-trip is ~20-30 min; the
- # build step retries once on a transient failure (see below), so
- # the worst case is two full signed builds back-to-back. 75 min covers
- # that plus cold caches and notary queue spikes. desktop-build.yml has no
- # retry and stays at 45.
timeout-minutes: 75
permissions:
# Explicit so a future step needing more fails at review, not with a
@@ -514,6 +509,13 @@ jobs:
smoke_verdict: ${{ steps.dmg_smoke.outputs.verdict }}
smoke_reason: ${{ steps.dmg_smoke.outputs.reason }}
steps:
+ - name: Start packaging deadline
+ env:
+ PACKAGING_BUDGET_MINUTES: "65"
+ run: |
+ set -euo pipefail
+ echo "PACKAGING_DEADLINE_EPOCH_MS=$(( $(date +%s) * 1000 + PACKAGING_BUDGET_MINUTES * 60 * 1000 ))" >> "$GITHUB_ENV"
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.client_payload.ref || github.ref }}
@@ -790,6 +792,10 @@ jobs:
working-directory: packages/desktop
run: node scripts/stage-parcel-watcher.mjs
+ - name: Build desktop main/preload/renderer
+ working-directory: packages/desktop
+ run: pnpm run build:desktop
+
- name: Build + sign + notarize DMG/ZIP
# electron-builder signs with the identity in CSC_KEYCHAIN (pushed
# into $GITHUB_ENV by the "Prepare signing keychain" step). CSC_LINK
@@ -823,6 +829,8 @@ jobs:
# via client_payload (see the Windows job's twin comment).
BUILD_VERSION: ${{ steps.channel.outputs.version }}
BUILD_CHANNEL: ${{ steps.channel.outputs.channel }}
+ GITHUB_WORKFLOW_SHA: ${{ github.workflow_sha }}
+ PKG_CMD: 'rm -rf dist-desktop && pnpm exec electron-builder --mac --publish never --config.extraMetadata.version="${BUILD_VERSION}" --config.publish.channel="${BUILD_CHANNEL}"'
working-directory: packages/desktop
run: |
# extraMetadata.version override: kept as defense-in-depth so the
@@ -836,53 +844,32 @@ jobs:
# asset's metadata.version; those are baked in at Build time, so
# the workspace-wide on-disk override step above is what makes
# them match the tag. See PRD-6650.
- #
- # Packaging is retried on TRANSIENT signing/notarize/download failures
- # only, through the shared wrapper. The macOS-specific classes it
- # covers, none of which electron-builder will retry itself:
- # - codesign's timestamp step calling Apple's TSA (timestamp.apple.com).
- # Each `codesign --timestamp` is a single un-retried shell-out, so a
- # dropped TSA response aborts the whole build. This is the class that
- # stuck v0.30.2 as a draft (run 29445364406).
- # - @electron/notarize's notarytool HTTP round-trip to
- # appstoreconnect.apple.com timing out (NSURLError -1001 /
- # HTTPError(statusCode: nil)), which failed attempt 1 of that same run.
- # - a bare EOF fetching the Electron runtime zip during packaging
- # (app-builder's DownloadNoRetry path aborts on one dropped connection).
- # Re-running the whole invocation is the only lever available. The
- # allowlist that gates the retry β€” and the guarantee that a real
- # cert/notary/entitlements/build failure fails LOUD on the first attempt
- # rather than being masked β€” lives in the wrapper.
- set -uo pipefail
- # The wrapper is CI tooling, not release content, so take it from the
- # workflow's own SHA when the release checkout predates it β€” same
- # reasoning as the smoke harness below. This is not a rare path:
- # promote-stable tags the stable at the soaked beta's SHA, which is by
- # definition older than any fix landed since. Best-effort: if it cannot
- # be obtained, package WITHOUT retry rather than block the release.
- WRAPPER="${GITHUB_WORKSPACE}/.github/scripts/retry-transient.mjs"
- if [[ ! -f "$WRAPPER" ]]; then
- # The fetch/checkout pair sits in an `if` CONDITION deliberately.
- # Steps run under errexit, and bash exempts only the NON-final
- # commands of an `&&` list β€” so a bare failing checkout would exit
- # the shell before reaching the fallback below, defeating the
- # best-effort behavior this exists for. A condition is exempt.
- if git -C "${GITHUB_WORKSPACE}" fetch --depth=1 origin "$GITHUB_SHA" >/dev/null 2>&1 &&
- git -C "${GITHUB_WORKSPACE}" checkout "$GITHUB_SHA" -- .github/scripts/retry-transient.mjs 2>/dev/null; then
- echo "::notice::Retry wrapper taken from ${GITHUB_SHA} rather than the release ref."
+ set -euo pipefail
+ WRAPPER="${RUNNER_TEMP}/retry-transient.mjs"
+ WRAPPER_TMP="${WRAPPER}.tmp.$$"
+ trap 'rm -f "$WRAPPER_TMP"' EXIT
+ materialized=false
+ for materialize_attempt in 1 2 3; do
+ if git -C "${GITHUB_WORKSPACE}" fetch --no-tags --depth=1 origin "$GITHUB_WORKFLOW_SHA" &&
+ git -C "${GITHUB_WORKSPACE}" show "$GITHUB_WORKFLOW_SHA:.github/scripts/retry-transient.mjs" > "$WRAPPER_TMP" &&
+ [[ -s "$WRAPPER_TMP" ]] &&
+ mv "$WRAPPER_TMP" "$WRAPPER"; then
+ materialized=true
+ break
fi
+ rm -f "$WRAPPER_TMP"
+ if [[ "$materialize_attempt" -lt 3 ]]; then
+ materialize_delay=$((materialize_attempt * 5))
+ echo "::warning::Retry wrapper materialization attempt ${materialize_attempt}/3 failed; retrying in ${materialize_delay}s."
+ sleep "$materialize_delay"
+ fi
+ done
+ if [[ "$materialized" != true ]]; then
+ echo "::error::Failed to materialize retry wrapper from workflow SHA ${GITHUB_WORKFLOW_SHA} after 3 attempts."
+ exit 1
fi
- # Defined once, single-quoted ON PURPOSE: it is handed to the wrapper as
- # a literal string and expanded by the inner shell, which inherits this
- # step's env. Expanding here would bake attempt 1's values into retries.
- # shellcheck disable=SC2016
- PKG_CMD='pnpm run build:desktop && pnpm exec electron-builder --mac --publish never --config.extraMetadata.version="${BUILD_VERSION}" --config.publish.channel="${BUILD_CHANNEL}"'
- if [[ -f "$WRAPPER" ]]; then
- node "$WRAPPER" --label "electron-builder (macOS)" --shell -- "$PKG_CMD"
- else
- echo "::warning::Retry wrapper absent at this ref; packaging without transient retry."
- bash -c "$PKG_CMD"
- fi
+ echo "::notice::Retry wrapper materialized from workflow SHA ${GITHUB_WORKFLOW_SHA}."
+ node "$WRAPPER" --label "electron-builder (macOS)" --max-attempts 3 --deadline-epoch-ms "$PACKAGING_DEADLINE_EPOCH_MS" --attempt-timeout "30m" --retry-warning "A macOS packaging retry can create a duplicate Apple notarization submission; up to 3 attempts bound this to at most 2 duplicates." --shell -- "$PKG_CMD"
# Placed HERE, not at the end of the job (where desktop-build.yml keeps
# its twin): release-cascade-shape.test.mjs forbids `if: always()` on any
@@ -1032,6 +1019,13 @@ jobs:
run:
shell: bash
steps:
+ - name: Start packaging deadline
+ env:
+ PACKAGING_BUDGET_MINUTES: "35"
+ run: |
+ set -euo pipefail
+ echo "PACKAGING_DEADLINE_EPOCH_MS=$(( $(date +%s) * 1000 + PACKAGING_BUDGET_MINUTES * 60 * 1000 ))" >> "$GITHUB_ENV"
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.client_payload.ref || github.ref }}
@@ -1134,6 +1128,8 @@ jobs:
AZURE_SIGNING_PUBLISHER: ${{ vars.AZURE_SIGNING_PUBLISHER }}
BUILD_VERSION: ${{ needs.prepare.outputs.version }}
BUILD_CHANNEL: ${{ needs.prepare.outputs.channel }}
+ GITHUB_WORKFLOW_SHA: ${{ github.workflow_sha }}
+ PKG_CMD: 'rm -rf dist-desktop && pnpm exec electron-builder --win --publish never --config.extraMetadata.version="${BUILD_VERSION}" --config.publish.channel="${BUILD_CHANNEL}" --config.win.azureSignOptions.endpoint="${AZURE_SIGNING_ENDPOINT}" --config.win.azureSignOptions.codeSigningAccountName="${AZURE_SIGNING_ACCOUNT}" --config.win.azureSignOptions.certificateProfileName="${AZURE_SIGNING_PROFILE}" --config.win.azureSignOptions.publisherName="${AZURE_SIGNING_PUBLISHER}"'
run: |
set -euo pipefail
missing=()
@@ -1149,31 +1145,31 @@ jobs:
exit 1
fi
echo "::notice::Azure Trusted Signing credentials present β€” building SIGNED installers."
- # Signing here is a network round-trip to the Azure Trusted Signing
- # endpoint, so this invocation carries the same transient exposure the
- # macOS job does. The wrapper still fails loud on a real signing
- # rejection or build error β€” only allowlisted infra flakes retry.
- # Wrapper sourcing + single-quoting rationale: see the macOS step.
- WRAPPER="${GITHUB_WORKSPACE}/.github/scripts/retry-transient.mjs"
- if [[ ! -f "$WRAPPER" ]]; then
- # The fetch/checkout pair sits in an `if` CONDITION deliberately.
- # Steps run under errexit, and bash exempts only the NON-final
- # commands of an `&&` list β€” so a bare failing checkout would exit
- # the shell before reaching the fallback below, defeating the
- # best-effort behavior this exists for. A condition is exempt.
- if git -C "${GITHUB_WORKSPACE}" fetch --depth=1 origin "$GITHUB_SHA" >/dev/null 2>&1 &&
- git -C "${GITHUB_WORKSPACE}" checkout "$GITHUB_SHA" -- .github/scripts/retry-transient.mjs 2>/dev/null; then
- echo "::notice::Retry wrapper taken from ${GITHUB_SHA} rather than the release ref."
+ WRAPPER="${RUNNER_TEMP}/retry-transient.mjs"
+ WRAPPER_TMP="${WRAPPER}.tmp.$$"
+ trap 'rm -f "$WRAPPER_TMP"' EXIT
+ materialized=false
+ for materialize_attempt in 1 2 3; do
+ if git -C "${GITHUB_WORKSPACE}" fetch --no-tags --depth=1 origin "$GITHUB_WORKFLOW_SHA" &&
+ git -C "${GITHUB_WORKSPACE}" show "$GITHUB_WORKFLOW_SHA:.github/scripts/retry-transient.mjs" > "$WRAPPER_TMP" &&
+ [[ -s "$WRAPPER_TMP" ]] &&
+ mv "$WRAPPER_TMP" "$WRAPPER"; then
+ materialized=true
+ break
fi
+ rm -f "$WRAPPER_TMP"
+ if [[ "$materialize_attempt" -lt 3 ]]; then
+ materialize_delay=$((materialize_attempt * 5))
+ echo "::warning::Retry wrapper materialization attempt ${materialize_attempt}/3 failed; retrying in ${materialize_delay}s."
+ sleep "$materialize_delay"
+ fi
+ done
+ if [[ "$materialized" != true ]]; then
+ echo "::error::Failed to materialize retry wrapper from workflow SHA ${GITHUB_WORKFLOW_SHA} after 3 attempts."
+ exit 1
fi
- # shellcheck disable=SC2016
- PKG_CMD='pnpm exec electron-builder --win --publish never --config.extraMetadata.version="${BUILD_VERSION}" --config.publish.channel="${BUILD_CHANNEL}" --config.win.azureSignOptions.endpoint="${AZURE_SIGNING_ENDPOINT}" --config.win.azureSignOptions.codeSigningAccountName="${AZURE_SIGNING_ACCOUNT}" --config.win.azureSignOptions.certificateProfileName="${AZURE_SIGNING_PROFILE}" --config.win.azureSignOptions.publisherName="${AZURE_SIGNING_PUBLISHER}"'
- if [[ -f "$WRAPPER" ]]; then
- node "$WRAPPER" --label "electron-builder (Windows)" --shell -- "$PKG_CMD"
- else
- echo "::warning::Retry wrapper absent at this ref; packaging without transient retry."
- bash -c "$PKG_CMD"
- fi
+ echo "::notice::Retry wrapper materialized from workflow SHA ${GITHUB_WORKFLOW_SHA}."
+ node "$WRAPPER" --label "electron-builder (Windows)" --max-attempts 3 --deadline-epoch-ms "$PACKAGING_DEADLINE_EPOCH_MS" --attempt-timeout "15m" --shell -- "$PKG_CMD"
# This workflow always executes main's YAML (repository_dispatch), while
# the checkout above follows `client_payload[ref]` β€” typically a tag
@@ -1354,6 +1350,13 @@ jobs:
targets: deb:arm64 rpm:arm64
unpacked_dir: linux-arm64-unpacked
steps:
+ - name: Start packaging deadline
+ env:
+ PACKAGING_BUDGET_MINUTES: "35"
+ run: |
+ set -euo pipefail
+ echo "PACKAGING_DEADLINE_EPOCH_MS=$(( $(date +%s) * 1000 + PACKAGING_BUDGET_MINUTES * 60 * 1000 ))" >> "$GITHUB_ENV"
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.client_payload.ref || github.ref }}
@@ -1482,33 +1485,35 @@ jobs:
working-directory: packages/desktop
env:
LINUX_TARGETS: ${{ matrix.targets }}
+ GITHUB_WORKFLOW_SHA: ${{ github.workflow_sha }}
+ PKG_CMD: 'rm -rf dist-desktop && pnpm exec electron-builder --linux ${LINUX_TARGETS} --publish never --config electron-builder.release-linux.yml'
run: |
- # electron-builder downloads its fpm packaging backend on first use,
- # and a dropped connection there aborts the whole build. That is what
- # blocked the v0.52.2 and v0.52.3 cuts, on a step that had no retry
- # while the macOS job had carried one for months.
- set -uo pipefail
- # Wrapper sourcing + single-quoting rationale: see the macOS step.
- WRAPPER="${GITHUB_WORKSPACE}/.github/scripts/retry-transient.mjs"
- if [[ ! -f "$WRAPPER" ]]; then
- # The fetch/checkout pair sits in an `if` CONDITION deliberately.
- # Steps run under errexit, and bash exempts only the NON-final
- # commands of an `&&` list β€” so a bare failing checkout would exit
- # the shell before reaching the fallback below, defeating the
- # best-effort behavior this exists for. A condition is exempt.
- if git -C "${GITHUB_WORKSPACE}" fetch --depth=1 origin "$GITHUB_SHA" >/dev/null 2>&1 &&
- git -C "${GITHUB_WORKSPACE}" checkout "$GITHUB_SHA" -- .github/scripts/retry-transient.mjs 2>/dev/null; then
- echo "::notice::Retry wrapper taken from ${GITHUB_SHA} rather than the release ref."
+ set -euo pipefail
+ WRAPPER="${RUNNER_TEMP}/retry-transient.mjs"
+ WRAPPER_TMP="${WRAPPER}.tmp.$$"
+ trap 'rm -f "$WRAPPER_TMP"' EXIT
+ materialized=false
+ for materialize_attempt in 1 2 3; do
+ if git -C "${GITHUB_WORKSPACE}" fetch --no-tags --depth=1 origin "$GITHUB_WORKFLOW_SHA" &&
+ git -C "${GITHUB_WORKSPACE}" show "$GITHUB_WORKFLOW_SHA:.github/scripts/retry-transient.mjs" > "$WRAPPER_TMP" &&
+ [[ -s "$WRAPPER_TMP" ]] &&
+ mv "$WRAPPER_TMP" "$WRAPPER"; then
+ materialized=true
+ break
fi
+ rm -f "$WRAPPER_TMP"
+ if [[ "$materialize_attempt" -lt 3 ]]; then
+ materialize_delay=$((materialize_attempt * 5))
+ echo "::warning::Retry wrapper materialization attempt ${materialize_attempt}/3 failed; retrying in ${materialize_delay}s."
+ sleep "$materialize_delay"
+ fi
+ done
+ if [[ "$materialized" != true ]]; then
+ echo "::error::Failed to materialize retry wrapper from workflow SHA ${GITHUB_WORKFLOW_SHA} after 3 attempts."
+ exit 1
fi
- # shellcheck disable=SC2016
- PKG_CMD='pnpm exec electron-builder --linux ${LINUX_TARGETS} --publish never --config electron-builder.release-linux.yml'
- if [[ -f "$WRAPPER" ]]; then
- node "$WRAPPER" --label "electron-builder (Linux ${LINUX_TARGETS})" --shell -- "$PKG_CMD"
- else
- echo "::warning::Retry wrapper absent at this ref; packaging without transient retry."
- bash -c "$PKG_CMD"
- fi
+ echo "::notice::Retry wrapper materialized from workflow SHA ${GITHUB_WORKFLOW_SHA}."
+ node "$WRAPPER" --label "electron-builder (Linux ${LINUX_TARGETS})" --max-attempts 3 --deadline-epoch-ms "$PACKAGING_DEADLINE_EPOCH_MS" --attempt-timeout "15m" --shell -- "$PKG_CMD"
# Fail before staging an installer that cannot start (see the Windows
# step for the full rationale). This is the control that would have
diff --git a/packages/desktop/scripts/afterPack.mjs b/packages/desktop/scripts/afterPack.mjs
index b718bea4b..a25797503 100644
--- a/packages/desktop/scripts/afterPack.mjs
+++ b/packages/desktop/scripts/afterPack.mjs
@@ -3,29 +3,11 @@ import { chmodSync, copyFileSync, existsSync, mkdirSync, writeFileSync } from 'n
import { dirname, join } from 'node:path';
import { FuseV1Options, FuseVersion, flipFuses } from '@electron/fuses';
import { ensureNodePtySpawnHelperExecutable } from './ensure-node-pty-exec.mjs';
+import { createFuseFailure } from './packaging-diagnostics.mjs';
import { resolveElectronBinary } from './resolve-electron-binary.mjs';
import { targetFuses } from './target-fuses.mjs';
-export default async function afterPack(context) {
- const { appOutDir, packager, electronPlatformName } = context;
-
- if (appOutDir.endsWith('-temp')) {
- console.log(
- `[afterPack] skipping per-arch temp "${appOutDir}" β€” fuses flip on the merged universal app`,
- );
- return;
- }
-
- const appName = packager.appInfo.productFilename;
- const electronBinary = resolveElectronBinary(electronPlatformName, appOutDir, packager);
-
- if (!existsSync(electronBinary)) {
- throw new Error(
- `[afterPack] Electron binary not found at ${electronBinary}. ` +
- `Expected electron-builder to have packed the app before afterPack ran.`,
- );
- }
-
+async function flipElectronFuses(electronBinary, electronPlatformName) {
console.log(`[afterPack] flipping fuses on ${electronBinary}`);
for (const [optIndex, value] of Object.entries(targetFuses)) {
const name = FuseV1Options[Number(optIndex)];
@@ -39,7 +21,7 @@ export default async function afterPack(context) {
...targetFuses,
});
} catch (err) {
- throw new Error(
+ throw createFuseFailure(
`[afterPack] fuse flip failed on ${electronBinary}: ${
err instanceof Error ? err.message : String(err)
}`,
@@ -48,6 +30,29 @@ export default async function afterPack(context) {
}
console.log('[afterPack] fuses flipped successfully; electron-builder will re-sign next');
+}
+
+export default async function afterPack(context) {
+ const { appOutDir, packager, electronPlatformName } = context;
+
+ if (appOutDir.endsWith('-temp')) {
+ console.log(
+ `[afterPack] skipping per-arch temp "${appOutDir}" β€” fuses flip on the merged universal app`,
+ );
+ return;
+ }
+
+ const appName = packager.appInfo.productFilename;
+ const electronBinary = resolveElectronBinary(electronPlatformName, appOutDir, packager);
+
+ if (!existsSync(electronBinary)) {
+ throw new Error(
+ `[afterPack] Electron binary not found at ${electronBinary}. ` +
+ `Expected electron-builder to have packed the app before afterPack ran.`,
+ );
+ }
+
+ await flipElectronFuses(electronBinary, electronPlatformName);
if (electronPlatformName !== 'darwin') {
console.log(
diff --git a/packages/desktop/scripts/afterSign.mjs b/packages/desktop/scripts/afterSign.mjs
index fb3a4297b..549e821a1 100644
--- a/packages/desktop/scripts/afterSign.mjs
+++ b/packages/desktop/scripts/afterSign.mjs
@@ -4,6 +4,7 @@ import { existsSync } from 'node:fs';
import { join } from 'node:path';
import { FuseV1Options, getCurrentFuseWire } from '@electron/fuses';
import { notarize } from '@electron/notarize';
+import { createFuseFailure } from './packaging-diagnostics.mjs';
import { resolveElectronBinary } from './resolve-electron-binary.mjs';
import { expectedFuseState, fuseStateName, targetFuses } from './target-fuses.mjs';
@@ -114,8 +115,8 @@ async function verifyFuses(electronBinary, expected) {
try {
wire = await getCurrentFuseWire(electronBinary);
} catch (err) {
- throw new Error(
- `[afterSign] getCurrentFuseWire failed on ${electronBinary}: ${
+ throw createFuseFailure(
+ `[afterSign] fuse verification read failed on ${electronBinary}: ${
err instanceof Error ? err.message : String(err)
}`,
{ cause: err },
@@ -134,8 +135,8 @@ async function verifyFuses(electronBinary, expected) {
}
}
if (mismatches.length > 0) {
- throw new Error(
- `[afterSign] Fuse verification failed (D17 paranoid check):\n ${mismatches.join('\n ')}`,
+ throw createFuseFailure(
+ `[afterSign] fuse verification failed (D17 paranoid check):\n ${mismatches.join('\n ')}`,
);
}
console.log('[afterSign] fuse verification passed β€” all 6 fuses match targetFuses');
diff --git a/packages/desktop/scripts/packaging-diagnostics.mjs b/packages/desktop/scripts/packaging-diagnostics.mjs
new file mode 100644
index 000000000..9a9a04fbd
--- /dev/null
+++ b/packages/desktop/scripts/packaging-diagnostics.mjs
@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+export const FUSE_FAILURE_MARKER = 'OK_PACKAGING_FUSE_FAILURE';
+
+export function createFuseFailure(message, options) {
+ return new Error(`${message} [${FUSE_FAILURE_MARKER}]`, options);
+}