Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): harden release packaging retries (#4502)
Andrew Mikofalvy997d886be0d2
diff --git a/.github/scripts/build-smoke-alert-payload.mjs b/.github/scripts/build-smoke-alert-payload.mjsindex e8f6221cc..006745c7b 100644--- a/.github/scripts/build-smoke-alert-payload.mjs+++ b/.github/scripts/build-smoke-alert-payload.mjs@@ -64,6 +64,17 @@ function summaryLine(tag, verdict, blockedBy) {return `π¨ ${ALERT_HEADLINE}: ${tag} ${subject} β nothing shipped`;}+function recoveryGuidance() {+ return [+ "*Recovery guidance:* Inspect the failed packaging job's final `outcome=`.",+ 'With no newer stable release, `transient-exhausted`, `deadline`, or `signal` can re-fire the same tag; investigate a recurring deadline.',+ '`unknown-exhausted` requires manual log review. `terminal` requires fixing the cause in `reason=`.',+ '`reason=rule:download-integrity` is the terminal exception: re-fire once on a fresh runner, then stop and investigate upstream if the same artifact fails again.',+ '`cleanup-failure` reports its runner/tooling cause in `cleanup=`; `spawn-failure`, `attempt-timeout`, and `child-signal` are also runner/tooling failures. Confirm the runner/tooling failure is absent before re-firing.',+ 'Never re-fire a tag once a newer stable release has shipped; cut a new release from `main` instead.',+ ].join('\n');+}+function bodyLines({ tag, verdict, reason, runUrl, repo, blockedBy }) {const { detail, smokePassed } = describeBlock({ verdict, blockedBy });const other = String(blockedBy ?? '').trim();@@ -76,7 +87,8 @@ function bodyLines({ tag, verdict, reason, runUrl, repo, blockedBy }) {: `*Why:* ${other} failed (${reason}).`: `*Why:* ${reason}`,'*State:* the GitHub Release is still a DRAFT and npm `latest` has NOT moved.',- `*Recovery (fix the cause above first β re-firing alone repairs nothing):*\n\`${recoveryCommand(tag, repo)}\``,+ recoveryGuidance(),+ `*Recovery command:*\n\`${recoveryCommand(tag, repo)}\``,`*Run:* ${runUrl}`,];}diff --git a/.github/scripts/build-smoke-alert-payload.test.mjs b/.github/scripts/build-smoke-alert-payload.test.mjsindex aeb2f18d7..e03d7e2d9 100644--- a/.github/scripts/build-smoke-alert-payload.test.mjs+++ b/.github/scripts/build-smoke-alert-payload.test.mjs@@ -9,12 +9,14 @@ import {parseArgs,recoveryCommand,} from './build-smoke-alert-payload.mjs';+import { STOP_OUTCOMES } from './retry-transient.mjs';const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');const desktopRelease = readFileSync(join(REPO_ROOT, '.github', 'workflows', 'desktop-release.yml'),'utf8',);+const SHELL_DOLLAR = '$';const alertStep = (source = desktopRelease) => {const start = source.indexOf('- name: Alert on a blocked release');@@ -62,10 +64,23 @@ describe('alert content', () => {expect(recoveryCommand('v9.9.9', 'acme/fork')).toContain('repos/acme/fork/dispatches');});- test('recovery points at the cause instead of promising a repair-free retry', () => {+ test('recovery maps every packaging outcome to an operator action', () => {const body = buildSlackPayload(base).blocks[1].text.text;expect(body).toContain('*Recovery');- expect(body).toContain('fix the cause');+ expect(body).toContain("Inspect the failed packaging job's final `outcome=`");+ for (const outcome of STOP_OUTCOMES) {+ expect(body).toContain(`\`${outcome}\``);+ }+ expect(body).toContain('no newer stable release');+ expect(body).toContain('`transient-exhausted`, `deadline`, or `signal` can re-fire');+ expect(body).toContain('`unknown-exhausted` requires manual log review');+ expect(body).toContain('`terminal` requires fixing the cause in `reason=`');+ expect(body).toContain('`reason=rule:download-integrity` is the terminal exception');+ expect(body).toContain('stop and investigate upstream');+ expect(body).toContain('`cleanup-failure` reports its runner/tooling cause in `cleanup=`');+ expect(body).toContain('Confirm the runner/tooling failure is absent before re-firing');+ expect(body).toContain('Never re-fire a tag once a newer stable release has shipped');+ expect(body).not.toContain('re-firing alone repairs nothing');expect(body).not.toContain('no manual repair needed');});});@@ -252,16 +267,18 @@ describe('workflow wiring', () => {test('an unset webhook is a notice-level skip, and a failed POST is a warning', () => {const step = alertStep();- expect(step).toContain('::notice::${label} webhook not set');+ expect(step).toContain(`::notice::${SHELL_DOLLAR}{label} webhook not set`);expect(step).toMatch(/if \[\[ -z "\$webhook" \]\]; then[\s\S]{0,200}?return 0/);- expect(step).toContain('::warning::${label} smoke alert failed to POST');- expect(step).toContain('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');+ expect(step).toContain(`::warning::${SHELL_DOLLAR}{label} smoke alert failed to POST`);+ expect(step).toContain(+ `post "${SHELL_DOLLAR}{SLACK_RELEASES_WEBHOOK_URL:-${SHELL_DOLLAR}{SLACK_WEBHOOK_URL:-}}" Slack`,+ );});test('a passing smoke verdict is not coerced into an error', () => {const step = alertStep();expect(step).not.toMatch(/if \[\[ "\$VERDICT" != "fail" \]\]/);- expect(step).toContain('VERDICT="${SMOKE_VERDICT:-error}"');+ expect(step).toContain(`VERDICT="${SHELL_DOLLAR}{SMOKE_VERDICT:-error}"`);expect(step).toMatch(/skipped.*\]\]; then\s*\n\s*VERDICT="error"/);});@@ -302,7 +319,9 @@ describe('workflow wiring', () => {test('the annotation is emitted in addition to the page, not instead of it', () => {const step = alertStep();- const slackAt = step.indexOf('post "${SLACK_RELEASES_WEBHOOK_URL:-${SLACK_WEBHOOK_URL:-}}" Slack');+ const slackAt = step.indexOf(+ `post "${SHELL_DOLLAR}{SLACK_RELEASES_WEBHOOK_URL:-${SHELL_DOLLAR}{SLACK_WEBHOOK_URL:-}}" Slack`,+ );const annotationAt = step.indexOf('::error::RELEASE BLOCKED');expect(slackAt).toBeGreaterThan(-1);expect(annotationAt).toBeGreaterThan(slackAt);diff --git a/.github/scripts/retry-transient.mjs b/.github/scripts/retry-transient.mjsindex 1332126af..b8cb7a003 100644--- a/.github/scripts/retry-transient.mjs+++ b/.github/scripts/retry-transient.mjs@@ -1,138 +1,947 @@#!/usr/bin/env nodeimport { spawn } from 'node:child_process';+import { StringDecoder } from 'node:string_decoder';import { pathToFileURL } from 'node:url';export const DEFAULT_MAX_ATTEMPTS = 3;+export const STOP_OUTCOMES = Object.freeze([+ 'terminal',+ 'transient-exhausted',+ 'unknown-exhausted',+ 'deadline',+ 'signal',+ 'attempt-timeout',+ 'child-signal',+ 'spawn-failure',+ 'cleanup-failure',+]);-export const TRANSIENT_SIGNATURES = [- 'A timestamp was expected but was not found',- 'The timestamp service is not available',- 'HTTPError\\(statusCode: nil',- 'The request timed out',- 'NSURLErrorDomain Code=-100[13459]',- 'kCFErrorDomainCFNetwork',- 'The network connection was lost',- 'Could not connect to the server',+const MAX_CAPTURE_BYTES = 16_384;+const MIN_PROJECTED_ATTEMPT_MS = 1_000;+const DEFAULT_ATTEMPT_TIMEOUT_MS = 15 * 60 * 1000;+const DEFAULT_CLEANUP_GRACE_MS = 5_000;+const DEFAULT_CLEANUP_RESERVE_MS = 15_000;+const DEFAULT_POLL_INTERVAL_MS = 25;+const STOP_OUTCOME_SET = new Set(STOP_OUTCOMES);+const TRANSIENT_HTTP_STATUSES = new Set([408, 429, 500, 502, 503, 504, 521, 522, 524]);+const TERMINAL_HTTP_STATUSES = new Set([400, 401, 404, 413, 422]);+const NETWORK_CODES = [+ 'EAI_AGAIN',+ 'ENOTFOUND',+ 'ECONNREFUSED',+ 'ENETUNREACH',+ 'EHOSTUNREACH',+ 'ECONNABORTED','ECONNRESET','ETIMEDOUT','EPIPE',- 'socket hang up',- 'Client network socket disconnected',- 'unexpected EOF',- '502 Bad Gateway',- '503 Service Unavailable',- '504 Gateway Time-?out',- 'You have exceeded a secondary rate limit',- 'abuse detection mechanism',- 'was submitted too quickly',];+// STOP: Order selects the operator-facing reason; keep root causes first and download-integrity last.+const TERMINAL_RULES = [+ ['disk-full', /\bENOSPC\b|no space left on device/i],+ [+ 'compiler',+ /(?:\berror TS\d+:|\b(?:compilation|electron-vite build|vite build|Rolldown build) failed\b)/i,+ ],+ [+ 'configuration',+ /[\r\n]\s*(?:β¨―\s*|(?:Error|FATAL):?\s*)?(?:invalid configuration object|configuration (?:is invalid|validation failed)|cannot find configuration|unknown (?:configuration )?option|ERR_PNPM_OUTDATED_LOCKFILE)\b/i,+ ],+ [+ 'credentials',+ /\b(?:authentication failed|unauthorized credentials?|invalid credentials?|credentials? (?:are )?(?:missing|expired)|CSC_LINK is not set)\b/i,+ ],+ [+ 'certificate',+ /\b(?:certificate (?:has )?(?:expired|not trusted|revoked)|CSSMERR_TP_CERT_|no (?:valid )?signing identity|specified item could not be found in the keychain|SecKeychainUnlock)\b/i,+ ],+ [+ 'notarization-invalid',+ /(?:notari[sz]ation[^\r\n]*\bstatus["']?\s*[:=]\s*["']?Invalid\b|\bstatus["']?\s*[:=]\s*["']?Invalid\b[^\r\n]*notari[sz]|\bnotarytool\b[\s\S]{0,200}?\bstatus["']?\s*[:=]\s*["']?Invalid\b)/i,+ ],+ [+ 'entitlement',+ /(?:\bentitlements?\b[^\r\n]*(?:not permitted|mismatch|rejected|failed)|(?:invalid|rejected)[^\r\n]*\bentitlements?\b)/i,+ ],+ [+ 'fuse',+ /\bOK_PACKAGING_FUSE_FAILURE\b|\[afterSign\]\s+getCurrentFuseWire\s+failed\b|(?:\[after(?:Pack|Sign)\]\s+|\belectron\s+)fuse(?:s)?\b[^\r\n]*(?:\bmismatch(?:es|ed)?\b|\binvalid\b|\bfailed\b)/i,+ ],+ ['tls-trust', /\bNSURLErrorDomain\b[^\r\n]*\bCode=(?:-1201|-1202|-1203|-1204|-1205|-1206)\b/i],+ [+ 'integrity',+ /\b(?:integrity check failed|code signature invalid|signature verification failed)\b/i,+ ],+ [+ 'missing-executable',+ /(?:\bspawn\s+\S+\s+ENOENT\b|[\r\n]\s*Error:\s*Cannot find module\b|\bERR_PNPM_RECURSIVE_EXEC_FIRST_FAIL\b[^\r\n]*\bCommand\b[^\r\n]*\bnot found\b|[\r\n]\s*\S{0,256}(?:ba)?sh(?:\.exe)?:[^\r\n]{0,512}:\s*command not found\b)/i,+ ],+ ['download-integrity', /\b(?:checksum|hash) mismatch,\s*expected\b/i],+];+const TERMINAL_RULE_IDS = Object.freeze(TERMINAL_RULES.map(([id]) => id));+const TRANSIENT_RULES = [+ ['timestamp-missing', /A timestamp was expected but was not found/i],+ ['timestamp-service', /The timestamp service is not available/i],+ ['http-status-nil', /HTTPError\s*\(\s*statusCode\s*:\s*nil/i],+ ['request-timeout', /\bThe request timed out\b/i],+ ['network-lost', /\bThe network connection was lost\b/i],+ ['server-connect', /\bCould not connect to the server\b/i],+ ['socket-hangup', /\bsocket hang up\b/i],+ ['tls-disconnect', /\bClient network socket disconnected\b/i],+ ['unexpected-eof', /\bunexpected EOF\b/i],+ ['secondary-rate-limit', /\b(?:secondary rate limit|rate limit exceeded)\b/i],+ ['rate-limit-header', /\bX-RateLimit-Remaining\s*:\s*0\b/i],+ ['abuse-limit', /\babuse detection mechanism\b/i],+ ['submitted-too-fast', /\bwas submitted too quickly\b/i],+];++function appendUtf8Tail(current, chunk, limit) {+ const combined = current.length === 0 ? chunk : Buffer.concat([current, chunk]);+ if (combined.length <= limit) return combined;+ let start = combined.length - limit;+ while (start < combined.length && (combined[start] & 0xc0) === 0x80) start += 1;+ return combined.subarray(start);+}++function collectStructuredEvidence(evidence, text) {+ const statusPattern =+ /(?:\bHTTP(?:Error)?(?:\/[0-9.]+)?(?:\s+status)?\s*[:=]?\s*|\bresponse\s+(?:status|code)\s*[:=]?\s*|\bstatusCode["']?\s*[:=]\s*)(\d{3})\b/gi;+ for (const match of text.matchAll(statusPattern)) {+ evidence.httpStatuses.add(Number(match[1]));+ }+ for (const match of text.matchAll(+ /[\r\n]\s*(?:β¨―\s*|Error:\s*)?(408|429|500|502|503|504|521|522|524)\s+(?:Request Timeout|Too Many Requests|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Time-?out|Web Server Is Down|Connection Timed Out|A Timeout Occurred)\b/gi,+ )) {+ evidence.httpStatuses.add(Number(match[1]));+ }+ const networkPattern = new RegExp(`\\b(${NETWORK_CODES.join('|')})\\b`, 'gi');+ for (const match of text.matchAll(networkPattern)) {+ evidence.errorCodes.add(match[1].toUpperCase());+ }+ for (const match of text.matchAll(/\b(UND_ERR_(?:[A-Z_]*TIMEOUT|SOCKET))\b/gi)) {+ evidence.errorCodes.add(match[1].toUpperCase());+ }+ for (const match of text.matchAll(/\bcurl:\s*\((28|56)\)/gi)) {+ evidence.errorCodes.add(`CURL_${match[1]}`);+ }+ for (const match of text.matchAll(+ /\bNSURLErrorDomain\b[^\r\n]*\bCode=(-1001|-1003|-1004|-1005|-1006|-1008|-1009|-1011)\b/gi,+ )) {+ evidence.errorCodes.add(`NSURLError_${match[1]}`);+ }+}++function collectRetryAfterEvidence(evidence, text) {+ if (evidence.retryAfterValid) return;+ const match = /^\s*(?:<\s*)?Retry-After\s*:\s*(.*?)\s*$/i.exec(text);+ if (!match) return;+ const raw = match[1].replace(/^["']|["']$/g, '');+ const retryAt = /^\d+$/.test(raw) ? evidence.nowFn() + Number(raw) * 1000 : Date.parse(raw);+ if (!Number.isFinite(retryAt)) return;+ evidence.retryAfterValid = true;+ evidence.retryAfterEpochMs = retryAt;+}++function collectDecodedEvidence(evidence, decoded, final) {+ const joined = evidence.scanCarry + decoded;+ const scanText = evidence.scanStarted ? joined : `\n${joined}`;+ collectStructuredEvidence(evidence, scanText);+ collectPhraseEvidence(evidence, scanText);+ evidence.scanCarry = joined.slice(-2048);+ evidence.scanStarted ||= joined.length > 0;++ const retryText = evidence.retryAfterCarry + decoded;+ const retryLines = retryText.split(/\r\n|\r|\n/);+ evidence.retryAfterCarry = final ? '' : (retryLines.pop() ?? '').slice(-2048);+ for (const line of retryLines) collectRetryAfterEvidence(evidence, line);+}++function collectPhraseEvidence(evidence, text) {+ for (const [id, pattern] of TERMINAL_RULES) {+ if (pattern.test(text)) evidence.terminalRules.add(id);+ }+ for (const [id, pattern] of TRANSIENT_RULES) {+ if (pattern.test(text)) evidence.transientRules.add(id);+ }+}++export class FailureEvidence {+ constructor({ maxTailBytes = MAX_CAPTURE_BYTES, nowFn = Date.now } = {}) {+ this.httpStatuses = new Set();+ this.errorCodes = new Set();+ this.terminalRules = new Set();+ this.transientRules = new Set();+ this.retryAfterEpochMs = undefined;+ this.retryAfterValid = false;+ this.tailBytes = Buffer.alloc(0);+ this.tail = '';+ this.maxTailBytes = maxTailBytes;+ this.nowFn = nowFn;+ this.decoder = new StringDecoder('utf8');+ this.scanCarry = '';+ this.scanStarted = false;+ this.retryAfterCarry = '';+ }++ ingest(value) {+ const chunk = Buffer.isBuffer(value) ? value : Buffer.from(String(value));+ this.tailBytes = appendUtf8Tail(this.tailBytes, chunk, this.maxTailBytes);+ collectDecodedEvidence(this, this.decoder.write(chunk), false);+ }++ finish() {+ collectDecodedEvidence(this, this.decoder.end(), true);+ this.tail = this.tailBytes.toString('utf8');+ return this;+ }+}++class AttemptResult {+ constructor({+ code = null,+ closeSignal = null,+ cancellationSignal = null,+ cancelled = false,+ timedOut = false,+ deadlineExpired = false,+ spawnError = null,+ cleanup = { ok: true, reason: 'clean' },+ evidence = new FailureEvidence(),+ } = {}) {+ this.code = code;+ this.closeSignal = closeSignal;+ this.cancellationSignal = cancellationSignal;+ this.cancelled = cancelled;+ this.timedOut = timedOut;+ this.deadlineExpired = deadlineExpired;+ this.spawnError = spawnError;+ this.cleanup = cleanup;+ this.evidence = evidence;+ }+}++class RetryState {+ constructor() {+ this.attemptsStarted = 0;+ this.unknownRetryUsed = false;+ this.lastFailureClassification = undefined;+ this.lastAttemptDurationMs = undefined;+ }+}++export function classifyEvidence(evidence) {+ if (evidence.terminalRules.size > 0) return 'terminal';+ for (const status of evidence.httpStatuses) {+ if (TERMINAL_HTTP_STATUSES.has(status)) return 'terminal';+ }+ const rateLimited =+ evidence.transientRules.has('secondary-rate-limit') ||+ evidence.transientRules.has('rate-limit-header') ||+ evidence.transientRules.has('abuse-limit') ||+ evidence.transientRules.has('submitted-too-fast');+ if (evidence.httpStatuses.has(403) && !rateLimited) return 'terminal';+ if (evidence.errorCodes.size > 0 || evidence.transientRules.size > 0) return 'transient';+ for (const status of evidence.httpStatuses) {+ if (TRANSIENT_HTTP_STATUSES.has(status)) return 'transient';+ }+ if (evidence.httpStatuses.has(403) && rateLimited) return 'transient';+ return 'unknown';+}++function firstSorted(values) {+ return [...values].sort()[0];+}++function firstTerminalRule(values) {+ return TERMINAL_RULE_IDS.find((id) => values.has(id));+}++function matchedReason(evidence, classification) {+ const statuses = [...evidence.httpStatuses].sort((left, right) => left - right);+ if (classification === 'terminal') {+ const terminalRule = firstTerminalRule(evidence.terminalRules);+ if (terminalRule) return `rule:${terminalRule}`;+ const status = statuses.find(+ (candidate) => TERMINAL_HTTP_STATUSES.has(candidate) || candidate === 403,+ );+ return `http:${status}`;+ }+ if (classification === 'transient') {+ const errorCode = firstSorted(evidence.errorCodes);+ if (errorCode) return `code:${errorCode}`;+ const transientRule = firstSorted(evidence.transientRules);+ if (transientRule) return `rule:${transientRule}`;+ const status = statuses.find((candidate) => TRANSIENT_HTTP_STATUSES.has(candidate));+ return `http:${status}`;+ }+ return 'diagnostic:unknown';+}++function redactDiagnosticTail(tail) {+ const withoutPem = tail.replace(+ /(?:-----BEGIN [^-\r\n]+-----[\s\S]*?(?:-----END [^-\r\n]+-----|$)|(?:[A-Z0-9+/=]{20,}\r?\n)+-----END [^-\r\n]+-----)/gi,+ '[REDACTED PEM]',+ );+ return withoutPem.replace(+ /^(\s*["']?[A-Z0-9_.-]*(?:API[-_]?KEY|AUTHORIZATION|CREDENTIAL|PASSWORD|SECRET|TOKEN)[A-Z0-9_.-]*["']?\s*[:=]\s*)[^\r\n]*/gim,+ '$1[REDACTED]',+ );+}++function emitDiagnostic(log, label, evidence) {+ log(`::group::${label} bounded failure diagnostic`);+ const safe = redactDiagnosticTail(evidence.tail || '(no child diagnostic captured)');+ for (const line of safe.split(/\r\n|\r|\n/)) log(`| ${line.replaceAll('##[', '# #[')}`);+ log('::endgroup::');+}++function projectedAttemptMs(durationMs, attemptTimeoutMs) {+ return Math.min(+ attemptTimeoutMs,+ Math.max(MIN_PROJECTED_ATTEMPT_MS, Math.ceil(durationMs * 1.25)),+ );+}++function logDecision({+ log,+ level,+ label,+ action,+ reason,+ attempt,+ maxAttempts,+ result,+ signal,+ matched,+ detail,+ outcome,+ classification,+}) {+ const annotation = level === 'warning' ? '::warning::' : '::error::';+ const code = result?.code ?? 'none';+ const signalFact = signal ?? result?.cancellationSignal ?? result?.closeSignal ?? 'none';+ const matchedFact = matched ? ` matched=${matched}` : '';+ const detailFact = detail ? ` ${detail}` : '';+ const outcomeFact = outcome ? ` outcome=${outcome}` : '';+ const classificationFact = classification ? ` classification=${classification}` : '';+ const cleanupFact = result?.cleanup?.ok === false ? ` cleanup=${result.cleanup.reason}` : '';+ log(+ `${annotation}${label} decision=${action} reason=${reason}${outcomeFact}${classificationFact} attempt=${attempt}/${maxAttempts} code=${code} signal=${signalFact}${cleanupFact}${matchedFact}${detailFact}`,+ );+ if (action === 'stop' && !STOP_OUTCOME_SET.has(outcome)) {+ if (result) emitDiagnostic(log, label, result.evidence);+ throw new Error(`${label}: stop decision has unregistered outcome ${outcome ?? 'none'}`);+ }+}-const TRANSIENT_RE = new RegExp(TRANSIENT_SIGNATURES.join('|'), 'i');+function positiveJitterMs(minimumMs, randomFn) {+ const ceiling = Math.max(1, Math.min(5_000, Math.ceil(minimumMs * 0.1)));+ return Math.min(ceiling, Math.max(1, Math.floor(randomFn() * ceiling) + 1));+}++export function computeRetryDelayMs(+ retryNumber,+ evidence,+ { randomFn = Math.random, nowMs = Date.now() } = {},+) {+ const baseMs = retryNumber === 1 ? 30_000 : 60_000;+ const minimumMs = Math.max(baseMs, retryAfterDelayMs(evidence, nowMs) ?? 0);+ return minimumMs + positiveJitterMs(minimumMs, randomFn);+}-export function isTransient(output) {- return TRANSIENT_RE.test(String(output ?? ''));+function retryAfterDelayMs(evidence, nowMs) {+ if (!evidence.retryAfterValid) return undefined;+ return Math.max(0, (evidence.retryAfterEpochMs ?? nowMs) - nowMs);}-export function backoffSeconds(attempt) {- return attempt * 30;+function delay(ms) {+ return new Promise((resolve) => setTimeout(resolve, ms));}-const MAX_CAPTURE_BYTES = 2_000_000;+async function waitUntil(predicate, timeoutMs, intervalMs, sleepFn) {+ const end = Date.now() + timeoutMs;+ while (!predicate()) {+ if (Date.now() >= end) return false;+ await sleepFn(Math.min(intervalMs, Math.max(1, end - Date.now())));+ }+ return true;+}++function processGroupExists(pid, killFn) {+ try {+ killFn(-pid, 0);+ return true;+ } catch (error) {+ return error?.code !== 'ESRCH';+ }+}-function appendCapped(buffer, chunk) {- const next = buffer + chunk;- return next.length > MAX_CAPTURE_BYTES ? next.slice(next.length - MAX_CAPTURE_BYTES) : next;+function sendGroupSignal(pid, signal, killFn) {+ try {+ killFn(-pid, signal);+ return true;+ } catch (error) {+ return error?.code === 'ESRCH';+ }}-function runOnce(command, { shell = false, spawnFn = spawn } = {}) {+function defaultTaskkill(args) {return new Promise((resolve) => {- const child = shell- ? spawnFn('bash', ['-c', command[0]], { stdio: ['inherit', 'pipe', 'pipe'] })- : spawnFn(command[0], command.slice(1), { stdio: ['inherit', 'pipe', 'pipe'] });-- let captured = '';- child.stdout?.on('data', (d) => {- const text = d.toString();- captured = appendCapped(captured, text);- process.stdout.write(text);+ const child = spawn('taskkill', args, { stdio: 'ignore', windowsHide: true });+ let settled = false;+ child.once('error', () => {+ if (!settled) resolve({ ok: false });+ settled = true;});- child.stderr?.on('data', (d) => {- const text = d.toString();- captured = appendCapped(captured, text);- process.stderr.write(text);+ child.once('close', (code) => {+ if (!settled) resolve({ ok: code === 0 });+ settled = true;});+ });+}- child.on('error', (err) => {- const text = `${err.message}\n`;- captured = appendCapped(captured, text);- process.stderr.write(text);- resolve({ code: 1, output: captured });+export function createOwnedTreeController({+ platform = process.platform,+ killFn = process.kill.bind(process),+ taskkillFn = defaultTaskkill,+ sleepFn = delay,+ pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,+} = {}) {+ if (platform === 'win32') {+ return {+ async cleanup(pid, { graceMs, waitForClose }) {+ if (await waitForClose(0)) return { ok: true, reason: 'clean' };+ const graceful = await taskkillFn(['/PID', String(pid), '/T']);+ if (graceful.ok && (await waitForClose(graceMs))) {+ return { ok: true, reason: 'clean' };+ }+ const forced = await taskkillFn(['/PID', String(pid), '/T', '/F']);+ if (!forced.ok) return { ok: false, reason: 'taskkill-failure' };+ if (!(await waitForClose(graceMs))) {+ return { ok: false, reason: 'close-not-observed' };+ }+ return { ok: true, reason: 'clean' };+ },+ };+ }+ return {+ async cleanup(pid, { graceMs, cleanupReserveMs, waitForClose }) {+ if (!processGroupExists(pid, killFn)) {+ return (await waitForClose(graceMs))+ ? { ok: true, reason: 'clean' }+ : { ok: false, reason: 'close-not-observed' };+ }+ if (!sendGroupSignal(pid, 'SIGTERM', killFn)) {+ return { ok: false, reason: 'signal-send-failure' };+ }+ const goneAfterTerm = await waitUntil(+ () => !processGroupExists(pid, killFn),+ graceMs,+ pollIntervalMs,+ sleepFn,+ );+ if (!goneAfterTerm) {+ if (!sendGroupSignal(pid, 'SIGKILL', killFn)) {+ return { ok: false, reason: 'signal-send-failure' };+ }+ const goneAfterKill = await waitUntil(+ () => !processGroupExists(pid, killFn),+ cleanupReserveMs,+ pollIntervalMs,+ sleepFn,+ );+ if (!goneAfterKill) return { ok: false, reason: 'tree-survived-kill' };+ }+ return (await waitForClose(cleanupReserveMs))+ ? { ok: true, reason: 'clean' }+ : { ok: false, reason: 'close-not-observed' };+ },+ };+}++function waitForPromise(promise, timeoutMs) {+ return new Promise((resolve) => {+ const timeout = setTimeout(() => resolve(false), Math.max(0, timeoutMs));+ promise.then(() => {+ clearTimeout(timeout);+ resolve(true);+ });+ });+}++function commandInvocation(command, shell, platform) {+ if (!shell) return { executable: command[0], args: command.slice(1) };+ if (platform === 'win32') {+ return {+ executable: 'bash',+ args: ['-c', command[0]],+ };+ }+ return { executable: '/bin/bash', args: ['-c', command[0]] };+}++async function runAttempt({+ command,+ shell,+ spawnFn,+ platform,+ cancellationSignal,+ deadlineEpochMs,+ attemptTimeoutMs,+ cleanupReserveMs,+ cleanupGraceMs,+ treeController,+ nowFn,+}) {+ const evidence = new FailureEvidence({ nowFn });+ const invocation = commandInvocation(command, shell, platform);+ let child;+ try {+ child = spawnFn(invocation.executable, invocation.args, {+ stdio: ['inherit', 'pipe', 'pipe'],+ detached: platform !== 'win32',+ windowsHide: platform === 'win32',+ });+ } catch (error) {+ evidence.ingest(Buffer.from(`${error.message}\n`));+ evidence.finish();+ return new AttemptResult({+ spawnError: error,+ cleanup: { ok: true, reason: 'not-required' },+ evidence,+ });+ }++ child.stdout?.on('data', (chunk) => {+ evidence.ingest(chunk);+ process.stdout.write(chunk);+ });+ child.stderr?.on('data', (chunk) => {+ evidence.ingest(chunk);+ process.stderr.write(chunk);+ });++ let closeValue;+ let resolveClose;+ const closePromise = new Promise((resolve) => {+ resolveClose = resolve;+ });+ let spawnError;+ child.once('error', (error) => {+ spawnError = error;+ evidence.ingest(Buffer.from(`${error.message}\n`));+ resolveClose({ kind: 'spawn-error', code: null, signal: null });+ });+ child.once('close', (code, signal) => {+ closeValue = { kind: 'close', code, signal };+ resolveClose(closeValue);+ });++ let timeoutId;+ let deadlineId;+ let cancelListener;+ const timeoutPromise = new Promise((resolve) => {+ timeoutId = setTimeout(() => resolve({ kind: 'attempt-timeout' }), attemptTimeoutMs);+ });+ const remainingToDeadline = Math.max(0, deadlineEpochMs - cleanupReserveMs - nowFn());+ const deadlinePromise = new Promise((resolve) => {+ deadlineId = setTimeout(() => resolve({ kind: 'deadline' }), remainingToDeadline);+ });+ const cancellationPromise = new Promise((resolve) => {+ cancelListener = () =>+ resolve({ kind: 'cancelled', signal: cancellationSignal.reason || 'SIGTERM' });+ if (cancellationSignal.aborted) cancelListener();+ else cancellationSignal.addEventListener('abort', cancelListener, { once: true });+ });++ const outcome = await Promise.race([+ closePromise,+ timeoutPromise,+ deadlinePromise,+ cancellationPromise,+ ]);+ clearTimeout(timeoutId);+ clearTimeout(deadlineId);+ cancellationSignal.removeEventListener('abort', cancelListener);++ const waitForClose = (timeoutMs) => waitForPromise(closePromise, timeoutMs);+ let cleanup = { ok: true, reason: 'clean' };+ if (child.pid) {+ cleanup = await treeController.cleanup(child.pid, {+ graceMs: cleanupGraceMs,+ cleanupReserveMs,+ waitForClose,});- child.on('close', (code) => resolve({ code: code ?? 1, output: captured }));+ } else if (outcome.kind !== 'close' && outcome.kind !== 'spawn-error') {+ cleanup = { ok: false, reason: 'close-not-observed' };+ }+ if (!closeValue && outcome.kind !== 'spawn-error') {+ await waitForClose(cleanupReserveMs);+ }+ evidence.finish();++ return new AttemptResult({+ code: closeValue?.code ?? outcome.code ?? null,+ closeSignal: closeValue?.signal ?? null,+ cancellationSignal: outcome.kind === 'cancelled' ? outcome.signal : null,+ cancelled: outcome.kind === 'cancelled',+ timedOut: outcome.kind === 'attempt-timeout',+ deadlineExpired: outcome.kind === 'deadline',+ spawnError,+ cleanup,+ evidence,});}+function processStop(result, attempts) {+ if (result.cancelled) {+ return { ok: false, reason: 'signal', signal: result.cancellationSignal, attempts };+ }+ if (result.timedOut) return { ok: false, reason: 'attempt-timeout', attempts };+ if (result.deadlineExpired) return { ok: false, reason: 'deadline', attempts };+ if (result.spawnError) {+ return { ok: false, reason: 'spawn-failure', attempts, error: result.spawnError.message };+ }+ if (!result.cleanup.ok) {+ return {+ ok: false,+ reason: 'cleanup-failure',+ cleanup: result.cleanup.reason,+ attempts,+ };+ }+ if (result.closeSignal) {+ return { ok: false, reason: 'child-signal', signal: result.closeSignal, attempts };+ }+ return undefined;+}++function installSignalHandlers(signalSource, controller) {+ const handlers = new Map();+ for (const signal of ['SIGINT', 'SIGTERM']) {+ const handler = () => {+ if (!controller.signal.aborted) controller.abort(signal);+ };+ handlers.set(signal, handler);+ signalSource.on(signal, handler);+ }+ return () => {+ for (const [signal, handler] of handlers) signalSource.off(signal, handler);+ };+}++async function waitForBackoff(ms, cancellationSignal, sleepFn) {+ if (cancellationSignal.aborted) return false;+ let abortListener;+ const aborted = new Promise((resolve) => {+ abortListener = () => resolve(false);+ cancellationSignal.addEventListener('abort', abortListener, { once: true });+ });+ const slept = Promise.resolve(sleepFn(ms, cancellationSignal)).then(+ () => !cancellationSignal.aborted,+ );+ const completed = await Promise.race([slept, aborted]);+ cancellationSignal.removeEventListener('abort', abortListener);+ return completed;+}+export async function runWithRetry({command,label = 'command',shell = false,maxAttempts = DEFAULT_MAX_ATTEMPTS,+ deadlineEpochMs = Date.now() + 60 * 60 * 1000,+ attemptTimeoutMs = DEFAULT_ATTEMPT_TIMEOUT_MS,+ cleanupReserveMs = DEFAULT_CLEANUP_RESERVE_MS,+ cleanupGraceMs = DEFAULT_CLEANUP_GRACE_MS,+ pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,spawnFn = spawn,- sleepFn = (s) => new Promise((r) => setTimeout(r, s * 1000)),+ sleepFn = (ms, signal) =>+ new Promise((resolve) => {+ const timer = setTimeout(resolve, ms);+ signal?.addEventListener(+ 'abort',+ () => {+ clearTimeout(timer);+ resolve();+ },+ { once: true },+ );+ }),+ randomFn = Math.random,+ nowFn = Date.now,log = console.log,+ retryWarning = '',+ signalSource = process,+ platform = process.platform,+ treeController,+ attemptRunner = runAttempt,} = {}) {- for (let attempt = 1; ; attempt += 1) {- log(`::group::${label} attempt ${attempt}/${maxAttempts}`);- const { code, output } = await runOnce(command, { shell, spawnFn });- log('::endgroup::');-- if (code === 0) {- log(`${label} succeeded on attempt ${attempt}.`);- return { ok: true, attempts: attempt };- }+ if (!Number.isInteger(maxAttempts) || maxAttempts < 1 || maxAttempts > DEFAULT_MAX_ATTEMPTS) {+ throw new Error(`maxAttempts must be an integer from 1 to ${DEFAULT_MAX_ATTEMPTS}`);+ }+ const state = new RetryState();+ const cancellation = new AbortController();+ const removeSignalHandlers = installSignalHandlers(signalSource, cancellation);+ const ownedTree =+ treeController ??+ createOwnedTreeController({+ platform,+ pollIntervalMs,+ });+ const recordDecision = (level, action, reason, result, facts = {}) =>+ logDecision({+ log,+ level,+ label,+ action,+ reason,+ attempt: state.attemptsStarted,+ maxAttempts,+ result,+ ...facts,+ });+ const finishFailure = (failure, result) => {+ if (result) emitDiagnostic(log, label, result.evidence);+ return failure;+ };- if (!isTransient(output)) {- log(- `::error::${label} failed on attempt ${attempt} with a NON-transient error (no known infra-flake signature matched). Not retrying β a real signing/notarize/cert/entitlements/build failure must fail loud, not be masked. See the log above.`,- );- return { ok: false, reason: 'non-transient', attempts: attempt, code };- }+ try {+ while (state.attemptsStarted < maxAttempts) {+ if (cancellation.signal.aborted) {+ recordDecision('error', 'stop', 'control:signal', undefined, {+ outcome: 'signal',+ signal: cancellation.signal.reason,+ });+ return {+ ok: false,+ reason: 'signal',+ signal: cancellation.signal.reason,+ attempts: state.attemptsStarted,+ };+ }+ const admissionMs = state.lastAttemptDurationMs+ ? projectedAttemptMs(state.lastAttemptDurationMs, attemptTimeoutMs)+ : MIN_PROJECTED_ATTEMPT_MS;+ if (nowFn() + admissionMs + cleanupReserveMs > deadlineEpochMs) {+ recordDecision('error', 'stop', 'control:deadline', undefined, {+ outcome: 'deadline',+ detail: 'phase=before-attempt',+ });+ return { ok: false, reason: 'deadline', attempts: state.attemptsStarted };+ }- if (attempt >= maxAttempts) {- log(- `::error::${label} failed after ${maxAttempts} attempts; the last failure matched a transient infra signature but retries are exhausted.`,- );- return { ok: false, reason: 'exhausted', attempts: attempt, code };- }+ state.attemptsStarted += 1;+ log(`::group::${label} attempt ${state.attemptsStarted}/${maxAttempts}`);+ const attemptStartedAt = nowFn();+ const result = await attemptRunner({+ command,+ shell,+ spawnFn,+ platform,+ cancellationSignal: cancellation.signal,+ deadlineEpochMs,+ attemptTimeoutMs,+ cleanupReserveMs,+ cleanupGraceMs,+ treeController: ownedTree,+ nowFn,+ });+ state.lastAttemptDurationMs = Math.max(0, nowFn() - attemptStartedAt);+ log('::endgroup::');++ const stopped = processStop(result, state.attemptsStarted);+ if (stopped) {+ recordDecision('error', 'stop', `control:${stopped.reason}`, result, {+ outcome: stopped.reason,+ signal: stopped.signal,+ detail: stopped.reason === 'deadline' ? 'phase=mid-attempt' : undefined,+ });+ return finishFailure(stopped, result);+ }+ if (result.code === 0) {+ log(`${label} succeeded on attempt ${state.attemptsStarted}.`);+ return {+ ok: true,+ attempts: state.attemptsStarted,+ recoveredFrom: state.attemptsStarted > 1 ? state.lastFailureClassification : undefined,+ };+ }++ const classification = classifyEvidence(result.evidence);+ const reasonId = matchedReason(result.evidence, classification);+ state.lastFailureClassification = classification;+ if (classification === 'terminal') {+ recordDecision('error', 'stop', reasonId, result, { outcome: 'terminal' });+ return finishFailure(+ {+ ok: false,+ reason: 'terminal',+ attempts: state.attemptsStarted,+ code: result.code,+ },+ result,+ );+ }+ if (classification === 'unknown') {+ if (state.unknownRetryUsed) {+ recordDecision('error', 'stop', reasonId, result, {+ outcome: 'unknown-exhausted',+ detail: 'allowance=already-used',+ });+ return finishFailure(+ {+ ok: false,+ reason: 'unknown-exhausted',+ attempts: state.attemptsStarted,+ code: result.code,+ },+ result,+ );+ }+ state.unknownRetryUsed = true;+ }+ if (state.attemptsStarted >= maxAttempts) {+ const reason = classification === 'transient' ? 'transient-exhausted' : 'unknown-exhausted';+ recordDecision('error', 'stop', reasonId, result, {+ outcome: reason,+ detail: `attempt-bound=${reason}`,+ });+ return finishFailure(+ { ok: false, reason, attempts: state.attemptsStarted, code: result.code },+ result,+ );+ }- const backoff = backoffSeconds(attempt);- log(- `::warning::${label} attempt ${attempt} failed with a transient infra signature; retrying in ${backoff}s (bounded transient-only retry).`,+ const decisionNow = nowFn();+ const delayMs = computeRetryDelayMs(state.attemptsStarted, result.evidence, {+ randomFn,+ nowMs: decisionNow,+ });+ const nextAttemptMs = projectedAttemptMs(state.lastAttemptDurationMs, attemptTimeoutMs);+ if (decisionNow + delayMs + nextAttemptMs + cleanupReserveMs > deadlineEpochMs) {+ const retryAfterMs = retryAfterDelayMs(result.evidence, decisionNow);+ recordDecision('error', 'stop', 'control:deadline', result, {+ outcome: 'deadline',+ matched: reasonId,+ detail: [+ 'phase=before-backoff',+ retryAfterMs === undefined ? undefined : `retry-after-ms=${retryAfterMs}`,+ `projected-attempt-ms=${nextAttemptMs}`,+ ]+ .filter(Boolean)+ .join(' '),+ });+ return finishFailure(+ { ok: false, reason: 'deadline', attempts: state.attemptsStarted },+ result,+ );+ }+ if (retryWarning) log(`::warning::${retryWarning}`);+ recordDecision('warning', 'retry', reasonId, result, {+ classification,+ detail:+ classification === 'unknown'+ ? `UNKNOWN_CLASSIFICATION_RETRY allowance=invocation-wide-single-use delay-ms=${delayMs}`+ : `delay-ms=${delayMs}`,+ });+ if (!(await waitForBackoff(delayMs, cancellation.signal, sleepFn))) {+ recordDecision('error', 'stop', 'control:signal', result, {+ outcome: 'signal',+ signal: cancellation.signal.reason,+ matched: reasonId,+ detail: 'phase=backoff',+ });+ return finishFailure(+ {+ ok: false,+ reason: 'signal',+ signal: cancellation.signal.reason,+ attempts: state.attemptsStarted,+ },+ result,+ );+ }+ }+ throw new Error(+ `${label}: retry state exhausted unexpectedly after ${state.attemptsStarted}/${maxAttempts} attempts`,);- await sleepFn(backoff);+ } finally {+ removeSignalHandlers();+ }+}++function parseDuration(value, flag) {+ const match = /^(\d+)(ms|s|m)$/.exec(value ?? '');+ if (!match) throw new Error(`${flag} must be a positive duration such as 30m`);+ const multiplier = match[2] === 'm' ? 60_000 : match[2] === 's' ? 1_000 : 1;+ const duration = Number(match[1]) * multiplier;+ if (!Number.isSafeInteger(duration) || duration <= 0) {+ throw new Error(`${flag} must be a positive duration`);}+ return duration;+}++function flagValue(flags, name, required = false) {+ const index = flags.indexOf(name);+ if (index === -1) {+ if (required) throw new Error(`${name} is required`);+ return undefined;+ }+ return flags[index + 1];}export function parseArgs(argv) {const rest = argv.slice(2);- const sep = rest.indexOf('--');- if (sep === -1) throw new Error('a `--` separator followed by the command is required');- const flags = rest.slice(0, sep);- const command = rest.slice(sep + 1);+ const separator = rest.indexOf('--');+ if (separator === -1) throw new Error('a `--` separator followed by the command is required');+ const flags = rest.slice(0, separator);+ const command = rest.slice(separator + 1);if (command.length === 0) throw new Error('no command given after `--`');+ const allowedFlags = new Set([+ '--label',+ '--max-attempts',+ '--deadline-epoch-ms',+ '--attempt-timeout',+ '--retry-warning',+ '--shell',+ ]);+ for (let index = 0; index < flags.length; index += 1) {+ const flag = flags[index];+ if (!allowedFlags.has(flag)) throw new Error(`unknown flag: ${flag}`);+ if (flag === '--shell') continue;+ const value = flags[index + 1];+ if (+ value === undefined ||+ value.startsWith('--') ||+ (value === '' && flag !== '--retry-warning')+ ) {+ throw new Error(`${flag} requires a value`);+ }+ index += 1;+ }const shell = flags.includes('--shell');- if (shell && command.length !== 1) {- throw new Error('--shell takes exactly one command string');+ if (shell && command.length !== 1) throw new Error('--shell takes exactly one command string');+ const maxAttempts = Number(flagValue(flags, '--max-attempts') ?? DEFAULT_MAX_ATTEMPTS);+ if (!Number.isInteger(maxAttempts) || maxAttempts < 1 || maxAttempts > DEFAULT_MAX_ATTEMPTS) {+ throw new Error(`--max-attempts must be between 1 and ${DEFAULT_MAX_ATTEMPTS}`);}-- const labelIdx = flags.indexOf('--label');- const attemptsIdx = flags.indexOf('--max-attempts');+ const deadlineEpochMs = Number(flagValue(flags, '--deadline-epoch-ms', true));+ if (!Number.isSafeInteger(deadlineEpochMs) || deadlineEpochMs <= 0) {+ throw new Error('--deadline-epoch-ms must be a positive integer');+ }+ const attemptTimeoutMs = parseDuration(+ flagValue(flags, '--attempt-timeout', true),+ '--attempt-timeout',+ );return {- label: labelIdx === -1 ? 'command' : (flags[labelIdx + 1] ?? 'command'),- maxAttempts:- attemptsIdx === -1 ? DEFAULT_MAX_ATTEMPTS : Number(flags[attemptsIdx + 1]) || DEFAULT_MAX_ATTEMPTS,+ label: flagValue(flags, '--label') ?? 'command',+ maxAttempts,+ deadlineEpochMs,+ attemptTimeoutMs,+ retryWarning: flagValue(flags, '--retry-warning') ?? '',shell,command,};@@ -141,9 +950,13 @@ export function parseArgs(argv) {if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {try {const result = await runWithRetry(parseArgs(process.argv));- process.exitCode = result.ok ? 0 : 1;- } catch (err) {- process.stderr.write(`${err.message}\n`);+ if (result.reason === 'signal' && result.signal) {+ process.kill(process.pid, result.signal);+ } else {+ process.exitCode = result.ok ? 0 : 1;+ }+ } catch (error) {+ process.stderr.write(`${error.message}\n`);process.exitCode = 1;}}diff --git a/.github/scripts/retry-transient.test.mjs b/.github/scripts/retry-transient.test.mjsindex 19adc148d..3a3ee038e 100644--- a/.github/scripts/retry-transient.test.mjs+++ b/.github/scripts/retry-transient.test.mjs@@ -1,271 +1,1454 @@-import { spawnSync } from 'node:child_process';-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { spawn, spawnSync } from 'node:child_process';+import { EventEmitter } from 'node:events';+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { dirname, join } from 'node:path';+import { PassThrough } from 'node:stream';import { fileURLToPath } from 'node:url';-import { afterAll, beforeEach, describe, expect, test, vi } from 'vitest';+import { afterAll, afterEach, beforeEach, describe, expect, test, vi } from 'vitest';import {- backoffSeconds,+ createFuseFailure,+ FUSE_FAILURE_MARKER,+} from '../../packages/desktop/scripts/packaging-diagnostics.mjs';+import {+ classifyEvidence,+ computeRetryDelayMs,+ createOwnedTreeController,DEFAULT_MAX_ATTEMPTS,- isTransient,+ FailureEvidence,parseArgs,runWithRetry,- TRANSIENT_SIGNATURES,} from './retry-transient.mjs';const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');+const SCRIPT = join(REPO_ROOT, '.github', 'scripts', 'retry-transient.mjs');+const afterPackSource = readFileSync(+ join(REPO_ROOT, 'packages', 'desktop', 'scripts', 'afterPack.mjs'),+ 'utf8',+);+const afterSignSource = readFileSync(+ join(REPO_ROOT, 'packages', 'desktop', 'scripts', 'afterSign.mjs'),+ 'utf8',+);const desktopRelease = readFileSync(join(REPO_ROOT, '.github', 'workflows', 'desktop-release.yml'),'utf8',);-+const WORKFLOW_JOBS = [+ ['build-macos', '65', '30m', 'Build + sign + notarize DMG/ZIP'],+ ['build-windows', '35', '15m', 'Package NSIS installers (x64 + arm64, signed)'],+ ['build-linux', '35', '15m', 'Package $' + '{{ matrix.targets }}'],+];+const workflowJob = (name) => {+ const start = desktopRelease.indexOf(`\n ${name}:`);+ if (start === -1) throw new Error(`missing job ${name}`);+ const rest = desktopRelease.slice(start + 1);+ const next = rest.slice(1).search(/\n {2}[a-z][a-z0-9-]*:/);+ return next === -1 ? rest : rest.slice(0, next + 1);+};+const workflowStep = (body, name) => {+ const start = body.indexOf(`- name: ${name}`);+ if (start === -1) throw new Error(`missing step ${name}`);+ const rest = body.slice(start);+ const ends = [rest.indexOf('\n - name: ', 1), rest.indexOf('\n - uses: ', 1)].filter(+ (index) => index !== -1,+ );+ return ends.length === 0 ? rest : rest.slice(0, Math.min(...ends));+};+const workflowTiming = ([jobName, , , packageName]) => {+ const body = workflowJob(jobName);+ const budget = /PACKAGING_BUDGET_MINUTES: "(\d+)"/.exec(+ workflowStep(body, 'Start packaging deadline'),+ )?.[1];+ const timeout = /--attempt-timeout "(\d+)m"/.exec(workflowStep(body, packageName))?.[1];+ if (!budget || !timeout) throw new Error(`missing workflow timing for ${jobName}`);+ return { budgetMs: Number(budget) * 60_000, attemptTimeoutMs: Number(timeout) * 60_000 };+};const scratch = mkdtempSync(join(tmpdir(), 'retry-transient-'));-afterAll(() => rmSync(scratch, { recursive: true, force: true }));+afterAll(() => rmSync(scratch, { recursive: true, force: true }));+afterEach(() => vi.restoreAllMocks());beforeEach(() => {vi.spyOn(process.stdout, 'write').mockImplementation(() => true);vi.spyOn(process.stderr, 'write').mockImplementation(() => true);- return () => vi.restoreAllMocks();});-const nodeCmd = (src) => ['node', '-e', src];--const run = (overrides) => {+const nodeCmd = (src) => [process.execPath, '-e', src];+const sourceSection = (source, start, end) => {+ const startIndex = source.indexOf(start);+ const endIndex = source.indexOf(end, startIndex);+ if (startIndex === -1 || endIndex === -1) throw new Error(`missing source section: ${start}`);+ return source.slice(startIndex, endIndex);+};+const throwTargets = (source) =>+ new Set(+ [...source.matchAll(/\bthrow\s+((?:new\s+)?[A-Za-z_$][\w$]*)/g)].map((match) => match[1]),+ );+const run = (overrides = {}) => {const lines = [];+ const now = Date.now();return runWithRetry({+ command: nodeCmd('process.exit(0)'),+ deadlineEpochMs: now + 120_000,+ attemptTimeoutMs: 10_000,+ cleanupReserveMs: 100,+ cleanupGraceMs: 20,+ pollIntervalMs: 2,sleepFn: () => Promise.resolve(),- log: (l) => lines.push(l),+ randomFn: () => 0,+ log: (line) => lines.push(line),...overrides,}).then((result) => ({ ...result, lines, log: lines.join('\n') }));};-describe('the retry decision', () => {- test('a command that succeeds runs exactly once', async () => {- const r = await run({ command: nodeCmd('console.log("built")') });- expect(r.ok).toBe(true);- expect(r.attempts).toBe(1);+const inspect = (text, options) => {+ const evidence = new FailureEvidence(options);+ evidence.ingest(Buffer.from(text));+ evidence.finish();+ return { evidence, classification: classifyEvidence(evidence) };+};++describe('failure evidence classification', () => {+ test.each([408, 429, 500, 502, 503, 504, 521, 522, 524])(+ 'classifies structured HTTP %s as transient',+ (status) => {+ expect(inspect(`HTTPError: Response code ${status} (service response)`).classification).toBe(+ 'transient',+ );+ },+ );++ test.each(['502 Bad Gateway', '503 Service Unavailable', '504 Gateway Time-out'])(+ 'keeps structured status-line coverage: %s',+ (text) => {+ expect(inspect(text).classification).toBe('transient');+ },+ );++ test.each([+ 'getaddrinfo EAI_AGAIN github.com',+ 'getaddrinfo ENOTFOUND github.com',+ 'connect ECONNREFUSED 127.0.0.1:443',+ 'connect ENETUNREACH 10.0.0.1:443',+ 'connect EHOSTUNREACH 10.0.0.1:443',+ 'read ECONNABORTED',+ 'read ECONNRESET',+ 'connect ETIMEDOUT',+ 'write EPIPE',+ 'cause: UND_ERR_CONNECT_TIMEOUT',+ 'cause: UND_ERR_HEADERS_TIMEOUT',+ 'cause: UND_ERR_BODY_TIMEOUT',+ 'cause: UND_ERR_SOCKET',+ 'curl: (28) Operation timed out',+ 'curl: (56) Recv failure: Connection reset by peer',+ ])('classifies scoped network evidence as transient: %s', (text) => {+ expect(inspect(text).classification).toBe('transient');+ });++ test.each([-1001, -1003, -1004, -1005, -1006, -1008, -1009, -1011])(+ 'classifies NSURLErrorDomain Code=%s as transient',+ (code) => {+ expect(inspect(`Error Domain=NSURLErrorDomain Code=${code}`).classification).toBe(+ 'transient',+ );+ },+ );++ test.each([-1201, -1202, -1203, -1204, -1205, -1206])(+ 'classifies bare NSURLErrorDomain TLS trust Code=%s as terminal',+ (code) => {+ expect(inspect(`Error Domain=NSURLErrorDomain Code=${code}`).classification).toBe('terminal');+ },+ );++ test('leaves generic NSURLErrorDomain secure-connection failure unknown', () => {+ expect(inspect('Error Domain=NSURLErrorDomain Code=-1200').classification).toBe('unknown');+ });++ test.each([+ 'A timestamp was expected but was not found',+ 'The timestamp service is not available',+ 'HTTPError(statusCode: nil)',+ 'The request timed out',+ 'The network connection was lost',+ 'Could not connect to the server',+ 'socket hang up',+ 'Client network socket disconnected before secure TLS connection was established',+ 'unexpected EOF',+ 'You have exceeded a secondary rate limit',+ 'abuse detection mechanism',+ 'was submitted too quickly',+ ])('keeps narrow transient phrase coverage: %s', (text) => {+ expect(inspect(text).classification).toBe('transient');+ });++ test.each([+ 'HTTP 401 Unauthorized',+ 'HTTP 400 Bad Request',+ 'HTTPError: Response code 422 (Unprocessable Entity)',+ 'response status: 404',+ 'statusCode=413',+ 'error TS2345: Argument of type string is not assignable',+ 'electron-vite build failed',+ 'Invalid configuration object',+ 'configuration is invalid',+ 'β¨― Invalid configuration object. electron-builder 26.0.1 has been initialized using a configuration object that does not match the API schema.',+ 'Error: unknown option "--foo"',+ ' ERR_PNPM_OUTDATED_LOCKFILE Cannot install with "frozen-lockfile" because pnpm-lock.yaml is not up to date',+ 'authentication failed for signing service',+ 'invalid credentials supplied to Azure Trusted Signing',+ 'The specified item could not be found in the keychain',+ 'certificate has expired',+ 'certificate not trusted',+ 'notarization failed with status: Invalid',+ 'notarytool submission completed\nstatus: Invalid',+ 'notarytool submission completed\n{"status":"Invalid"}',+ 'entitlement com.apple.security.foo is not permitted',+ 'Electron fuse verification failed',+ '[afterSign] fuse verification failed (D17 paranoid check):\n RunAsNode: expected ENABLE (target=true), got DISABLE [OK_PACKAGING_FUSE_FAILURE]',+ '[afterSign] fuse verification read failed on /tmp/OpenKnowledge: EACCES [OK_PACKAGING_FUSE_FAILURE]',+ '[afterPack] fuse flip failed on /tmp/OpenKnowledge: Could not find sentinel [OK_PACKAGING_FUSE_FAILURE]',+ 'Electron fuse mismatch',+ 'Electron fuse mismatches target',+ 'Electron fuse mismatched target',+ 'integrity check failed: checksum mismatch',+ 'sha512 checksum mismatch, expected AAA, got BBB',+ 'sha512 hash mismatch, expected AAA, got BBB',+ 'Error: Cannot find module ./missing',+ 'bash: pnpm: command not found',+ 'ERR_PNPM_RECURSIVE_EXEC_FIRST_FAIL Command "electron-builder" not found',+ 'spawn electron-builder ENOENT',+ 'ENOSPC: no space left on device',+ ])('classifies explicit terminal evidence as terminal: %s', (text) => {+ expect(inspect(text).classification).toBe('terminal');+ });++ test('classifies errors from the owned fuse-failure factory', () => {+ const error = createFuseFailure('fuse detail changed');+ expect(error.message).toBe(`fuse detail changed [${FUSE_FAILURE_MARKER}]`);+ expect(inspect(error.message).classification).toBe('terminal');+ });++ test('routes every throw in the owned fuse functions through the shared factory', () => {+ const afterPackFuseFunction = sourceSection(+ afterPackSource,+ 'async function flipElectronFuses',+ '\n\nexport default async function afterPack',+ );+ const afterSignFuseFunction = sourceSection(+ afterSignSource,+ 'async function verifyFuses',+ '\n\nexport default async function afterSign',+ );+ expect(throwTargets(afterPackFuseFunction)).toEqual(new Set(['createFuseFailure']));+ expect(throwTargets(afterSignFuseFunction)).toEqual(new Set(['createFuseFailure']));+ });++ test.each([+ '[afterSign] getCurrentFuseWire failed on /tmp/OpenKnowledge: EACCES',+ '[afterSign] Fuse verification failed (D17 paranoid check):\n RunAsNode: expected ENABLE (target=true), got DISABLE',+ '[afterPack] fuse flip failed on /tmp/OpenKnowledge: Could not find sentinel',+ ])('classifies old-tag fuse failures without the owned marker: %s', (text) => {+ expect(inspect(text).classification).toBe('terminal');+ });++ test.each([+ 'warning: optional tool: command not found\nsocket hang up',+ 'download retry noted checksum mismatch\nECONNRESET',+ 'warning: unknown option "--foo"\nsocket hang up',+ ])('does not let incidental terminal-like prose outrank network evidence: %s', (text) => {+ expect(inspect(text).classification).toBe('transient');+ });++ test.each([+ '[afterSign] fuse verification passed β all 6 fuses match targetFuses',+ '[afterSign] fuse verification done; no notarize step on platform "linux"',+ '[afterSign] signed + notarized + stapled + fuse-verified successfully',+ '[afterPack] skipping per-arch temp "/tmp/app-temp" β fuses flip on the merged universal app',+ '[afterPack] flipping fuses on /tmp/OpenKnowledge',+ '[afterPack] fuses flipped successfully; electron-builder will re-sign next',+ '[afterPack] fuses done; skipping darwin-only helper-bundle + node-pty steps on "linux"',+ 'fuse: failed to exec fusermount: No such file or directory',+ ])('does not classify excluded fuse output as terminal: %s', (text) => {+ expect(inspect(`${text}\nHTTP 500`).classification).toBe('transient');+ });++ test.each(['unknown option "--foo"', 'bash: pnpm: command not found'])(+ 'does not treat a chunk-carry boundary as a line boundary: %s',+ (phrase) => {+ const evidence = new FailureEvidence();+ evidence.ingest(+ Buffer.from(`${'x'.repeat(2048)}${phrase}${'y'.repeat(2048 - phrase.length)}`),+ );+ evidence.ingest(Buffer.from('\nsocket hang up'));+ evidence.finish();+ expect(classifyEvidence(evidence)).toBe('transient');+ },+ );++ test('does not treat a chunk-carry boundary as a module-error line start', () => {+ const phrase = 'Error: Cannot find module ./missing';+ const evidence = new FailureEvidence();+ evidence.ingest(Buffer.from(`${'x'.repeat(2048)}${phrase}${'y'.repeat(2048 - phrase.length)}`));+ evidence.ingest(Buffer.from('\nsocket hang up'));+ evidence.finish();+ expect(classifyEvidence(evidence)).toBe('transient');+ });++ test('does not treat a chunk-carry boundary as a decorated-status line start', () => {+ const phrase = 'β¨― 502 Bad Gateway';+ const evidence = new FailureEvidence();+ evidence.ingest(Buffer.from(`${'x'.repeat(2048)}${phrase}${'y'.repeat(2048 - phrase.length)}`));+ evidence.finish();+ expect(classifyEvidence(evidence)).toBe('unknown');+ });++ test.each([+ ['Error: unknown option "--foo"', 'terminal'],+ ['Error: Cannot find module ./missing', 'terminal'],+ ['bash: pnpm: command not found', 'terminal'],+ ['β¨― 502 Bad Gateway', 'transient'],+ ])('retains a real line boundary across chunks: %s', (line, classification) => {+ const evidence = new FailureEvidence();+ evidence.ingest(Buffer.from('progress without a newline'));+ evidence.ingest(Buffer.from(`\n${line}`));+ evidence.finish();+ expect(classifyEvidence(evidence)).toBe(classification);+ });++ test('distinguishes permission-denied and rate-limited HTTP 403', () => {+ expect(inspect('HTTP 403 Forbidden').classification).toBe('terminal');+ expect(inspect('HTTP 403 Forbidden\nsecondary rate limit exceeded').classification).toBe(+ 'transient',+ );+ expect(inspect('HTTP 403 Forbidden\nRetry-After: 45').classification).toBe('terminal');+ expect(inspect('HTTP 403 Forbidden\nX-RateLimit-Remaining: 0').classification).toBe(+ 'transient',+ );+ expect(inspect('HTTP 403 Forbidden\nRetry-After: tomorrow').classification).toBe('terminal');+ });++ test.each(['β¨― 502 Bad Gateway', 'Error: 503 Service Unavailable'])(+ 'accepts explicit decorated status lines: %s',+ (text) => expect(inspect(text).classification).toBe('transient'),+ );++ test('rejects arbitrary decorated status lines', () => {+ expect(inspect('download failed: 502 Bad Gateway').classification).toBe('unknown');+ });++ test('matched reasons follow classifier precedence for mixed evidence', async () => {+ const terminal = await run({+ command: nodeCmd('console.error("certificate has expired\\nHTTP 404");process.exit(1)'),+ });+ expect(terminal.log).toContain('reason=rule:certificate outcome=terminal');+ expect(terminal.log).not.toContain('reason=http:404');+ const downloadIntegrity = await run({+ command: nodeCmd(+ 'console.error("sha512 checksum mismatch, expected AAA, got BBB");process.exit(1)',+ ),+ });+ expect(downloadIntegrity.log).toContain('reason=rule:download-integrity outcome=terminal');+ const maskedIntegrity = await run({+ command: nodeCmd(+ 'console.error("sha512 checksum mismatch, expected AAA, got BBB\\nrejected entitlements for app");process.exit(1)',+ ),+ });+ expect(maskedIntegrity.log).toContain('reason=rule:entitlement outcome=terminal');+ const maskedTlsTrust = await run({+ command: nodeCmd(+ 'console.error("sha512 checksum mismatch, expected AAA, got BBB\\nError Domain=NSURLErrorDomain Code=-1202");process.exit(1)',+ ),+ });+ expect(maskedTlsTrust.log).toContain('reason=rule:tls-trust outcome=terminal');+ expect(maskedTlsTrust.log).not.toContain('rule:download-integrity');+ const tlsTrustWithSymptom = await run({+ command: nodeCmd(+ 'console.error("Error Domain=NSURLErrorDomain Code=-1202\\nError: Exit code: ENOENT. spawn /Users/runner/Library/Caches/electron-builder/app-builder/app-builder ENOENT");process.exit(1)',+ ),+ });+ expect(tlsTrustWithSymptom.log).toContain('reason=rule:tls-trust outcome=terminal');+ const tlsTrustWithIntegrity = await run({+ command: nodeCmd(+ 'console.error("Error Domain=NSURLErrorDomain Code=-1202\\nintegrity check failed");process.exit(1)',+ ),+ });+ expect(tlsTrustWithIntegrity.log).toContain('reason=rule:tls-trust outcome=terminal');+ const notarizationInvalidWithTlsNoise = await run({+ command: nodeCmd(+ 'console.error("Error Domain=NSURLErrorDomain Code=-1202");console.error(`Failed to notarize via notarytool\\n{"id":"abc","status":"Invalid","message":"Processing complete"}`);process.exit(1)',+ ),+ });+ expect(notarizationInvalidWithTlsNoise.log).toContain(+ 'reason=rule:notarization-invalid outcome=terminal',+ );+ const fuseFailure = await run({+ command: nodeCmd(+ 'console.error("[afterPack] fuse flip failed on /tmp/OpenKnowledge: Could not find sentinel [OK_PACKAGING_FUSE_FAILURE]");process.exit(1)',+ ),+ });+ expect(fuseFailure.log).toContain('reason=rule:fuse outcome=terminal');+ const d17FuseFailure = await run({+ command: nodeCmd(+ 'console.error("[afterSign] fuse verification failed (D17 paranoid check):\\n RunAsNode: expected ENABLE (target=true), got DISABLE [OK_PACKAGING_FUSE_FAILURE]");process.exit(1)',+ ),+ });+ expect(d17FuseFailure.log).toContain('reason=rule:fuse outcome=terminal');+ const diskFull = await run({+ command: nodeCmd(+ 'console.error("ENOSPC: no space left on device\\nintegrity check failed");process.exit(1)',+ ),+ });+ expect(diskFull.log).toContain('reason=rule:disk-full outcome=terminal');+ const diskFullWithCertificate = await run({+ command: nodeCmd(+ 'console.error("certificate has expired\\nENOSPC: no space left on device");process.exit(1)',+ ),+ });+ expect(diskFullWithCertificate.log).toContain('reason=rule:disk-full outcome=terminal');+ const signedIntegrity = await run({+ command: nodeCmd(+ 'console.error("integrity check failed: code signature invalid");process.exit(1)',+ ),+ });+ expect(signedIntegrity.log).toContain('reason=rule:integrity outcome=terminal');+ expect(signedIntegrity.log).not.toContain('download-integrity');+ const transient = await run({+ command: nodeCmd('console.error("ECONNRESET\\nHTTP 503");process.exit(1)'),+ maxAttempts: 1,+ });+ expect(transient.log).toContain('reason=code:ECONNRESET');+ expect(transient.log).not.toContain('reason=http:503');+ });++ test.each([+ 'earlier warning: ECONNRESET\nnotarization failed with status: Invalid',+ 'certificate has expired\nlater warning: socket hang up',+ ])('terminal evidence dominates transient evidence in either order', (text) => {+ expect(inspect(text).classification).toBe('terminal');+ });++ test('deletes broad CFNetwork matching and retains byte-capped early evidence', () => {+ expect(+ inspect('Error Domain=kCFErrorDomainCFNetwork Code=-1202 certificate not trusted')+ .classification,+ ).toBe('terminal');+ expect(inspect('Error Domain=NSURLErrorDomain Code=-1202').classification).toBe('terminal');+ const evidence = new FailureEvidence({ maxTailBytes: 19 });+ evidence.ingest(Buffer.from('certificate has expired\n'));+ evidence.ingest(Buffer.from('π'.repeat(100)));+ evidence.finish();+ expect(classifyEvidence(evidence)).toBe('terminal');+ expect(Buffer.byteLength(evidence.tail, 'utf8')).toBeLessThanOrEqual(19);+ expect(evidence.tail).not.toContain('certificate');+ });++ test('does not consume the first-line anchor on an empty chunk', () => {+ const evidence = new FailureEvidence();+ evidence.ingest(Buffer.alloc(0));+ evidence.ingest(Buffer.from('Error: unknown option "--publish"'));+ evidence.finish();+ expect(classifyEvidence(evidence)).toBe('terminal');+ });++ test('leaves unrelated failures unknown', () => {+ expect(inspect('packager exited without a diagnostic').classification).toBe('unknown');});+});- test('a NON-transient failure fails loud on the first attempt', async () => {- const r = await run({- command: nodeCmd('console.error("notarization status: Invalid");process.exit(1)'),+describe('retry state machine', () => {+ test('success runs once and terminal evidence never retries', async () => {+ expect(await run({ command: nodeCmd('console.log("built")') })).toMatchObject({+ ok: true,+ attempts: 1,});- expect(r.ok).toBe(false);- expect(r.reason).toBe('non-transient');- expect(r.attempts).toBe(1);- expect(r.log).toContain('NON-transient');+ const terminal = await run({+ command: nodeCmd(+ 'console.error("socket hang up");console.error("notarization failed with status: Invalid");process.exit(1)',+ ),+ });+ expect(terminal).toMatchObject({ ok: false, reason: 'terminal', attempts: 1 });+ expect(terminal.log).toContain(+ 'decision=stop reason=rule:notarization-invalid outcome=terminal attempt=1/3 code=1 signal=none',+ );});- test('a transient failure is retried and can succeed', async () => {- const counter = join(scratch, 'flaky-count');- writeFileSync(counter, '0');- const r = await run({+ test('shell mode executes the workflow command with Bash', async () => {+ expect(await run({ command: ['true && printf composed'], shell: true })).toMatchObject({+ ok: true,+ attempts: 1,+ });+ });++ test('transient failures retry within three total attempts and can recover', async () => {+ const count = join(scratch, 'transient-recovery-count');+ writeFileSync(count, '0');+ const result = await run({command: nodeCmd(- `const fs=require('fs');const p=${JSON.stringify(counter)};` +- `const n=Number(fs.readFileSync(p,'utf8'))+1;fs.writeFileSync(p,String(n));` +- `if(n<2){console.error('β¨― socket hang up failedTask=build');process.exit(1)}` +- `console.log('packaged')`,+ `const fs=require('fs');const p=${JSON.stringify(count)};const n=+fs.readFileSync(p,'utf8')+1;fs.writeFileSync(p,String(n));if(n<3){console.error('HTTPError: Response code 500 (Internal Server Error)');process.exit(1)}`,),});- expect(r.ok).toBe(true);- expect(r.attempts).toBe(2);- expect(readFileSync(counter, 'utf8')).toBe('2');+ expect(result).toMatchObject({ ok: true, attempts: 3 });+ expect(readFileSync(count, 'utf8')).toBe('3');+ expect(result.log).toContain(+ 'decision=retry reason=http:500 classification=transient attempt=1/3 code=1 signal=none',+ );});- test('a permanently transient failure stops at the attempt bound', async () => {- const r = await run({+ test('persistent transient failure stops at the total-attempt bound', async () => {+ const result = await run({command: nodeCmd('console.error("socket hang up");process.exit(1)'),- maxAttempts: 3,});- expect(r.ok).toBe(false);- expect(r.reason).toBe('exhausted');- expect(r.attempts).toBe(3);- expect(r.log).toContain('retries are exhausted');+ expect(result).toMatchObject({+ ok: false,+ reason: 'transient-exhausted',+ attempts: DEFAULT_MAX_ATTEMPTS,+ });+ expect(result.log).toContain('outcome=transient-exhausted');+ });++ test('one unknown can recover but a second unknown stops', async () => {+ const count = join(scratch, 'unknown-recovery-count');+ writeFileSync(count, '0');+ const recovered = await run({+ command: nodeCmd(+ `const fs=require('fs');const p=${JSON.stringify(count)};const n=+fs.readFileSync(p,'utf8')+1;fs.writeFileSync(p,String(n));if(n===1){console.error('unrecognized packager failure');process.exit(1)}`,+ ),+ });+ expect(recovered).toMatchObject({ ok: true, attempts: 2, recoveredFrom: 'unknown' });+ expect(recovered.log).toContain(+ 'decision=retry reason=diagnostic:unknown classification=unknown attempt=1/3 code=1 signal=none',+ );+ expect(recovered.log).toContain(+ 'UNKNOWN_CLASSIFICATION_RETRY allowance=invocation-wide-single-use',+ );+ const exhausted = await run({+ command: nodeCmd('console.error("unrecognized packager failure");process.exit(1)'),+ });+ expect(exhausted).toMatchObject({ ok: false, reason: 'unknown-exhausted', attempts: 2 });+ expect(exhausted.log).toContain('outcome=unknown-exhausted');+ const bounded = await run({+ command: nodeCmd('console.error("unrecognized packager failure");process.exit(1)'),+ maxAttempts: 1,+ });+ expect(bounded).toMatchObject({ ok: false, reason: 'unknown-exhausted', attempts: 1 });+ expect(bounded.log).toContain('outcome=unknown-exhausted');});- test('a command that cannot be spawned at all is not retried forever', async () => {- const r = await run({ command: ['definitely-not-a-real-binary-xyz'], maxAttempts: 2 });- expect(r.ok).toBe(false);- expect(r.reason).toBe('non-transient');+ test('the unknown allowance is shared across a mixed sequence', async () => {+ const count = join(scratch, 'mixed-count');+ writeFileSync(count, '0');+ const result = await run({+ command: nodeCmd(+ `const fs=require('fs');const p=${JSON.stringify(count)};const n=+fs.readFileSync(p,'utf8')+1;fs.writeFileSync(p,String(n));console.error(n===2?'socket hang up':'unrecognized packager failure');process.exit(1)`,+ ),+ });+ expect(result).toMatchObject({ ok: false, reason: 'unknown-exhausted', attempts: 3 });+ expect(result.log).toContain('outcome=unknown-exhausted');});- test('--shell mode runs a compound command', async () => {- const r = await run({ command: ['true && echo composed'], shell: true });- expect(r.ok).toBe(true);+ test('spawn failure and child signals are process-control stops', async () => {+ const spawnFailed = await run({ command: ['definitely-not-a-real-binary-xyz'] });+ expect(spawnFailed).toMatchObject({+ ok: false,+ reason: 'spawn-failure',+ attempts: 1,+ });+ expect(spawnFailed.log).toContain('outcome=spawn-failure');+ expect(spawnFailed.log).not.toContain('cleanup=');+ const signalled = await run({ command: nodeCmd('process.kill(process.pid, "SIGTERM")') });+ expect(signalled).toMatchObject({+ ok: false,+ reason: 'child-signal',+ attempts: 1,+ signal: 'SIGTERM',+ });+ expect(signalled.log).toContain(+ 'decision=stop reason=control:child-signal outcome=child-signal attempt=1/3 code=none signal=SIGTERM',+ );});-});-describe('the transient allowlist', () => {- test('matches the signature that blocked the 0.52.2 and 0.52.3 cuts', () => {- expect(isTransient('β¨― socket hang up failedTask=build stackTrace=RequestError: socket hang up')).toBe(- true,+ test('the re-emitted bounded tail is redacted and workflow-command safe', async () => {+ const secret = 'super-secret-output-value';+ const sensitive = [+ `CSC_KEY_PASSWORD=${secret}`,+ `Authorization: Bearer ${secret}`,+ `"api_key": "${secret}"`,+ '-----BEGIN PRIVATE KEY-----',+ 'private-body',+ '-----END PRIVATE KEY-----',+ '::error::injected',+ '##[error]legacy-injected',+ ].join('\n');+ const result = await run({+ command: nodeCmd(`console.error(${JSON.stringify(sensitive)});process.exit(1)`),+ maxAttempts: 1,+ });+ expect(result.log).toContain(+ 'reason=diagnostic:unknown outcome=unknown-exhausted attempt=1/1 code=1 signal=none',);+ expect(result.log).toContain('::group::command bounded failure diagnostic');+ expect(result.log).toContain('| CSC_KEY_PASSWORD=[REDACTED]');+ expect(result.log).toContain('| Authorization: [REDACTED]');+ expect(result.log).toContain('| "api_key": [REDACTED]');+ expect(result.log).toContain('| [REDACTED PEM]');+ expect(result.log).toContain('| ::error::injected');+ expect(result.log).not.toContain('\n::error::injected');+ expect(result.log).toContain('| # #[error]legacy-injected');+ expect(result.log).not.toContain('##[');+ expect(result.log).not.toContain(secret);+ expect(result.log).not.toContain('private-body');+ expect(result).not.toHaveProperty('diagnosticTail');});- test('matches case-insensitively and mid-line', () => {- expect(isTransient('...Socket Hang Up...')).toBe(true);- expect(isTransient('fetch failed: ECONNRESET')).toBe(true);+ test('the live child stream remains verbatim before the bounded duplicate', async () => {+ const marker = 'PUBLIC_TRANSCRIPT_MARKER';+ const result = await run({+ command: nodeCmd(`console.error(${JSON.stringify(marker)});process.exit(1)`),+ maxAttempts: 1,+ });+ const liveStderr = process.stderr.write.mock.calls.map(([chunk]) => String(chunk)).join('');+ expect(liveStderr).toContain(marker);+ expect(result.log).toContain(`| ${marker}`);});- test('does NOT match real build, cert, or notarization failures', () => {- for (const real of [- 'error TS2345: Argument of type string is not assignable',- 'The specified item could not be found in the keychain',- 'certificate has expired',- 'notarization failed with status: Invalid',- 'Command failed with exit code 1: tsc --noEmit',- 'Error: Cannot find module ./missing',- 'ENOSPC: no space left on device',- ]) {- expect(isTransient(real), `${real} must not be treated as transient`).toBe(false);- }+ test('production diagnostic output remains byte bounded', async () => {+ const result = await run({+ command: nodeCmd(+ `console.error(${JSON.stringify(`rolled-off-marker\n${'x'.repeat(40_000)}\nfinal-marker`)});process.exit(1)`,+ ),+ maxAttempts: 1,+ });+ expect(result.log).not.toContain('rolled-off-marker');+ expect(result.log).toContain('final-marker');+ expect(Buffer.byteLength(result.log, 'utf8')).toBeLessThan(18_000);});- test('handles empty and nullish output without throwing', () => {- expect(isTransient('')).toBe(false);- expect(isTransient(undefined)).toBe(false);- expect(isTransient(null)).toBe(false);+ test('unproven cleanup stops before diagnostic classification', async () => {+ const result = await run({+ command: nodeCmd('console.error("socket hang up");process.exit(1)'),+ treeController: { cleanup: async () => ({ ok: false, reason: 'tree-survived-kill' }) },+ });+ expect(result).toMatchObject({+ ok: false,+ reason: 'cleanup-failure',+ cleanup: 'tree-survived-kill',+ attempts: 1,+ });+ expect(result.log).toContain('cleanup=tree-survived-kill');});- test('every signature is a real regex and none is so broad it matches bare text', () => {- for (const sig of TRANSIENT_SIGNATURES) {- expect(() => new RegExp(sig)).not.toThrow();- expect(new RegExp(sig, 'i').test('Building desktop app')).toBe(false);- }+ test('exit zero with unproven cleanup fails closed', async () => {+ const result = await run({+ command: nodeCmd('process.exit(0)'),+ treeController: { cleanup: async () => ({ ok: false, reason: 'close-not-observed' }) },+ });+ expect(result).toMatchObject({ ok: false, reason: 'cleanup-failure', attempts: 1 });+ expect(result.log).toContain('reason=control:cleanup-failure outcome=cleanup-failure');+ expect(result.log).toContain('cleanup=close-not-observed');+ });++ test('retry warning emits only when a retry is actually scheduled', async () => {+ const warning = 'duplicate Apple notarization submission risk';+ const success = await run({ retryWarning: warning });+ expect(success.log).not.toContain(warning);+ expect(success.log).not.toContain('bounded failure diagnostic');+ const terminal = await run({+ command: nodeCmd('console.error("certificate has expired");process.exit(1)'),+ retryWarning: warning,+ });+ expect(terminal.log).not.toContain(warning);+ const transient = await run({+ command: nodeCmd('console.error("socket hang up");process.exit(1)'),+ retryWarning: warning,+ });+ expect(transient.log.match(new RegExp(warning, 'g'))).toHaveLength(2);+ });++ test('the state machine rejects an invalid attempt bound before installing handlers', async () => {+ const signals = new EventEmitter();+ await expect(run({ maxAttempts: 0, signalSource: signals })).rejects.toThrow(+ /maxAttempts must be an integer from 1 to 3/,+ );+ expect(signals.listenerCount('SIGINT')).toBe(0);+ expect(signals.listenerCount('SIGTERM')).toBe(0);});});-describe('backoff', () => {- test('grows between attempts and stays bounded for the default attempt count', () => {- expect(backoffSeconds(1)).toBe(30);- expect(backoffSeconds(2)).toBeGreaterThan(backoffSeconds(1));- const total = [...Array(DEFAULT_MAX_ATTEMPTS - 1).keys()]- .map((i) => backoffSeconds(i + 1))- .reduce((a, b) => a + b, 0);- expect(total).toBeLessThan(10 * 60);+describe('retry delay', () => {+ test('uses jittered 30s and 60s bases', () => {+ const evidence = inspect('socket hang up').evidence;+ expect(computeRetryDelayMs(1, evidence, { randomFn: () => 0, nowMs: 0 })).toBe(30_001);+ expect(computeRetryDelayMs(2, evidence, { randomFn: () => 0, nowMs: 0 })).toBe(60_001);+ expect(computeRetryDelayMs(2, evidence, { randomFn: () => 1, nowMs: 0 })).toBeLessThanOrEqual(+ 65_000,+ );+ });++ test('honors both Retry-After forms as a minimum plus positive jitter', () => {+ const delta = inspect('HTTP 429\nRetry-After: 120', { nowFn: () => 0 }).evidence;+ expect(computeRetryDelayMs(1, delta, { randomFn: () => 0, nowMs: 0 })).toBe(120_001);+ const nowMs = Date.parse('2026-09-11T20:00:00Z');+ const retryAt = new Date(nowMs + 90_000).toUTCString();+ const date = inspect(`HTTP 429\nRetry-After: ${retryAt}`, { nowFn: () => nowMs }).evidence;+ expect(computeRetryDelayMs(1, date, { randomFn: () => 0, nowMs })).toBe(90_001);+ });++ test('does not reinterpret a carried delta Retry-After against a later clock', () => {+ let nowMs = 0;+ const evidence = new FailureEvidence({ nowFn: () => nowMs });+ evidence.ingest(Buffer.from('HTTP 429\nRetry-After: 120\n'));+ nowMs = 10_000;+ evidence.ingest(Buffer.from('download cleanup\n'));+ nowMs = 20_000;+ evidence.ingest(Buffer.from('another chunk\n'));+ evidence.finish();+ expect(evidence.retryAfterEpochMs).toBe(120_000);+ expect(computeRetryDelayMs(1, evidence, { randomFn: () => 0, nowMs })).toBe(100_001);+ });++ test('accepts only header-shaped lines and keeps the first valid Retry-After', () => {+ const evidence = new FailureEvidence({ nowFn: () => 0 });+ evidence.ingest(+ Buffer.from('noise Retry-After: 900\nError: Retry-After: 800\n* Retry-After: 700\n'),+ );+ evidence.ingest(Buffer.from('< Retry-After: 45\nRetry-After: 120\n'));+ evidence.finish();+ expect(evidence.retryAfterEpochMs).toBe(45_000);});});-describe('parseArgs', () => {- const argv = (...rest) => ['node', 'x', ...rest];+describe('deadlines and cancellation', () => {+ const completedAttempt = (text, code = 1, options) => ({+ code,+ closeSignal: null,+ cancellationSignal: null,+ cancelled: false,+ timedOut: false,+ deadlineExpired: false,+ spawnError: null,+ cleanup: { ok: true, reason: 'clean' },+ evidence: inspect(text, options).evidence,+ });+ const assertPidDead = (path) => {+ const pid = Number(readFileSync(path, 'utf8'));+ expect(() => process.kill(pid, 0)).toThrow();+ };- test('splits flags from the command at `--`', () => {- const p = parseArgs(argv('--label', 'pkg (linux)', '--', 'pnpm', 'exec', 'electron-builder'));- expect(p.label).toBe('pkg (linux)');- expect(p.command).toEqual(['pnpm', 'exec', 'electron-builder']);- expect(p.shell).toBe(false);- expect(p.maxAttempts).toBe(DEFAULT_MAX_ATTEMPTS);+ test('real macOS budget arithmetic admits a quick incident-style retry', async () => {+ const minute = 60_000;+ const timing = workflowTiming(WORKFLOW_JOBS[0]);+ let clock = 15 * minute;+ let attempt = 0;+ const result = await run({+ deadlineEpochMs: timing.budgetMs,+ attemptTimeoutMs: timing.attemptTimeoutMs,+ cleanupReserveMs: 15_000,+ nowFn: () => clock,+ sleepFn: async (ms) => {+ clock += ms;+ },+ attemptRunner: async () => {+ attempt += 1;+ clock += minute;+ return attempt === 1+ ? completedAttempt('HTTPError: Response code 500 (Internal Server Error)')+ : completedAttempt('', 0);+ },+ });+ expect(result).toMatchObject({ ok: true, attempts: 2 });});- test('a flag-looking token after `--` belongs to the command', () => {- const p = parseArgs(argv('--', 'pnpm', 'exec', 'electron-builder', '--linux', '--publish', 'never'));- expect(p.command).toContain('--linux');- expect(p.command).toContain('never');+ test('real Windows and Linux budget arithmetic rejects a doomed retry projection', async () => {+ const minute = 60_000;+ const windowsTiming = workflowTiming(WORKFLOW_JOBS[1]);+ const linuxTiming = workflowTiming(WORKFLOW_JOBS[2]);+ expect(linuxTiming).toEqual(windowsTiming);+ const execute = async (evidenceText) => {+ let clock = 19 * minute;+ return run({+ deadlineEpochMs: windowsTiming.budgetMs,+ attemptTimeoutMs: windowsTiming.attemptTimeoutMs,+ cleanupReserveMs: 15_000,+ nowFn: () => clock,+ attemptRunner: async () => {+ clock += 12 * minute;+ return completedAttempt(evidenceText, 1, { nowFn: () => clock });+ },+ });+ };+ const baseline = await execute('socket hang up');+ const withHeader = await execute('HTTP 429\nRetry-After: 31');+ for (const result of [baseline, withHeader]) {+ expect(result).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });+ expect(result.log).toContain('reason=control:deadline outcome=deadline');+ expect(result.log).toContain('phase=before-backoff');+ expect(result.log).toContain('projected-attempt-ms=900000');+ }+ expect(baseline.log).not.toContain('retry-after-ms=');+ expect(baseline.log).toContain('matched=rule:socket-hangup');+ expect(withHeader.log).toContain('retry-after-ms=31000');+ expect(withHeader.log).toContain('matched=http:429');});- test('requires a separator and a command', () => {- expect(() => parseArgs(argv('--label', 'x'))).toThrow(/--/);- expect(() => parseArgs(argv('--label', 'x', '--'))).toThrow(/no command/);+ test('a genuine Retry-After beyond the wall-clock budget stops explicitly', async () => {+ const timing = workflowTiming(WORKFLOW_JOBS[1]);+ let clock = 0;+ const result = await run({+ deadlineEpochMs: timing.budgetMs,+ attemptTimeoutMs: timing.attemptTimeoutMs,+ nowFn: () => clock,+ attemptRunner: async () => {+ clock += 1_000;+ return completedAttempt('HTTP 429\nRetry-After: 3600', 1, { nowFn: () => clock });+ },+ });+ expect(result).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });+ expect(result.log).toContain('reason=control:deadline outcome=deadline');+ expect(result.log).toContain('retry-after-ms=3600000');+ expect(result.log).toContain('projected-attempt-ms=1250');});- test('--shell demands exactly one command string', () => {- expect(() => parseArgs(argv('--shell', '--', 'a', 'b'))).toThrow(/exactly one/);- expect(parseArgs(argv('--shell', '--', 'a && b')).shell).toBe(true);+ test('attempt timeout stops and cleans the child without retrying', async () => {+ const pidFile = join(scratch, 'timeout-child-pid');+ const result = await run({+ command: nodeCmd(+ `require('fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));setInterval(() => {},1000)`,+ ),+ attemptTimeoutMs: 40,+ deadlineEpochMs: Date.now() + 5_000,+ });+ expect(result).toMatchObject({ ok: false, reason: 'attempt-timeout', attempts: 1 });+ expect(result.log).toContain(+ 'decision=stop reason=control:attempt-timeout outcome=attempt-timeout attempt=1/3 code=none signal=SIGTERM',+ );+ expect(result.log).not.toContain('phase=');+ assertPidDead(pidFile);+ });++ test('absolute deadline stops and cleans a live child', async () => {+ const pidFile = join(scratch, 'deadline-child-pid');+ const result = await run({+ command: nodeCmd(+ `require('fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));setInterval(() => {},1000)`,+ ),+ attemptTimeoutMs: 10_000,+ cleanupReserveMs: 10,+ deadlineEpochMs: Date.now() + 1_100,+ });+ expect(result).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });+ expect(result.log).toContain('reason=control:deadline outcome=deadline');+ expect(result.log).toContain('phase=mid-attempt');+ assertPidDead(pidFile);+ });++ test('does not start an attempt or backoff that cannot fit the deadline', async () => {+ const spawnFn = vi.fn();+ const nowMs = 10_000;+ const beforeAttempt = await run({+ spawnFn,+ nowFn: () => nowMs,+ deadlineEpochMs: nowMs + 1_099,+ attemptTimeoutMs: 1_000,+ cleanupReserveMs: 100,+ });+ expect(beforeAttempt).toMatchObject({ ok: false, reason: 'deadline', attempts: 0 });+ expect(beforeAttempt.log).toContain('reason=control:deadline outcome=deadline');+ expect(beforeAttempt.log).toContain('phase=before-attempt');+ expect(spawnFn).not.toHaveBeenCalled();+ expect(+ await run({+ command: nodeCmd('console.error("socket hang up");process.exit(1)'),+ nowFn: () => nowMs,+ deadlineEpochMs: nowMs + 31_000,+ attemptTimeoutMs: 1_000,+ cleanupReserveMs: 50,+ }),+ ).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });+ });++ test('global deadline exhaustion aborts an active attempt', async () => {+ let reads = 0;+ const result = await run({+ command: nodeCmd('setInterval(() => {}, 1000)'),+ nowFn: () => {+ reads += 1;+ return reads === 1 ? 0 : 1_401;+ },+ deadlineEpochMs: 1_500,+ attemptTimeoutMs: 1_000,+ cleanupReserveMs: 100,+ });+ expect(result).toMatchObject({ ok: false, reason: 'deadline', attempts: 1 });+ expect(result.log).toContain('reason=control:deadline outcome=deadline');+ });++ test('parent cancellation during a child stops the tree and removes handlers', async () => {+ const signals = new EventEmitter();+ const pidFile = join(scratch, 'cancelled-child-pid');+ const promise = run({+ command: nodeCmd(+ `require('fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));console.log('ready');setInterval(() => {},1000)`,+ ),+ signalSource: signals,+ });+ while (!existsSync(pidFile)) await new Promise((resolve) => setTimeout(resolve, 5));+ signals.emit('SIGINT');+ const result = await promise;+ expect(result).toMatchObject({ ok: false, reason: 'signal', signal: 'SIGINT', attempts: 1 });+ expect(result.log).toContain('reason=control:signal outcome=signal');+ expect(signals.listenerCount('SIGINT')).toBe(0);+ expect(signals.listenerCount('SIGTERM')).toBe(0);+ assertPidDead(pidFile);+ });++ test('parent cancellation during backoff prevents another attempt', async () => {+ const signals = new EventEmitter();+ const count = join(scratch, 'backoff-cancel-count');+ let enterBackoff;+ const backoffStarted = new Promise((resolve) => {+ enterBackoff = resolve;+ });+ writeFileSync(count, '0');+ const promise = run({+ command: nodeCmd(+ `const fs=require('fs');const p=${JSON.stringify(count)};fs.writeFileSync(p,String(+fs.readFileSync(p,'utf8')+1));console.error('socket hang up');process.exit(1)`,+ ),+ signalSource: signals,+ sleepFn: (_ms, signal) =>+ new Promise((resolve) => {+ enterBackoff();+ signal.addEventListener('abort', resolve, { once: true });+ }),+ });+ await backoffStarted;+ signals.emit('SIGTERM');+ const result = await promise;+ expect(result).toMatchObject({+ ok: false,+ reason: 'signal',+ signal: 'SIGTERM',+ attempts: 1,+ });+ expect(result.log).toContain('reason=control:signal outcome=signal');+ expect(readFileSync(count, 'utf8')).toBe('1');});++ test.runIf(process.platform !== 'win32')(+ 'reaps a real grandchild before the next attempt starts',+ async () => {+ const helper = join(scratch, 'tree-helper.mjs');+ const count = join(scratch, 'tree-count');+ const pidFile = join(scratch, 'grandchild-pid');+ const overlap = join(scratch, 'tree-overlap');+ writeFileSync(count, '0');+ writeFileSync(+ helper,+ `import { spawn } from 'node:child_process';import { readFileSync,writeFileSync } from 'node:fs';const [count,pidFile,overlap]=process.argv.slice(2);const n=+readFileSync(count,'utf8')+1;writeFileSync(count,String(n));if(n===1){const child=spawn(process.execPath,['-e','process.on("SIGTERM",()=>{});setInterval(()=>{},1000)'],{stdio:'ignore'});writeFileSync(pidFile,String(child.pid));console.error('socket hang up');process.exit(1)}const pid=+readFileSync(pidFile,'utf8');try{process.kill(pid,0);writeFileSync(overlap,'alive');process.exit(1)}catch{process.exit(0)}`,+ );+ const result = await run({+ command: [process.execPath, helper, count, pidFile, overlap],+ cleanupGraceMs: 30,+ });+ expect(result).toMatchObject({ ok: true, attempts: 2 });+ expect(existsSync(overlap)).toBe(false);+ expect(() => process.kill(Number(readFileSync(pidFile, 'utf8')), 0)).toThrow();+ },+ 5_000,+ );++ test.runIf(process.platform !== 'win32')(+ 'the CLI exits with the same parent signal after cleanup',+ async () => {+ const child = spawn(+ process.execPath,+ [+ SCRIPT,+ '--deadline-epoch-ms',+ String(Date.now() + 60_000),+ '--attempt-timeout',+ '30s',+ '--',+ process.execPath,+ '-e',+ 'console.log("ready");setInterval(()=>{},1000)',+ ],+ { stdio: ['ignore', 'pipe', 'pipe'] },+ );+ await new Promise((resolve, reject) => {+ child.once('error', reject);+ child.stdout.once('data', resolve);+ });+ child.kill('SIGTERM');+ const closed = await new Promise((resolve) => {+ child.once('close', (code, signal) => resolve({ code, signal }));+ });+ expect(closed).toEqual({ code: null, signal: 'SIGTERM' });+ },+ 5_000,+ );});-describe('workflow wiring', () => {- const step = (needle) => {- const start = desktopRelease.indexOf(`- name: ${needle}`);- if (start === -1) throw new Error(`desktop-release.yml has no step named ${needle}`);- const rest = desktopRelease.slice(start);- const end = rest.indexOf('\n - name: ');- return end === -1 ? rest : rest.slice(0, end);- };+describe('POSIX owned-tree cleanup reasons', () => {+ const options = { graceMs: 1, cleanupReserveMs: 1, waitForClose: async () => false };++ test('distinguishes signal-send failure', async () => {+ const controller = createOwnedTreeController({+ platform: 'darwin',+ killFn: (_pid, signal) => {+ if (signal !== 0) throw Object.assign(new Error('denied'), { code: 'EPERM' });+ },+ });+ expect(await controller.cleanup(4321, options)).toEqual({+ ok: false,+ reason: 'signal-send-failure',+ });+ });- test('step() throws on a missing name instead of returning a degenerate slice', () => {- expect(() => step('This step does not exist')).toThrow(/has no step named/);+ test('distinguishes a tree surviving SIGKILL', async () => {+ const controller = createOwnedTreeController({+ platform: 'linux',+ killFn: () => {},+ sleepFn: async () => {},+ pollIntervalMs: 1,+ });+ expect(await controller.cleanup(4321, options)).toEqual({+ ok: false,+ reason: 'tree-survived-kill',+ });});- const PACKAGING_STEPS = [- 'Build + sign + notarize DMG/ZIP',- 'Package NSIS installers',- 'Package ${{ matrix.targets }}',- ];+ test('distinguishes a missing close observation', async () => {+ const controller = createOwnedTreeController({+ platform: 'linux',+ killFn: () => {+ throw Object.assign(new Error('gone'), { code: 'ESRCH' });+ },+ });+ expect(await controller.cleanup(4321, options)).toEqual({+ ok: false,+ reason: 'close-not-observed',+ });+ });+});- test.each(PACKAGING_STEPS)('%s routes through the shared retry wrapper', (name) => {- expect(step(name)).toContain('.github/scripts/retry-transient.mjs');+describe('Windows owned-tree adapter', () => {+ test('shell mode uses Bash through the injected Windows spawn boundary', async () => {+ const calls = [];+ const spawnFn = (executable, args, options) => {+ calls.push({ executable, args, options });+ const child = new EventEmitter();+ child.stdout = new PassThrough();+ child.stderr = new PassThrough();+ process.nextTick(() => child.emit('close', 0, null));+ return child;+ };+ const result = await run({+ command: ['printf windows-shell'],+ shell: true,+ platform: 'win32',+ spawnFn,+ });+ expect(result).toMatchObject({ ok: true, attempts: 1 });+ expect(calls).toEqual([+ {+ executable: 'bash',+ args: ['-c', 'printf windows-shell'],+ options: { stdio: ['inherit', 'pipe', 'pipe'], detached: false, windowsHide: true },+ },+ ]);});- test.each(PACKAGING_STEPS)('%s defines its packaging command exactly once', (name) => {- const invocations = step(name)- .split('\n')- .filter((l) => !/^\s*#/.test(l))- .filter((l) => l.includes('pnpm exec electron-builder'));- expect(invocations).toHaveLength(1);- expect(invocations[0]).toMatch(/^\s*PKG_CMD=/);+ test('uses taskkill tree mode then forced tree mode after grace', async () => {+ const calls = [];+ const controller = createOwnedTreeController({+ platform: 'win32',+ taskkillFn: async (args) => {+ calls.push(args);+ return { ok: true };+ },+ sleepFn: () => Promise.resolve(),+ });+ const cleaned = await controller.cleanup(4321, {+ graceMs: 50,+ waitForClose: async () => calls.length > 1,+ });+ expect(cleaned).toEqual({ ok: true, reason: 'clean' });+ expect(calls).toEqual([+ ['/PID', '4321', '/T'],+ ['/PID', '4321', '/T', '/F'],+ ]);});- test.each(PACKAGING_STEPS)('%s runs that command through the wrapper', (name) => {- const body = step(name);- expect(body).toContain('node "$WRAPPER"');- expect(body).toContain('--shell -- "$PKG_CMD"');+ test('fails closed when forced taskkill cannot prove cleanup', async () => {+ const controller = createOwnedTreeController({+ platform: 'win32',+ taskkillFn: async () => ({ ok: false }),+ });+ expect(+ await controller.cleanup(4321, {+ graceMs: 1,+ waitForClose: async () => false,+ }),+ ).toEqual({ ok: false, reason: 'taskkill-failure' });});- test.each(PACKAGING_STEPS)('%s still packages when the wrapper is absent', (name) => {- const body = step(name);- expect(body).toContain('git -C "${GITHUB_WORKSPACE}" fetch --depth=1 origin "$GITHUB_SHA"');- expect(body).toContain('bash -c "$PKG_CMD"');- expect(body).toContain('::warning::Retry wrapper absent');+ test('distinguishes successful taskkill without a close observation', async () => {+ const controller = createOwnedTreeController({+ platform: 'win32',+ taskkillFn: async () => ({ ok: true }),+ });+ expect(+ await controller.cleanup(4321, {+ graceMs: 1,+ waitForClose: async () => false,+ }),+ ).toEqual({ ok: false, reason: 'close-not-observed' });});+});++describe('parseArgs', () => {+ const argv = (...rest) => ['node', 'x', ...rest];- test('the retry allowlist is not duplicated back into the workflow', () => {- expect(desktopRelease).not.toContain('transient_signatures=');+ test('parses the bounded caller contract', () => {+ expect(+ parseArgs(+ argv(+ '--label',+ 'pkg (linux)',+ '--max-attempts',+ '3',+ '--deadline-epoch-ms',+ '2000000000000',+ '--attempt-timeout',+ '15m',+ '--retry-warning',+ 'retry side effect',+ '--shell',+ '--',+ 'pnpm exec electron-builder',+ ),+ ),+ ).toEqual({+ label: 'pkg (linux)',+ maxAttempts: 3,+ deadlineEpochMs: 2_000_000_000_000,+ attemptTimeoutMs: 900_000,+ retryWarning: 'retry side effect',+ shell: true,+ command: ['pnpm exec electron-builder'],+ });});- const preamble = (name) => {- const body = step(name);- const start = body.indexOf('WRAPPER="');- const end = body.indexOf('PKG_CMD=');- expect(start, `${name}: WRAPPER= not found`).toBeGreaterThan(-1);- expect(end, `${name}: PKG_CMD= not found`).toBeGreaterThan(start);- return body.slice(start, end);- };+ test('rejects missing budgets, excess attempts, and malformed shell commands', () => {+ expect(() => parseArgs(argv('--', 'true'))).toThrow(/deadline/);+ expect(() => parseArgs(argv('--deadline-epoch-ms', '2000000000000', '--', 'true'))).toThrow(+ /attempt-timeout/,+ );+ expect(() =>+ parseArgs(+ argv(+ '--max-attempts',+ '4',+ '--deadline-epoch-ms',+ '2000000000000',+ '--attempt-timeout',+ '1m',+ '--',+ 'true',+ ),+ ),+ ).toThrow(/max-attempts/);+ expect(() =>+ parseArgs(+ argv(+ '--shell',+ '--deadline-epoch-ms',+ '2000000000000',+ '--attempt-timeout',+ '1m',+ '--',+ 'a',+ 'b',+ ),+ ),+ ).toThrow(/exactly one/);+ expect(() =>+ parseArgs(+ argv(+ '--unknown',+ 'value',+ '--deadline-epoch-ms',+ '2000000000000',+ '--attempt-timeout',+ '1m',+ '--',+ 'true',+ ),+ ),+ ).toThrow(/unknown flag/);+ expect(() =>+ parseArgs(+ argv(+ '--label',+ '--deadline-epoch-ms',+ '2000000000000',+ '--attempt-timeout',+ '1m',+ '--',+ 'true',+ ),+ ),+ ).toThrow(/--label requires a value/);+ });++ test('accepts an explicit empty retry warning', () => {+ const parsed = parseArgs(+ argv(+ '--deadline-epoch-ms',+ '2000000000000',+ '--attempt-timeout',+ '1m',+ '--retry-warning',+ '',+ '--',+ 'true',+ ),+ );+ expect(parsed.retryWarning).toBe('');+ });++ test('rejects an empty label while permitting only the optional warning to be empty', () => {+ expect(() =>+ parseArgs(+ argv(+ '--label',+ '',+ '--deadline-epoch-ms',+ '2000000000000',+ '--attempt-timeout',+ '1m',+ '--',+ 'true',+ ),+ ),+ ).toThrow(/--label requires a value/);+ });+});- let snippetSeq = 0;- const runUnderErrexit = (snippet) => {- const file = join(scratch, `preamble-${(snippetSeq += 1)}.sh`);+describe('workflow wiring', () => {+ const job = workflowJob;+ const step = workflowStep;+ const jobs = WORKFLOW_JOBS;++ test('download-integrity recovery depends on uncached packaging downloads', () => {+ expect(desktopRelease).not.toMatch(+ /ELECTRON_BUILDER_CACHE|ELECTRON_CACHE|electron_config_cache|electronDownload|[Cc]aches?[\\/]electron|electron(?:-builder)?[\\/][Cc]ache/,+ );+ });+ const materializationBlock = (packageStep) => {+ const start = packageStep.indexOf('WRAPPER="');+ const notice = packageStep.indexOf(+ 'echo "::notice::Retry wrapper materialized from workflow SHA $' + '{GITHUB_WORKFLOW_SHA}."',+ );+ if (start === -1 || notice === -1) throw new Error('materialization block not found');+ const end = packageStep.indexOf('\n', notice);+ return packageStep.slice(start, end);+ };+ const packageSteps = () => jobs.map((row) => step(job(row[0]), row[3]));+ const runMaterialization = ({+ fetchFailures = 0,+ showFailure = false,+ emptyShow = false,+ mvFailure = false,+ mutate,+ } = {}) => {+ const dir = mkdtempSync(join(scratch, 'materialize-'));+ let block = materializationBlock(packageSteps()[0]);+ if (mutate) block = mutate(block);+ const script = join(dir, 'run.sh');writeFileSync(- file,+ script,[- 'git() { case "$*" in *checkout*) return 1 ;; *) return 0 ;; esac; }',- 'GITHUB_WORKSPACE=/nonexistent-workspace',- 'GITHUB_SHA=deadbeefdeadbeefdeadbeefdeadbeefdeadbeef',- snippet,- 'echo REACHED_FALLBACK',+ 'set -euo pipefail',+ 'git() {',+ ' if [[ " $* " == *" fetch "* ]]; then',+ ' n=$(cat "$RUNNER_TEMP/fetch-count")',+ ' n=$((n + 1))',+ ' printf %s "$n" > "$RUNNER_TEMP/fetch-count"',+ ' [[ "$n" -gt "$FETCH_FAILURES" ]]',+ ' return',+ ' fi',+ ' if [[ "$SHOW_FAILURE" == true ]]; then printf partial; return 1; fi',+ ' if [[ "$EMPTY_SHOW" == true ]]; then return 0; fi',+ " printf '%s\\n' '#!/usr/bin/env node'",+ '}',+ 'sleep() { echo "SLEEP:$1"; }',+ ...(mvFailure ? ['mv() { return 1; }'] : []),+ block,+ 'echo REACHED',].join('\n'),);- return spawnSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', file], {+ writeFileSync(join(dir, 'fetch-count'), '0');+ const result = spawnSync('bash', ['--noprofile', '--norc', script], {encoding: 'utf8',+ env: {+ ...process.env,+ RUNNER_TEMP: dir,+ GITHUB_WORKSPACE: dir,+ GITHUB_WORKFLOW_SHA: 'deadbeef',+ FETCH_FAILURES: String(fetchFailures),+ SHOW_FAILURE: String(showFailure),+ EMPTY_SHOW: String(emptyShow),+ },});+ return { ...result, dir, files: readdirSync(dir) };};- test.each(PACKAGING_STEPS)('%s reaches its fallback when sourcing fails', (name) => {- const r = runUnderErrexit(preamble(name));- expect(r.status, `sourcing failure must not abort the step:\n${r.stderr}`).toBe(0);- expect(r.stdout).toContain('REACHED_FALLBACK');+ test.each(jobs)('%s starts an absolute %sm clock before checkout', (name, budget) => {+ const body = job(name);+ expect(body.indexOf('- name: Start packaging deadline')).toBeLessThan(+ body.indexOf('- uses: actions/checkout'),+ );+ expect(step(body, 'Start packaging deadline')).toContain(+ `PACKAGING_BUDGET_MINUTES: "${budget}"`,+ );+ });++ test.each(jobs)(+ '%s always materializes workflow-SHA tooling into RUNNER_TEMP',+ (name, _budget, _timeout, packageName) => {+ const packageStep = step(job(name), packageName);+ expect(packageStep).toContain('GITHUB_WORKFLOW_SHA: $' + '{{ github.workflow_sha }}');+ expect(packageStep).toContain('WRAPPER="$' + '{RUNNER_TEMP}/retry-transient.mjs"');+ expect(packageStep).toContain('fetch --no-tags --depth=1 origin "$GITHUB_WORKFLOW_SHA"');+ expect(packageStep).toContain(+ 'show "$GITHUB_WORKFLOW_SHA:.github/scripts/retry-transient.mjs"',+ );+ expect(packageStep).not.toContain('GITHUB_WORKSPACE}/.github/scripts/retry-transient.mjs');+ expect(packageStep).not.toMatch(/if \[\[ ! -f "\$WRAPPER"/);+ expect(packageStep).not.toContain('packaging without transient retry');+ },+ );++ test('all three workflow-SHA materialization blocks are byte-equivalent', () => {+ const blocks = packageSteps().map(materializationBlock);+ expect(new Set(blocks).size).toBe(1);+ expect(blocks[0]).not.toMatch(/do\s+rm -f "\$WRAPPER_TMP"/);+ expect(blocks[0]).toMatch(+ /\[\[ -s "\$WRAPPER_TMP" \]\] &&\s+mv "\$WRAPPER_TMP" "\$WRAPPER"; then/,+ );+ });++ test('materialization retries transient fetch failure with bounded delays', () => {+ const result = runMaterialization({ fetchFailures: 1 });+ expect(result.status).toBe(0);+ expect(result.stdout).toContain('SLEEP:5');+ expect(result.stdout).not.toContain('SLEEP:10');+ expect(readFileSync(join(result.dir, 'retry-transient.mjs'), 'utf8')).toContain(+ '#!/usr/bin/env node',+ );});- test('the errexit harness actually discriminates (mutation control)', () => {- const unExempt = [- 'WRAPPER="/nonexistent/retry-transient.mjs"',- 'if [[ ! -f "$WRAPPER" ]]; then',- ' git fetch --depth=1 origin "$GITHUB_SHA" >/dev/null 2>&1 &&',- ' git checkout "$GITHUB_SHA" -- .github/scripts/retry-transient.mjs 2>/dev/null',- 'fi',- ].join('\n');- const r = runUnderErrexit(unExempt);- expect(r.status).not.toBe(0);- expect(r.stdout).not.toContain('REACHED_FALLBACK');+ test('materialization fails before mv and cleans temporary output', () => {+ const terminal = runMaterialization({ showFailure: true });+ expect(terminal.status).toBe(1);+ expect(terminal.stdout).toContain('SLEEP:5');+ expect(terminal.stdout).toContain('SLEEP:10');+ expect(terminal.stdout).toContain('::error::Failed to materialize retry wrapper');+ expect(terminal.stdout).not.toContain('REACHED');+ expect(terminal.files.some((file) => file.startsWith('retry-transient.mjs'))).toBe(false);++ const trapped = runMaterialization({ mvFailure: true });+ expect(trapped.status).toBe(1);+ expect(trapped.stdout).toContain('SLEEP:5');+ expect(trapped.stdout).toContain('SLEEP:10');+ expect(trapped.stdout).toContain('::error::Failed to materialize retry wrapper');+ expect(trapped.files.some((file) => file.includes('.tmp.'))).toBe(false);+ });++ test('zero-byte successful show reaches the bounded final error', () => {+ const result = runMaterialization({ emptyShow: true });+ expect(result.status).toBe(1);+ expect(result.stdout).toContain('::error::Failed to materialize retry wrapper');+ expect(result.stdout).not.toContain('REACHED');+ expect(result.files.some((file) => file.startsWith('retry-transient.mjs'))).toBe(false);+ });++ test('the materialization harness discriminates a missing terminal exit', () => {+ const result = runMaterialization({+ fetchFailures: 99,+ mutate: (block) => block.replace(' exit 1', ' true'),+ });+ expect(result.status).toBe(0);+ expect(result.stdout).toContain('REACHED');+ });++ test.each(jobs)(+ '%s compiles once outside retry and passes exact budgets',+ (name, _budget, timeout, packageName) => {+ const body = job(name);+ expect(body.indexOf('- name: Build desktop main/preload/renderer')).toBeLessThan(+ body.indexOf(`- name: ${packageName}`),+ );+ const packageStep = step(body, packageName);+ expect(packageStep).not.toContain('pnpm run build:desktop');+ expect(packageStep).toContain(`--attempt-timeout "${timeout}"`);+ expect(packageStep).toContain('--deadline-epoch-ms "$PACKAGING_DEADLINE_EPOCH_MS"');+ const command = packageStep.split('\n').find((line) => /^\s*PKG_CMD:/.test(line));+ expect(command).toContain("PKG_CMD: 'rm -rf dist-desktop && pnpm exec electron-builder");+ expect(command).toContain('--publish never');+ },+ );++ test('logs duplicate Apple submissions and keeps strict downstream gates outside retry', () => {+ const macPackage = step(job('build-macos'), 'Build + sign + notarize DMG/ZIP');+ expect(macPackage).toContain('duplicate Apple notarization submission');+ expect(macPackage).toContain('at most 2');+ expect(macPackage).toContain('--retry-warning');+ expect(macPackage).not.toMatch(/echo .*duplicate Apple notarization submission/);+ for (const [body, packageName, downstream] of [+ [+ job('build-macos'),+ 'Build + sign + notarize DMG/ZIP',+ [+ 'Attest the signed macOS app',+ 'Smoke the packaged DMG (FR5b)',+ 'Upload macOS release assets for the fan-in publisher',+ ],+ ],+ [+ job('build-windows'),+ 'Package NSIS installers (x64 + arm64, signed)',+ [+ 'Attest signed Windows packages',+ 'Assert the packaged asar carries its dependencies',+ 'Upload Windows release assets for the fan-in publisher',+ ],+ ],+ [+ job('build-linux'),+ 'Package $' + '{{ matrix.targets }}',+ [+ 'Assert the packaged asar carries its dependencies',+ 'Assert app-update.yml + package-type are present and channel-correct',+ 'Upload Linux $' + '{{ matrix.arch }} release assets for the fan-in publisher',+ ],+ ],+ ]) {+ const packageAt = body.indexOf(`- name: ${packageName}`);+ const packageStep = step(body, packageName);+ for (const name of downstream) {+ expect(body.indexOf(`- name: ${name}`)).toBeGreaterThan(packageAt);+ expect(packageStep).not.toContain(name);+ }+ }+ for (const gate of [+ 'Assert the complete cross-platform inventory',+ 'Verify the Release carries the full inventory',+ 'Promote draft release to published',+ ]) {+ expect(desktopRelease).toContain(`- name: ${gate}`);+ }});});diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.ymlindex 5cc22c9a2..0068d5575 100644--- a/.github/workflows/desktop-release.yml+++ b/.github/workflows/desktop-release.yml@@ -497,11 +497,6 @@ jobs:# electron-builder compiles `build/okglass.icon` β `Assets.car` via# `actool`, which only ships with Xcode 26+.runs-on: macos-26- # 75 min budget. A signed build + notary round-trip is ~20-30 min; the- # build step retries once on a transient failure (see below), so- # the worst case is two full signed builds back-to-back. 75 min covers- # that plus cold caches and notary queue spikes. desktop-build.yml has no- # retry and stays at 45.timeout-minutes: 75permissions:# Explicit so a future step needing more fails at review, not with a@@ -514,6 +509,13 @@ jobs:smoke_verdict: ${{ steps.dmg_smoke.outputs.verdict }}smoke_reason: ${{ steps.dmg_smoke.outputs.reason }}steps:+ - name: Start packaging deadline+ env:+ PACKAGING_BUDGET_MINUTES: "65"+ run: |+ set -euo pipefail+ echo "PACKAGING_DEADLINE_EPOCH_MS=$(( $(date +%s) * 1000 + PACKAGING_BUDGET_MINUTES * 60 * 1000 ))" >> "$GITHUB_ENV"+- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:ref: ${{ github.event.client_payload.ref || github.ref }}@@ -790,6 +792,10 @@ jobs:working-directory: packages/desktoprun: node scripts/stage-parcel-watcher.mjs+ - name: Build desktop main/preload/renderer+ working-directory: packages/desktop+ run: pnpm run build:desktop+- name: Build + sign + notarize DMG/ZIP# electron-builder signs with the identity in CSC_KEYCHAIN (pushed# into $GITHUB_ENV by the "Prepare signing keychain" step). CSC_LINK@@ -823,6 +829,8 @@ jobs:# via client_payload (see the Windows job's twin comment).BUILD_VERSION: ${{ steps.channel.outputs.version }}BUILD_CHANNEL: ${{ steps.channel.outputs.channel }}+ GITHUB_WORKFLOW_SHA: ${{ github.workflow_sha }}+ PKG_CMD: 'rm -rf dist-desktop && pnpm exec electron-builder --mac --publish never --config.extraMetadata.version="${BUILD_VERSION}" --config.publish.channel="${BUILD_CHANNEL}"'working-directory: packages/desktoprun: |# extraMetadata.version override: kept as defense-in-depth so the@@ -836,53 +844,32 @@ jobs:# asset's metadata.version; those are baked in at Build time, so# the workspace-wide on-disk override step above is what makes# them match the tag. See PRD-6650.- #- # Packaging is retried on TRANSIENT signing/notarize/download failures- # only, through the shared wrapper. The macOS-specific classes it- # covers, none of which electron-builder will retry itself:- # - codesign's timestamp step calling Apple's TSA (timestamp.apple.com).- # Each `codesign --timestamp` is a single un-retried shell-out, so a- # dropped TSA response aborts the whole build. This is the class that- # stuck v0.30.2 as a draft (run 29445364406).- # - @electron/notarize's notarytool HTTP round-trip to- # appstoreconnect.apple.com timing out (NSURLError -1001 /- # HTTPError(statusCode: nil)), which failed attempt 1 of that same run.- # - a bare EOF fetching the Electron runtime zip during packaging- # (app-builder's DownloadNoRetry path aborts on one dropped connection).- # Re-running the whole invocation is the only lever available. The- # allowlist that gates the retry β and the guarantee that a real- # cert/notary/entitlements/build failure fails LOUD on the first attempt- # rather than being masked β lives in the wrapper.- set -uo pipefail- # The wrapper is CI tooling, not release content, so take it from the- # workflow's own SHA when the release checkout predates it β same- # reasoning as the smoke harness below. This is not a rare path:- # promote-stable tags the stable at the soaked beta's SHA, which is by- # definition older than any fix landed since. Best-effort: if it cannot- # be obtained, package WITHOUT retry rather than block the release.- WRAPPER="${GITHUB_WORKSPACE}/.github/scripts/retry-transient.mjs"- if [[ ! -f "$WRAPPER" ]]; then- # The fetch/checkout pair sits in an `if` CONDITION deliberately.- # Steps run under errexit, and bash exempts only the NON-final- # commands of an `&&` list β so a bare failing checkout would exit- # the shell before reaching the fallback below, defeating the- # best-effort behavior this exists for. A condition is exempt.- if git -C "${GITHUB_WORKSPACE}" fetch --depth=1 origin "$GITHUB_SHA" >/dev/null 2>&1 &&- git -C "${GITHUB_WORKSPACE}" checkout "$GITHUB_SHA" -- .github/scripts/retry-transient.mjs 2>/dev/null; then- echo "::notice::Retry wrapper taken from ${GITHUB_SHA} rather than the release ref."+ set -euo pipefail+ WRAPPER="${RUNNER_TEMP}/retry-transient.mjs"+ WRAPPER_TMP="${WRAPPER}.tmp.$$"+ trap 'rm -f "$WRAPPER_TMP"' EXIT+ materialized=false+ for materialize_attempt in 1 2 3; do+ if git -C "${GITHUB_WORKSPACE}" fetch --no-tags --depth=1 origin "$GITHUB_WORKFLOW_SHA" &&+ git -C "${GITHUB_WORKSPACE}" show "$GITHUB_WORKFLOW_SHA:.github/scripts/retry-transient.mjs" > "$WRAPPER_TMP" &&+ [[ -s "$WRAPPER_TMP" ]] &&+ mv "$WRAPPER_TMP" "$WRAPPER"; then+ materialized=true+ breakfi+ rm -f "$WRAPPER_TMP"+ if [[ "$materialize_attempt" -lt 3 ]]; then+ materialize_delay=$((materialize_attempt * 5))+ echo "::warning::Retry wrapper materialization attempt ${materialize_attempt}/3 failed; retrying in ${materialize_delay}s."+ sleep "$materialize_delay"+ fi+ done+ if [[ "$materialized" != true ]]; then+ echo "::error::Failed to materialize retry wrapper from workflow SHA ${GITHUB_WORKFLOW_SHA} after 3 attempts."+ exit 1fi- # Defined once, single-quoted ON PURPOSE: it is handed to the wrapper as- # a literal string and expanded by the inner shell, which inherits this- # step's env. Expanding here would bake attempt 1's values into retries.- # shellcheck disable=SC2016- PKG_CMD='pnpm run build:desktop && pnpm exec electron-builder --mac --publish never --config.extraMetadata.version="${BUILD_VERSION}" --config.publish.channel="${BUILD_CHANNEL}"'- if [[ -f "$WRAPPER" ]]; then- node "$WRAPPER" --label "electron-builder (macOS)" --shell -- "$PKG_CMD"- else- echo "::warning::Retry wrapper absent at this ref; packaging without transient retry."- bash -c "$PKG_CMD"- fi+ echo "::notice::Retry wrapper materialized from workflow SHA ${GITHUB_WORKFLOW_SHA}."+ node "$WRAPPER" --label "electron-builder (macOS)" --max-attempts 3 --deadline-epoch-ms "$PACKAGING_DEADLINE_EPOCH_MS" --attempt-timeout "30m" --retry-warning "A macOS packaging retry can create a duplicate Apple notarization submission; up to 3 attempts bound this to at most 2 duplicates." --shell -- "$PKG_CMD"# Placed HERE, not at the end of the job (where desktop-build.yml keeps# its twin): release-cascade-shape.test.mjs forbids `if: always()` on any@@ -1032,6 +1019,13 @@ jobs:run:shell: bashsteps:+ - name: Start packaging deadline+ env:+ PACKAGING_BUDGET_MINUTES: "35"+ run: |+ set -euo pipefail+ echo "PACKAGING_DEADLINE_EPOCH_MS=$(( $(date +%s) * 1000 + PACKAGING_BUDGET_MINUTES * 60 * 1000 ))" >> "$GITHUB_ENV"+- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:ref: ${{ github.event.client_payload.ref || github.ref }}@@ -1134,6 +1128,8 @@ jobs:AZURE_SIGNING_PUBLISHER: ${{ vars.AZURE_SIGNING_PUBLISHER }}BUILD_VERSION: ${{ needs.prepare.outputs.version }}BUILD_CHANNEL: ${{ needs.prepare.outputs.channel }}+ GITHUB_WORKFLOW_SHA: ${{ github.workflow_sha }}+ PKG_CMD: 'rm -rf dist-desktop && pnpm exec electron-builder --win --publish never --config.extraMetadata.version="${BUILD_VERSION}" --config.publish.channel="${BUILD_CHANNEL}" --config.win.azureSignOptions.endpoint="${AZURE_SIGNING_ENDPOINT}" --config.win.azureSignOptions.codeSigningAccountName="${AZURE_SIGNING_ACCOUNT}" --config.win.azureSignOptions.certificateProfileName="${AZURE_SIGNING_PROFILE}" --config.win.azureSignOptions.publisherName="${AZURE_SIGNING_PUBLISHER}"'run: |set -euo pipefailmissing=()@@ -1149,31 +1145,31 @@ jobs:exit 1fiecho "::notice::Azure Trusted Signing credentials present β building SIGNED installers."- # Signing here is a network round-trip to the Azure Trusted Signing- # endpoint, so this invocation carries the same transient exposure the- # macOS job does. The wrapper still fails loud on a real signing- # rejection or build error β only allowlisted infra flakes retry.- # Wrapper sourcing + single-quoting rationale: see the macOS step.- WRAPPER="${GITHUB_WORKSPACE}/.github/scripts/retry-transient.mjs"- if [[ ! -f "$WRAPPER" ]]; then- # The fetch/checkout pair sits in an `if` CONDITION deliberately.- # Steps run under errexit, and bash exempts only the NON-final- # commands of an `&&` list β so a bare failing checkout would exit- # the shell before reaching the fallback below, defeating the- # best-effort behavior this exists for. A condition is exempt.- if git -C "${GITHUB_WORKSPACE}" fetch --depth=1 origin "$GITHUB_SHA" >/dev/null 2>&1 &&- git -C "${GITHUB_WORKSPACE}" checkout "$GITHUB_SHA" -- .github/scripts/retry-transient.mjs 2>/dev/null; then- echo "::notice::Retry wrapper taken from ${GITHUB_SHA} rather than the release ref."+ WRAPPER="${RUNNER_TEMP}/retry-transient.mjs"+ WRAPPER_TMP="${WRAPPER}.tmp.$$"+ trap 'rm -f "$WRAPPER_TMP"' EXIT+ materialized=false+ for materialize_attempt in 1 2 3; do+ if git -C "${GITHUB_WORKSPACE}" fetch --no-tags --depth=1 origin "$GITHUB_WORKFLOW_SHA" &&+ git -C "${GITHUB_WORKSPACE}" show "$GITHUB_WORKFLOW_SHA:.github/scripts/retry-transient.mjs" > "$WRAPPER_TMP" &&+ [[ -s "$WRAPPER_TMP" ]] &&+ mv "$WRAPPER_TMP" "$WRAPPER"; then+ materialized=true+ breakfi+ rm -f "$WRAPPER_TMP"+ if [[ "$materialize_attempt" -lt 3 ]]; then+ materialize_delay=$((materialize_attempt * 5))+ echo "::warning::Retry wrapper materialization attempt ${materialize_attempt}/3 failed; retrying in ${materialize_delay}s."+ sleep "$materialize_delay"+ fi+ done+ if [[ "$materialized" != true ]]; then+ echo "::error::Failed to materialize retry wrapper from workflow SHA ${GITHUB_WORKFLOW_SHA} after 3 attempts."+ exit 1fi- # shellcheck disable=SC2016- PKG_CMD='pnpm exec electron-builder --win --publish never --config.extraMetadata.version="${BUILD_VERSION}" --config.publish.channel="${BUILD_CHANNEL}" --config.win.azureSignOptions.endpoint="${AZURE_SIGNING_ENDPOINT}" --config.win.azureSignOptions.codeSigningAccountName="${AZURE_SIGNING_ACCOUNT}" --config.win.azureSignOptions.certificateProfileName="${AZURE_SIGNING_PROFILE}" --config.win.azureSignOptions.publisherName="${AZURE_SIGNING_PUBLISHER}"'- if [[ -f "$WRAPPER" ]]; then- node "$WRAPPER" --label "electron-builder (Windows)" --shell -- "$PKG_CMD"- else- echo "::warning::Retry wrapper absent at this ref; packaging without transient retry."- bash -c "$PKG_CMD"- fi+ echo "::notice::Retry wrapper materialized from workflow SHA ${GITHUB_WORKFLOW_SHA}."+ node "$WRAPPER" --label "electron-builder (Windows)" --max-attempts 3 --deadline-epoch-ms "$PACKAGING_DEADLINE_EPOCH_MS" --attempt-timeout "15m" --shell -- "$PKG_CMD"# This workflow always executes main's YAML (repository_dispatch), while# the checkout above follows `client_payload[ref]` β typically a tag@@ -1354,6 +1350,13 @@ jobs:targets: deb:arm64 rpm:arm64unpacked_dir: linux-arm64-unpackedsteps:+ - name: Start packaging deadline+ env:+ PACKAGING_BUDGET_MINUTES: "35"+ run: |+ set -euo pipefail+ echo "PACKAGING_DEADLINE_EPOCH_MS=$(( $(date +%s) * 1000 + PACKAGING_BUDGET_MINUTES * 60 * 1000 ))" >> "$GITHUB_ENV"+- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:ref: ${{ github.event.client_payload.ref || github.ref }}@@ -1482,33 +1485,35 @@ jobs:working-directory: packages/desktopenv:LINUX_TARGETS: ${{ matrix.targets }}+ GITHUB_WORKFLOW_SHA: ${{ github.workflow_sha }}+ PKG_CMD: 'rm -rf dist-desktop && pnpm exec electron-builder --linux ${LINUX_TARGETS} --publish never --config electron-builder.release-linux.yml'run: |- # electron-builder downloads its fpm packaging backend on first use,- # and a dropped connection there aborts the whole build. That is what- # blocked the v0.52.2 and v0.52.3 cuts, on a step that had no retry- # while the macOS job had carried one for months.- set -uo pipefail- # Wrapper sourcing + single-quoting rationale: see the macOS step.- WRAPPER="${GITHUB_WORKSPACE}/.github/scripts/retry-transient.mjs"- if [[ ! -f "$WRAPPER" ]]; then- # The fetch/checkout pair sits in an `if` CONDITION deliberately.- # Steps run under errexit, and bash exempts only the NON-final- # commands of an `&&` list β so a bare failing checkout would exit- # the shell before reaching the fallback below, defeating the- # best-effort behavior this exists for. A condition is exempt.- if git -C "${GITHUB_WORKSPACE}" fetch --depth=1 origin "$GITHUB_SHA" >/dev/null 2>&1 &&- git -C "${GITHUB_WORKSPACE}" checkout "$GITHUB_SHA" -- .github/scripts/retry-transient.mjs 2>/dev/null; then- echo "::notice::Retry wrapper taken from ${GITHUB_SHA} rather than the release ref."+ set -euo pipefail+ WRAPPER="${RUNNER_TEMP}/retry-transient.mjs"+ WRAPPER_TMP="${WRAPPER}.tmp.$$"+ trap 'rm -f "$WRAPPER_TMP"' EXIT+ materialized=false+ for materialize_attempt in 1 2 3; do+ if git -C "${GITHUB_WORKSPACE}" fetch --no-tags --depth=1 origin "$GITHUB_WORKFLOW_SHA" &&+ git -C "${GITHUB_WORKSPACE}" show "$GITHUB_WORKFLOW_SHA:.github/scripts/retry-transient.mjs" > "$WRAPPER_TMP" &&+ [[ -s "$WRAPPER_TMP" ]] &&+ mv "$WRAPPER_TMP" "$WRAPPER"; then+ materialized=true+ breakfi+ rm -f "$WRAPPER_TMP"+ if [[ "$materialize_attempt" -lt 3 ]]; then+ materialize_delay=$((materialize_attempt * 5))+ echo "::warning::Retry wrapper materialization attempt ${materialize_attempt}/3 failed; retrying in ${materialize_delay}s."+ sleep "$materialize_delay"+ fi+ done+ if [[ "$materialized" != true ]]; then+ echo "::error::Failed to materialize retry wrapper from workflow SHA ${GITHUB_WORKFLOW_SHA} after 3 attempts."+ exit 1fi- # shellcheck disable=SC2016- PKG_CMD='pnpm exec electron-builder --linux ${LINUX_TARGETS} --publish never --config electron-builder.release-linux.yml'- if [[ -f "$WRAPPER" ]]; then- node "$WRAPPER" --label "electron-builder (Linux ${LINUX_TARGETS})" --shell -- "$PKG_CMD"- else- echo "::warning::Retry wrapper absent at this ref; packaging without transient retry."- bash -c "$PKG_CMD"- fi+ echo "::notice::Retry wrapper materialized from workflow SHA ${GITHUB_WORKFLOW_SHA}."+ node "$WRAPPER" --label "electron-builder (Linux ${LINUX_TARGETS})" --max-attempts 3 --deadline-epoch-ms "$PACKAGING_DEADLINE_EPOCH_MS" --attempt-timeout "15m" --shell -- "$PKG_CMD"# Fail before staging an installer that cannot start (see the Windows# step for the full rationale). This is the control that would havediff --git a/packages/desktop/scripts/afterPack.mjs b/packages/desktop/scripts/afterPack.mjsindex b718bea4b..a25797503 100644--- a/packages/desktop/scripts/afterPack.mjs+++ b/packages/desktop/scripts/afterPack.mjs@@ -3,29 +3,11 @@ import { chmodSync, copyFileSync, existsSync, mkdirSync, writeFileSync } from 'nimport { dirname, join } from 'node:path';import { FuseV1Options, FuseVersion, flipFuses } from '@electron/fuses';import { ensureNodePtySpawnHelperExecutable } from './ensure-node-pty-exec.mjs';+import { createFuseFailure } from './packaging-diagnostics.mjs';import { resolveElectronBinary } from './resolve-electron-binary.mjs';import { targetFuses } from './target-fuses.mjs';-export default async function afterPack(context) {- const { appOutDir, packager, electronPlatformName } = context;-- if (appOutDir.endsWith('-temp')) {- console.log(- `[afterPack] skipping per-arch temp "${appOutDir}" β fuses flip on the merged universal app`,- );- return;- }-- const appName = packager.appInfo.productFilename;- const electronBinary = resolveElectronBinary(electronPlatformName, appOutDir, packager);-- if (!existsSync(electronBinary)) {- throw new Error(- `[afterPack] Electron binary not found at ${electronBinary}. ` +- `Expected electron-builder to have packed the app before afterPack ran.`,- );- }-+async function flipElectronFuses(electronBinary, electronPlatformName) {console.log(`[afterPack] flipping fuses on ${electronBinary}`);for (const [optIndex, value] of Object.entries(targetFuses)) {const name = FuseV1Options[Number(optIndex)];@@ -39,7 +21,7 @@ export default async function afterPack(context) {...targetFuses,});} catch (err) {- throw new Error(+ throw createFuseFailure(`[afterPack] fuse flip failed on ${electronBinary}: ${err instanceof Error ? err.message : String(err)}`,@@ -48,6 +30,29 @@ export default async function afterPack(context) {}console.log('[afterPack] fuses flipped successfully; electron-builder will re-sign next');+}++export default async function afterPack(context) {+ const { appOutDir, packager, electronPlatformName } = context;++ if (appOutDir.endsWith('-temp')) {+ console.log(+ `[afterPack] skipping per-arch temp "${appOutDir}" β fuses flip on the merged universal app`,+ );+ return;+ }++ const appName = packager.appInfo.productFilename;+ const electronBinary = resolveElectronBinary(electronPlatformName, appOutDir, packager);++ if (!existsSync(electronBinary)) {+ throw new Error(+ `[afterPack] Electron binary not found at ${electronBinary}. ` ++ `Expected electron-builder to have packed the app before afterPack ran.`,+ );+ }++ await flipElectronFuses(electronBinary, electronPlatformName);if (electronPlatformName !== 'darwin') {console.log(diff --git a/packages/desktop/scripts/afterSign.mjs b/packages/desktop/scripts/afterSign.mjsindex fb3a4297b..549e821a1 100644--- a/packages/desktop/scripts/afterSign.mjs+++ b/packages/desktop/scripts/afterSign.mjs@@ -4,6 +4,7 @@ import { existsSync } from 'node:fs';import { join } from 'node:path';import { FuseV1Options, getCurrentFuseWire } from '@electron/fuses';import { notarize } from '@electron/notarize';+import { createFuseFailure } from './packaging-diagnostics.mjs';import { resolveElectronBinary } from './resolve-electron-binary.mjs';import { expectedFuseState, fuseStateName, targetFuses } from './target-fuses.mjs';@@ -114,8 +115,8 @@ async function verifyFuses(electronBinary, expected) {try {wire = await getCurrentFuseWire(electronBinary);} catch (err) {- throw new Error(- `[afterSign] getCurrentFuseWire failed on ${electronBinary}: ${+ throw createFuseFailure(+ `[afterSign] fuse verification read failed on ${electronBinary}: ${err instanceof Error ? err.message : String(err)}`,{ cause: err },@@ -134,8 +135,8 @@ async function verifyFuses(electronBinary, expected) {}}if (mismatches.length > 0) {- throw new Error(- `[afterSign] Fuse verification failed (D17 paranoid check):\n ${mismatches.join('\n ')}`,+ throw createFuseFailure(+ `[afterSign] fuse verification failed (D17 paranoid check):\n ${mismatches.join('\n ')}`,);}console.log('[afterSign] fuse verification passed β all 6 fuses match targetFuses');diff --git a/packages/desktop/scripts/packaging-diagnostics.mjs b/packages/desktop/scripts/packaging-diagnostics.mjsnew file mode 100644index 000000000..9a9a04fbd--- /dev/null+++ b/packages/desktop/scripts/packaging-diagnostics.mjs@@ -0,0 +1,6 @@+#!/usr/bin/env node+export const FUSE_FAILURE_MARKER = 'OK_PACKAGING_FUSE_FAILURE';++export function createFuseFailure(message, options) {+ return new Error(`${message} [${FUSE_FAILURE_MARKER}]`, options);+}