Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Fetch the Windows desktop lanes' inputs in setup, not in tests (#5174)
Nick Gomez0d3e31390934
diff --git a/.github/scripts/retry-transient.mjs b/.github/scripts/retry-transient.mjsindex 48e4cee37..369353fae 100644--- a/.github/scripts/retry-transient.mjs+++ b/.github/scripts/retry-transient.mjs@@ -1,6 +1,7 @@#!/usr/bin/env nodeimport { spawn } from 'node:child_process';+import { writeFileSync } from 'node:fs';import { StringDecoder } from 'node:string_decoder';import { pathToFileURL } from 'node:url';@@ -16,6 +17,7 @@ export const STOP_OUTCOMES = Object.freeze(['spawn-failure','cleanup-failure',]);+export const RETRY_ON_STOP_OUTCOMES = Object.freeze(['ineligible']);const MAX_CAPTURE_BYTES = 16_384;const MIN_PROJECTED_ATTEMPT_MS = 1_000;@@ -23,7 +25,7 @@ const DEFAULT_ATTEMPT_TIMEOUT_MS = 15 * 60 * 1000;const DEFAULT_CLEANUP_GRACE_MS = 5_000;const DEFAULT_CLEANUP_RESERVE_MS = 15_000;const DEFAULT_POLL_INTERVAL_MS = 25;-const STOP_OUTCOME_SET = new Set(STOP_OUTCOMES);+const STOP_OUTCOME_SET = new Set([...STOP_OUTCOMES, ...RETRY_ON_STOP_OUTCOMES]);const TRANSIENT_HTTP_STATUSES = new Set([408, 429, 500, 502, 503, 504, 521, 522, 524]);const TERMINAL_HTTP_STATUSES = new Set([400, 401, 404, 413, 422]);const NETWORK_CODES = [@@ -95,6 +97,15 @@ const TRANSIENT_RULES = [['abuse-limit', /\babuse detection mechanism\b/i],['submitted-too-fast', /\bwas submitted too quickly\b/i],];+const CONNECTION_TRANSIENT_RULE_IDS = Object.freeze([+ 'server-connect',+ 'socket-hangup',+ 'tls-disconnect',+ 'unexpected-eof',+ 'network-lost',+ 'request-timeout',+]);+const RULE_ID_PATTERN = /^[a-z][a-z0-9-]*$/;function appendUtf8Tail(current, chunk, limit) {const combined = current.length === 0 ? chunk : Buffer.concat([current, chunk]);@@ -164,10 +175,18 @@ function collectPhraseEvidence(evidence, text) {for (const [id, pattern] of TRANSIENT_RULES) {if (pattern.test(text)) evidence.transientRules.add(id);}+ for (const [id, pattern] of evidence.extraTransientRules) {+ if (pattern.test(text)) evidence.transientRules.add(id);+ }}export class FailureEvidence {- constructor({ maxTailBytes = MAX_CAPTURE_BYTES, nowFn = Date.now } = {}) {+ constructor({+ maxTailBytes = MAX_CAPTURE_BYTES,+ nowFn = Date.now,+ extraTransientRules = [],+ } = {}) {+ this.extraTransientRules = extraTransientRules;this.httpStatuses = new Set();this.errorCodes = new Set();this.terminalRules = new Set();@@ -278,6 +297,25 @@ function matchedReason(evidence, classification) {return 'diagnostic:unknown';}+function retryOnReason(evidence, retryOn) {+ if (retryOn.http5xx) {+ const status = [...evidence.httpStatuses]+ .sort((left, right) => left - right)+ .find((candidate) => candidate >= 500 && candidate <= 599);+ if (status !== undefined) return `http:${status}`;+ }+ if (retryOn.connection) {+ const errorCode = firstSorted(evidence.errorCodes);+ if (errorCode) return `code:${errorCode}`;+ const connectionRule = CONNECTION_TRANSIENT_RULE_IDS.find((id) =>+ evidence.transientRules.has(id),+ );+ if (connectionRule) return `rule:${connectionRule}`;+ }+ const callerRule = retryOn.rules.find((id) => evidence.transientRules.has(id));+ return callerRule ? `rule:${callerRule}` : undefined;+}+function redactDiagnosticTail(tail) {const withoutPem = tail.replace(/(?:-----BEGIN [^-\r\n]+-----[\s\S]*?(?:-----END [^-\r\n]+-----|$)|(?:[A-Z0-9+/=]{20,}\r?\n)+-----END [^-\r\n]+-----)/gi,@@ -523,8 +561,9 @@ async function runAttempt({cleanupGraceMs,treeController,nowFn,+ transientRules = [],}) {- const evidence = new FailureEvidence({ nowFn });+ const evidence = new FailureEvidence({ nowFn, extraTransientRules: transientRules });const invocation = commandInvocation(command, shell, platform);let child;try {@@ -707,6 +746,8 @@ export async function runWithRetry({platform = process.platform,treeController,attemptRunner = runAttempt,+ retryOn,+ transientRules = [],} = {}) {if (!Number.isInteger(maxAttempts) || maxAttempts < 1 || maxAttempts > DEFAULT_MAX_ATTEMPTS) {throw new Error(`maxAttempts must be an integer from 1 to ${DEFAULT_MAX_ATTEMPTS}`);@@ -777,6 +818,7 @@ export async function runWithRetry({cleanupGraceMs,treeController: ownedTree,nowFn,+ transientRules,});state.lastAttemptDurationMs = Math.max(0, nowFn() - attemptStartedAt);log('::endgroup::');@@ -799,8 +841,8 @@ export async function runWithRetry({};}- const classification = classifyEvidence(result.evidence);- const reasonId = matchedReason(result.evidence, classification);+ let classification = classifyEvidence(result.evidence);+ let reasonId = matchedReason(result.evidence, classification);state.lastFailureClassification = classification;if (classification === 'terminal') {recordDecision('error', 'stop', reasonId, result, { outcome: 'terminal' });@@ -814,7 +856,27 @@ export async function runWithRetry({result,);}- if (classification === 'unknown') {+ if (retryOn) {+ const eligibleReason = retryOnReason(result.evidence, retryOn);+ if (eligibleReason === undefined) {+ recordDecision('error', 'stop', reasonId, result, {+ outcome: 'ineligible',+ classification,+ });+ return finishFailure(+ {+ ok: false,+ reason: 'ineligible',+ attempts: state.attemptsStarted,+ code: result.code,+ },+ result,+ );+ }+ classification = 'transient';+ reasonId = eligibleReason;+ state.lastFailureClassification = classification;+ } else if (classification === 'unknown') {if (state.unknownRetryUsed) {recordDecision('error', 'stop', reasonId, result, {outcome: 'unknown-exhausted',@@ -922,6 +984,54 @@ function flagValue(flags, name, required = false) {return flags[index + 1];}+function flagValues(flags, name) {+ const values = [];+ for (let index = 0; index < flags.length; index += 1) {+ if (flags[index] === '--shell') continue;+ if (flags[index] === name) values.push(flags[index + 1]);+ index += 1;+ }+ return values;+}++const BUILT_IN_RULE_IDS = new Set([+ ...TERMINAL_RULES.map(([id]) => id),+ ...TRANSIENT_RULES.map(([id]) => id),+]);++function parseTransientRule(raw) {+ const separator = raw.indexOf('=');+ const id = separator === -1 ? '' : raw.slice(0, separator);+ if (!RULE_ID_PATTERN.test(id)) {+ throw new Error('--transient-rule must be <id>=<regex> with a lowercase id');+ }+ if (BUILT_IN_RULE_IDS.has(id)) throw new Error(`--transient-rule ${id} shadows a built-in rule`);+ const source = raw.slice(separator + 1);+ if (source === '') throw new Error(`--transient-rule ${id} has an empty pattern`);+ try {+ return [id, new RegExp(source)];+ } catch (error) {+ throw new Error(`--transient-rule ${id} is not a valid pattern: ${error.message}`);+ }+}++function parseRetryOn(raw, transientRules) {+ const retryOn = { http5xx: false, connection: false, rules: [] };+ const callerRuleIds = new Set(transientRules.map(([id]) => id));+ for (const token of raw.split(',').map((part) => part.trim())) {+ if (token === 'http-5xx') retryOn.http5xx = true;+ else if (token === 'connection') retryOn.connection = true;+ else if (token.startsWith('rule:') && callerRuleIds.has(token.slice('rule:'.length))) {+ retryOn.rules.push(token.slice('rule:'.length));+ } else if (token.startsWith('rule:')) {+ throw new Error(`--retry-on ${token} names no --transient-rule`);+ } else {+ throw new Error(`--retry-on has an unknown token: ${token || '(empty)'}`);+ }+ }+ return retryOn;+}+export function parseArgs(argv) {const rest = argv.slice(2);const separator = rest.indexOf('--');@@ -936,6 +1046,9 @@ export function parseArgs(argv) {'--attempt-timeout','--retry-warning','--shell',+ '--retry-on',+ '--transient-rule',+ '--result-file',]);for (let index = 0; index < flags.length; index += 1) {const flag = flags[index];@@ -966,7 +1079,7 @@ export function parseArgs(argv) {flagValue(flags, '--attempt-timeout', true),'--attempt-timeout',);- return {+ const parsed = {label: flagValue(flags, '--label') ?? 'command',maxAttempts,deadlineEpochMs,@@ -975,11 +1088,28 @@ export function parseArgs(argv) {shell,command,};+ const transientRules = flagValues(flags, '--transient-rule').map(parseTransientRule);+ if (new Set(transientRules.map(([id]) => id)).size !== transientRules.length) {+ throw new Error('--transient-rule ids must be unique');+ }+ const retryOn = flagValue(flags, '--retry-on');+ if (transientRules.length > 0 && retryOn === undefined) {+ throw new Error(+ '--transient-rule requires --retry-on, which decides whether a rule match is retried',+ );+ }+ if (transientRules.length > 0) parsed.transientRules = transientRules;+ if (retryOn !== undefined) parsed.retryOn = parseRetryOn(retryOn, transientRules);+ const resultFile = flagValue(flags, '--result-file');+ if (resultFile !== undefined) parsed.resultFile = resultFile;+ return parsed;}if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {try {- const result = await runWithRetry(parseArgs(process.argv));+ const options = parseArgs(process.argv);+ const result = await runWithRetry(options);+ if (options.resultFile) writeFileSync(options.resultFile, `${JSON.stringify(result)}\n`);if (result.reason === 'signal' && result.signal) {process.kill(process.pid, result.signal);} else {diff --git a/.github/scripts/retry-transient.test.mjs b/.github/scripts/retry-transient.test.mjsindex 5a237a8a4..cd062c432 100644--- a/.github/scripts/retry-transient.test.mjs+++ b/.github/scripts/retry-transient.test.mjs@@ -19,7 +19,9 @@ import {DEFAULT_MAX_ATTEMPTS,FailureEvidence,parseArgs,+ RETRY_ON_STOP_OUTCOMES,runWithRetry,+ STOP_OUTCOMES,} from './retry-transient.mjs';const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');@@ -679,6 +681,227 @@ describe('retry state machine', () => {});});+describe('opt-in fail-closed eligibility', () => {+ const undiciPausedParser = [+ 'undici-paused-parser',+ /assert\(!this\.paused\)[\s\S]{0,512}?\bParser\.finish\b/,+ ];+ const pausedParserCrash = [+ '[prepare-platform-natives] @napi-rs/[email protected] missing — fetching',+ 'node:internal/assert/utils:77',+ ' throw err;',+ ' ^',+ '',+ 'AssertionError [ERR_ASSERTION]: The expression evaluated to a falsy value:',+ '',+ ' assert(!this.paused)',+ '',+ ' at Parser.finish (node:internal/deps/undici/undici:7380:9)',+ ' at TLSSocket.onHttpSocketEnd (node:internal/deps/undici/undici:7819:34)',+ ].join('\n');+ const failClosed = { retryOn: { http5xx: true, connection: true, rules: [] } };+ const withPausedParserRule = {+ retryOn: { http5xx: true, connection: true, rules: ['undici-paused-parser'] },+ transientRules: [undiciPausedParser],+ };+ const failOnceThen = (name, text) => {+ const count = join(scratch, name);+ writeFileSync(count, '0');+ return nodeCmd(+ `const fs=require('fs');const p=${JSON.stringify(count)};const n=+fs.readFileSync(p,'utf8')+1;fs.writeFileSync(p,String(n));if(n===1){console.error(${JSON.stringify(text)});process.exit(1)}`,+ );+ };+ const failAlways = (text, exitCode = 1) =>+ nodeCmd(`console.error(${JSON.stringify(text)});process.exit(${exitCode})`);++ test('the default policy is unchanged when no opt-in flag is given', async () => {+ const unknown = await run({+ command: failOnceThen('default-unknown', 'unrecognized packager failure'),+ });+ expect(unknown).toMatchObject({ ok: true, attempts: 2, recoveredFrom: 'unknown' });+ expect(unknown.log).toContain(+ 'UNKNOWN_CLASSIFICATION_RETRY allowance=invocation-wide-single-use',+ );+ const rateLimited = await run({+ command: failOnceThen('default-429', 'HTTPError: Response code 429 (Too Many Requests)'),+ });+ expect(rateLimited).toMatchObject({ ok: true, attempts: 2, recoveredFrom: 'transient' });+ const crash = await run({ command: failOnceThen('default-crash', pausedParserCrash) });+ expect(crash).toMatchObject({ ok: true, attempts: 2, recoveredFrom: 'unknown' });+ expect(unknown.log).not.toContain('ineligible');+ });++ test('the ineligible outcome is registered apart from the default outcomes', () => {+ expect(RETRY_ON_STOP_OUTCOMES).toEqual(['ineligible']);+ expect(STOP_OUTCOMES).not.toContain('ineligible');+ });++ test.each([+ ['http-500', 'HTTPError: Response code 500 (Internal Server Error)', 'http:500'],+ ['http-501', 'fetch https://registry.example/x.tgz → HTTP 501 Not Implemented', 'http:501'],+ [+ 'connection',+ 'TypeError: fetch failed\nFETCH 1: connection to host errored - read ECONNRESET',+ 'code:ECONNRESET',+ ],+ ['socket', 'Error: socket hang up', 'rule:socket-hangup'],+ ])('retries %s evidence and recovers', async (name, text, reason) => {+ const result = await run({ command: failOnceThen(`fail-closed-${name}`, text), ...failClosed });+ expect(result).toMatchObject({ ok: true, attempts: 2, recoveredFrom: 'transient' });+ expect(result.log).toContain(+ `decision=retry reason=${reason} classification=transient attempt=1/3`,+ );+ });++ test('a persistent eligible failure stops as transient-exhausted', async () => {+ const result = await run({+ command: failAlways('HTTPError: Response code 503 (Service Unavailable)'),+ ...failClosed,+ });+ expect(result).toMatchObject({ ok: false, reason: 'transient-exhausted', attempts: 3 });+ });++ test.each([+ [+ 'an unknown failure',+ 'unrecognized packager failure',+ 'reason=diagnostic:unknown outcome=ineligible classification=unknown',+ ],+ [+ 'a 429 outside the policy',+ 'HTTPError: Response code 429 (Too Many Requests)',+ 'reason=http:429 outcome=ineligible classification=transient',+ ],+ [+ 'a checksum mismatch',+ 'Error: Generated checksum for "electron-v43.4.0-win32-x64.zip" did not match expected checksum.',+ 'reason=diagnostic:unknown outcome=ineligible classification=unknown',+ ],+ [+ 'an unrelated crash',+ 'Error: planted unrelated crash\n at Object.<anonymous> ([eval]:1:7)',+ 'reason=diagnostic:unknown outcome=ineligible classification=unknown',+ ],+ [+ 'the libuv abort line alone',+ 'Assertion failed: !(handle->flags & UV_HANDLE_CLOSING), file src\\win\\async.c, line 94',+ 'reason=diagnostic:unknown outcome=ineligible classification=unknown',+ ],+ ])('stops %s at once, without a retry', async (_name, text, decision) => {+ const result = await run({ command: failAlways(text, 127), ...withPausedParserRule });+ expect(result).toMatchObject({ ok: false, reason: 'ineligible', attempts: 1 });+ expect(result.log).toContain(`decision=stop ${decision}`);+ });++ test.each([+ ['an HTTP 404', 'HTTPError: Response code 404 (Not Found)', 'reason=http:404'],+ [+ 'an integrity mismatch',+ '[prepare-platform-natives] @napi-rs/[email protected]: sha512 hash mismatch, expected sha512-AAA, got sha512-BBB',+ 'reason=rule:download-integrity',+ ],+ ])('keeps %s terminal', async (_name, text, decision) => {+ const result = await run({ command: failAlways(text), ...withPausedParserRule });+ expect(result).toMatchObject({ ok: false, reason: 'terminal', attempts: 1 });+ expect(result.log).toContain(`decision=stop ${decision} outcome=terminal`);+ });++ test('retries the paused-parser crash only through a listed caller rule', async () => {+ const listed = await run({+ command: failOnceThen('rule-listed', pausedParserCrash),+ ...withPausedParserRule,+ });+ expect(listed).toMatchObject({ ok: true, attempts: 2, recoveredFrom: 'transient' });+ expect(listed.log).toContain(+ 'decision=retry reason=rule:undici-paused-parser classification=transient',+ );+ const defined = await run({+ command: failAlways(pausedParserCrash),+ retryOn: { http5xx: true, connection: true, rules: [] },+ transientRules: [undiciPausedParser],+ });+ expect(defined).toMatchObject({ ok: false, reason: 'ineligible', attempts: 1 });+ const undefinedRule = await run({ command: failAlways(pausedParserCrash), ...failClosed });+ expect(undefinedRule).toMatchObject({ ok: false, reason: 'ineligible', attempts: 1 });+ });++ test('a process abort without the diagnostic is never retried', async () => {+ const result = await run({ command: nodeCmd('process.abort()'), ...withPausedParserRule });+ expect(result.ok).toBe(false);+ expect(result.attempts).toBe(1);+ expect(['child-signal', 'ineligible']).toContain(result.reason);+ expect(result.log).not.toContain('decision=retry');+ });+});++describe('the result file', () => {+ const cli = (resultFile, ...rest) =>+ spawnSync(+ process.execPath,+ [+ SCRIPT,+ '--label',+ 'result-file probe',+ '--deadline-epoch-ms',+ String(Date.now() + 60_000),+ '--attempt-timeout',+ '30s',+ '--retry-on',+ 'http-5xx,connection',+ '--result-file',+ resultFile,+ '--',+ ...rest,+ ],+ { encoding: 'utf8' },+ );++ test('records a success and an ineligible stop for the caller', () => {+ const ok = join(scratch, 'result-ok.json');+ expect(cli(ok, process.execPath, '-e', 'process.exit(0)').status).toBe(0);+ expect(JSON.parse(readFileSync(ok, 'utf8'))).toEqual({ ok: true, attempts: 1 });+ const stopped = join(scratch, 'result-stopped.json');+ const failed = cli(+ stopped,+ process.execPath,+ '-e',+ 'console.error("unrecognized failure");process.exit(2)',+ );+ expect(failed.status).toBe(1);+ expect(JSON.parse(readFileSync(stopped, 'utf8'))).toEqual({+ ok: false,+ reason: 'ineligible',+ attempts: 1,+ code: 2,+ });+ expect(failed.stdout).toContain(+ '::error::result-file probe decision=stop reason=diagnostic:unknown outcome=ineligible',+ );+ });++ test('writes nothing unless asked', () => {+ const empty = mkdtempSync(join(scratch, 'no-result-'));+ const result = spawnSync(+ process.execPath,+ [+ SCRIPT,+ '--deadline-epoch-ms',+ String(Date.now() + 60_000),+ '--attempt-timeout',+ '30s',+ '--',+ process.execPath,+ '-e',+ 'process.exit(0)',+ ],+ { encoding: 'utf8', cwd: empty },+ );+ expect(result.status).toBe(0);+ expect(result.stdout).toContain('command succeeded on attempt 1.');+ expect(readdirSync(empty)).toEqual([]);+ });+});+describe('retry delay', () => {test('uses jittered 30s and 60s bases', () => {const evidence = inspect('socket hang up').evidence;@@ -1494,6 +1717,69 @@ describe('parseArgs', () => {).toThrow(/--label requires a value/);});+ test('parses the opt-in policy flags only when given', () => {+ const parsed = parseArgs(+ argv(+ '--deadline-epoch-ms',+ '2000000000000',+ '--attempt-timeout',+ '1m',+ '--transient-rule',+ 'undici-paused-parser=assert\\(!this\\.paused\\)',+ '--transient-rule',+ 'undici-terminated=\\bTypeError: terminated\\b',+ '--retry-on',+ 'http-5xx,connection,rule:undici-paused-parser',+ '--result-file',+ '/tmp/result.json',+ '--',+ 'true',+ ),+ );+ expect(parsed.retryOn).toEqual({+ http5xx: true,+ connection: true,+ rules: ['undici-paused-parser'],+ });+ expect(parsed.transientRules.map(([id]) => id)).toEqual([+ 'undici-paused-parser',+ 'undici-terminated',+ ]);+ expect(parsed.transientRules[0][1].test('assert(!this.paused)')).toBe(true);+ expect(parsed.resultFile).toBe('/tmp/result.json');+ const plain = parseArgs(+ argv('--deadline-epoch-ms', '2000000000000', '--attempt-timeout', '1m', '--', 'true'),+ );+ expect(Object.keys(plain)).not.toEqual(expect.arrayContaining(['retryOn']));+ expect('transientRules' in plain || 'resultFile' in plain || 'retryOn' in plain).toBe(false);+ });++ test.each([+ [['--retry-on', 'http-5xx,retry-everything'], /unknown token: retry-everything/],+ [['--retry-on', 'rule:undici-paused-parser'], /names no --transient-rule/],+ [['--transient-rule', 'Bad_Id=x'], /lowercase id/],+ [['--transient-rule', 'no-pattern='], /empty pattern/],+ [['--transient-rule', 'bad-regex=('], /not a valid pattern/],+ [['--transient-rule', 'socket-hangup=x'], /shadows a built-in rule/],+ [['--transient-rule', 'twice=a', '--transient-rule', 'twice=b'], /must be unique/],+ [['--transient-rule', 'lonely=x'], /--transient-rule requires --retry-on/],+ [['--result-file'], /--result-file requires a value/],+ ])('rejects the malformed opt-in %j', (flags, message) => {+ expect(() =>+ parseArgs(+ argv(+ '--deadline-epoch-ms',+ '2000000000000',+ '--attempt-timeout',+ '1m',+ ...flags,+ '--',+ 'true',+ ),+ ),+ ).toThrow(message);+ });+test('accepts an explicit empty retry warning', () => {const parsed = parseArgs(argv(diff --git a/knip.config.ts b/knip.config.tsindex be41ab16c..37d9ce912 100644--- a/knip.config.ts+++ b/knip.config.ts@@ -145,11 +145,15 @@ export default {'packages/server': {entry: ['src/**/*.test.ts', 'src/parse-worker.ts'],project: 'src/**',- ignoreDependencies: ['@types/shell-quote'],+ ignoreDependencies: [+ '@types/shell-quote',+ ],},'packages/cli': {entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts', 'src/parse-worker.ts'],- ignoreDependencies: ['yjs'],+ ignoreDependencies: [+ 'yjs',+ ],},'packages/desktop': {entry: [diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.tsindex 8a60e5826..a28cc2b18 100644--- a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts+++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts@@ -1,3 +1,4 @@+import { readFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts b/packages/app/tests/integration/indented-jsx-byte-sacred.test.tsindex 6fcdc23f2..4fc2c11f9 100644--- a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts+++ b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts@@ -1,3 +1,4 @@+import { setTimeout as wait } from 'node:timers/promises';import { normalizeBridge } from '@inkeep/open-knowledge-core';import { updateYFragment } from '@tiptap/y-tiptap';diff --git a/packages/app/tests/integration/interior-edit-freshness.test.ts b/packages/app/tests/integration/interior-edit-freshness.test.tsindex 40b5cdfd6..9f2f6feb6 100644--- a/packages/app/tests/integration/interior-edit-freshness.test.ts+++ b/packages/app/tests/integration/interior-edit-freshness.test.ts@@ -1,3 +1,4 @@+import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts b/packages/app/tests/integration/keystroke-cadence-danger-space.test.tsindex 93666af70..2a085742b 100644--- a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts+++ b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts@@ -1,3 +1,4 @@+import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/producer-guard-wiring.test.ts b/packages/app/tests/integration/producer-guard-wiring.test.tsindex 867598d34..878d5196b 100644--- a/packages/app/tests/integration/producer-guard-wiring.test.ts+++ b/packages/app/tests/integration/producer-guard-wiring.test.ts@@ -1,3 +1,4 @@+import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.tsindex 0970ac12f..4ada3cf26 100644--- a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts+++ b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts@@ -1,3 +1,4 @@+import { randomUUID } from 'node:crypto';import { afterEach, describe, expect, test } from 'vitest';import { agentWriteMd, createTestServer, type TestServer } from './test-harness';diff --git a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts b/packages/app/tests/integration/qa-045-restore-after-gc.test.tsindex b632185e1..cbe2b705f 100644--- a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts+++ b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts@@ -1,3 +1,4 @@+import { execFileSync } from 'node:child_process';import { randomUUID } from 'node:crypto';import { join } from 'node:path';diff --git a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.tsindex c530ce4ba..89bc9b4d9 100644--- a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts+++ b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts@@ -1,3 +1,4 @@+import { randomUUID } from 'node:crypto';import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest';import type * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-after-rollback.test.ts b/packages/app/tests/integration/undo-after-rollback.test.tsindex d3767adc9..a78aeb0e4 100644--- a/packages/app/tests/integration/undo-after-rollback.test.ts+++ b/packages/app/tests/integration/undo-after-rollback.test.ts@@ -1,3 +1,4 @@+import { ROLLBACK_ORIGIN } from '@inkeep/open-knowledge-server';import { afterAll, beforeAll, describe, expect, test } from 'vitest';import * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-recycle-reset.test.ts b/packages/app/tests/integration/undo-recycle-reset.test.tsindex b076f955c..e7cc69c3f 100644--- a/packages/app/tests/integration/undo-recycle-reset.test.ts+++ b/packages/app/tests/integration/undo-recycle-reset.test.ts@@ -1,3 +1,4 @@+import { writeFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/undo-rederive-survival.test.ts b/packages/app/tests/integration/undo-rederive-survival.test.tsindex 7935e35f8..9ddcd0730 100644--- a/packages/app/tests/integration/undo-rederive-survival.test.ts+++ b/packages/app/tests/integration/undo-rederive-survival.test.ts@@ -1,3 +1,4 @@+import { prependFrontmatter } from '@inkeep/open-knowledge-core';import { Plugin, PluginKey } from '@tiptap/pm/state';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.tsindex f4cc8a286..2a597ec3d 100644--- a/packages/core/src/bridge/compose.ts+++ b/packages/core/src/bridge/compose.ts@@ -1,3 +1,4 @@+import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts';export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved';diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.tsindex 8fb428b90..01ef76a9f 100644--- a/packages/core/src/bridge/doc-boundary-space.ts+++ b/packages/core/src/bridge/doc-boundary-space.ts@@ -1,3 +1,4 @@+import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';import { carriedEdgeEmpties, type DocEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts';diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.tsindex fc267cbea..fb0ba5a7c 100644--- a/packages/core/src/markdown/comment-promoter.ts+++ b/packages/core/src/markdown/comment-promoter.ts@@ -1,3 +1,4 @@+import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.tsindex bfd7e3109..fa3cfcd86 100644--- a/packages/core/src/markdown/merged-walker.ts+++ b/packages/core/src/markdown/merged-walker.ts@@ -1,3 +1,4 @@+import type { Nodes, Parent, Root } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.tsindex 4d3250d3a..8be632ba4 100644--- a/packages/core/src/markdown/safe-url.ts+++ b/packages/core/src/markdown/safe-url.ts@@ -1,3 +1,4 @@+export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');diff --git a/packages/desktop/scripts/prepare-platform-natives.mjs b/packages/desktop/scripts/prepare-platform-natives.mjsindex 232d3fc07..763d91497 100644--- a/packages/desktop/scripts/prepare-platform-natives.mjs+++ b/packages/desktop/scripts/prepare-platform-natives.mjs@@ -1,113 +1,423 @@#!/usr/bin/env nodeimport { execFileSync } from 'node:child_process';import { createHash } from 'node:crypto';-import { createWriteStream, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';+import {+ existsSync,+ lstatSync,+ mkdirSync,+ readdirSync,+ readFileSync,+ realpathSync,+ renameSync,+ rmSync,+ writeFileSync,+} from 'node:fs';import { tmpdir } from 'node:os';-import { dirname, join, resolve } from 'node:path';-import { Readable } from 'node:stream';-import { pipeline } from 'node:stream/promises';+import { dirname, join, relative, resolve, sep } from 'node:path';import { fileURLToPath } from 'node:url';+import { gunzipSync } from 'node:zlib';-const PLATFORM_PACKAGES = {+export const PLATFORM_PACKAGES = {win32: ['win32-x64-msvc', 'win32-arm64-msvc'],linux: ['linux-x64-gnu', 'linux-arm64-gnu'],};-const suffixes = PLATFORM_PACKAGES[process.platform];-if (!suffixes) {- console.log(- `[prepare-platform-natives] platform=${process.platform} — no-op (win32/linux only; darwin uses prepare-universal.mjs).`,- );- process.exit(0);+const LOG_PREFIX = '[prepare-platform-natives]';+const DEFAULT_REGISTRY = 'https://registry.npmjs.org/';+const STAGING_PREFIX = '.staging-keyring-';+const BLOCK = 512;++export class NativesError extends Error {}++function octalField(header, start, length) {+ const raw = header+ .subarray(start, start + length)+ .toString('latin1')+ .replace(/\0.*$/s, '')+ .trim();+ if (raw === '') return 0;+ if (!/^[0-7]+$/.test(raw)) throw new NativesError(`archive rejected: bad octal field "${raw}"`);+ return Number.parseInt(raw, 8);}-const HERE = dirname(fileURLToPath(import.meta.url));-const REPO_ROOT = resolve(HERE, '..', '..', '..');-const NAPI_DIR = join(REPO_ROOT, 'node_modules', '@napi-rs');+function stringField(header, start, length) {+ return header+ .subarray(start, start + length)+ .toString('utf8')+ .replace(/\0.*$/s, '');+}-const wrapperPkgJson = join(NAPI_DIR, 'keyring', 'package.json');-if (!existsSync(wrapperPkgJson)) {- console.error(- `[prepare-platform-natives] @napi-rs/keyring not present at ${wrapperPkgJson}. Run \`pnpm install\` first.`,- );- process.exit(1);+function headerChecksumMatches(header) {+ const stored = octalField(header, 148, 8);+ let unsigned = 0;+ let signed = 0;+ for (let index = 0; index < BLOCK; index += 1) {+ const byte = index >= 148 && index < 156 ? 32 : header[index];+ unsigned += byte;+ signed += byte > 127 ? byte - 256 : byte;+ }+ return stored === unsigned || stored === signed;+}++function strippedEntryPath(name) {+ const parts = name.split('/').filter((part) => part !== '' && part !== '.');+ if (name.startsWith('/') || parts.includes('..')) {+ throw new NativesError(`archive rejected: unsafe path "${name}"`);+ }+ return parts.slice(1).join('/');+}++export function readTarEntries(tar) {+ const files = new Map();+ let offset = 0;+ let pendingName = null;+ while (offset + BLOCK <= tar.length) {+ const header = tar.subarray(offset, offset + BLOCK);+ if (header.every((byte) => byte === 0)) break;+ if (!headerChecksumMatches(header)) {+ throw new NativesError(`archive rejected: bad header checksum at byte ${offset}`);+ }+ const type = header[156] === 0 ? '0' : String.fromCharCode(header[156]);+ const size = octalField(header, 124, 12);+ const prefix = stringField(header, 345, 155);+ const headerName = prefix+ ? `${prefix}/${stringField(header, 0, 100)}`+ : stringField(header, 0, 100);+ const data = tar.subarray(offset + BLOCK, offset + BLOCK + size);+ if (data.length !== size) throw new NativesError('archive rejected: truncated entry data');+ offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK;+ if (type === 'x') {+ pendingName = /(?:^|\n)\d+ path=([^\n]*)\n/.exec(data.toString('utf8'))?.[1] ?? pendingName;+ continue;+ }+ if (type === 'L') {+ pendingName = data.toString('utf8').replace(/\0.*$/s, '');+ continue;+ }+ if (type === 'g') continue;+ const name = pendingName ?? headerName;+ pendingName = null;+ if (type === '5') continue;+ if (type !== '0') {+ throw new NativesError(`archive rejected: unsupported entry type "${type}" for "${name}"`);+ }+ const path = strippedEntryPath(name);+ if (path === '') continue;+ if (files.has(path)) throw new NativesError(`archive rejected: duplicate entry "${path}"`);+ files.set(path, createHash('sha256').update(data).digest('hex'));+ }+ return files;+}++export function treeFiles(dir) {+ const files = new Map();+ const walk = (current) => {+ for (const entry of readdirSync(current, { withFileTypes: true })) {+ const full = join(current, entry.name);+ if (entry.isDirectory()) walk(full);+ else if (entry.isFile()) {+ files.set(+ relative(dir, full).split(sep).join('/'),+ createHash('sha256').update(readFileSync(full)).digest('hex'),+ );+ } else throw new NativesError(`unexpected non-file "${relative(dir, full)}"`);+ }+ };+ walk(dir);+ return files;+}++function sameFiles(actual, expected) {+ if (actual.size !== expected.size) {+ return `${actual.size} file(s) on disk, ${expected.size} in the verified archive`;+ }+ for (const [path, digest] of expected) {+ if (!actual.has(path)) return `"${path}" is missing`;+ if (actual.get(path) !== digest) return `"${path}" differs from the verified archive`;+ }+ return null;}-const version = JSON.parse(readFileSync(wrapperPkgJson, 'utf8')).version;-function lockfileIntegrityFor(pkgName, pkgVersion) {- const lockfile = readFileSync(join(REPO_ROOT, 'pnpm-lock.yaml'), 'utf8');- const key = `${pkgName}@${pkgVersion}`;- const at = lockfile.indexOf(`'${key}':`);+export function lockfileIntegrityFor(lockfile, pkgName, pkgVersion) {+ const at = lockfile.indexOf(`'${pkgName}@${pkgVersion}':`);if (at === -1) return null;- const window = lockfile.slice(at, at + 500);- const m = /integrity: (sha512-[A-Za-z0-9+/=]+)/.exec(window);- return m?.[1] ?? null;+ return /integrity: (sha512-[A-Za-z0-9+/=]+)/.exec(lockfile.slice(at, at + 500))?.[1] ?? null;}-console.log(`[prepare-platform-natives] target version: @napi-rs/keyring-* v${version}`);-console.log(`[prepare-platform-natives] @napi-rs root: ${NAPI_DIR}`);+function markerPathFor(repoRoot, pkgName) {+ return join(+ repoRoot,+ 'node_modules',+ '.cache',+ 'prepare-platform-natives',+ `${pkgName.replace('/', '__')}.json`,+ );+}-for (const suffix of suffixes) {- const pkgName = `@napi-rs/keyring-${suffix}`;- const targetDir = join(NAPI_DIR, `keyring-${suffix}`);- const targetPkgJson = join(targetDir, 'package.json');+function readJson(path) {+ try {+ return { value: JSON.parse(readFileSync(path, 'utf8')), problem: null };+ } catch (error) {+ return { value: null, problem: error.code === 'ENOENT' ? null : error.message };+ }+}- if (existsSync(targetPkgJson)) {- const installed = JSON.parse(readFileSync(targetPkgJson, 'utf8'));- if (installed.version === version) {- console.log(`[prepare-platform-natives] ${pkgName}@${version} present — skip`);- continue;+export function installedState({ repoRoot, targetDir, pkgName, version, integrity }) {+ let stat;+ try {+ stat = lstatSync(targetDir);+ } catch (error) {+ return {+ valid: false,+ reason: error.code === 'ENOENT' ? 'missing' : `unreadable: ${error.message}`,+ };+ }+ if (stat.isSymbolicLink()) {+ let real;+ let store;+ try {+ real = realpathSync(targetDir);+ store = realpathSync(join(repoRoot, 'node_modules', '.pnpm')) + sep;+ } catch (error) {+ return {+ valid: false,+ reason: error.code === 'ENOENT' ? 'dangling link' : `link unreadable: ${error.message}`,+ };}- console.log(- `[prepare-platform-natives] ${pkgName} version mismatch (have=${installed.version}, want=${version}) — re-extracting`,- );- rmSync(targetDir, { recursive: true, force: true });- } else {- console.log(`[prepare-platform-natives] ${pkgName}@${version} missing — fetching`);+ const { value: manifest, problem } = readJson(join(real, 'package.json'));+ if (problem) return { valid: false, reason: `manifest unreadable: ${problem}` };+ if (real.startsWith(store) && manifest?.version === version) {+ return { valid: true, reason: 'pnpm-managed' };+ }+ return { valid: false, reason: 'a link outside the pnpm store or at another version' };+ }+ const { value: marker, problem } = readJson(markerPathFor(repoRoot, pkgName));+ if (problem) return { valid: false, reason: `completion marker unreadable: ${problem}` };+ if (!marker) return { valid: false, reason: 'no completion marker' };+ if (!integrity || marker.version !== version || marker.integrity !== integrity) {+ return { valid: false, reason: 'completion marker is for another version or archive' };}+ let actual;+ try {+ actual = treeFiles(targetDir);+ } catch (error) {+ return { valid: false, reason: error.message };+ }+ const mismatch = sameFiles(actual, new Map(Object.entries(marker.files ?? {})));+ return mismatch ? { valid: false, reason: mismatch } : { valid: true, reason: 'verified' };+}- const tarballUrl = `https://registry.npmjs.org/${pkgName}/-/keyring-${suffix}-${version}.tgz`;- const tmpTarball = join(tmpdir(), `keyring-${suffix}-${version}-${process.pid}.tgz`);+function describeCause(error) {+ const cause = error?.cause;+ if (cause?.code) return `${cause.code}: ${cause.message}`;+ return cause?.message ?? error?.message ?? String(error);+}- const expectedIntegrity = lockfileIntegrityFor(pkgName, version);- if (!expectedIntegrity) {- console.error(- `[prepare-platform-natives] ${pkgName}@${version} has no integrity entry in pnpm-lock.yaml — refusing an unpinned registry fetch.`,+async function fetchVerified({ fetchImpl, url, pkgName, version, integrity }) {+ let response;+ try {+ response = await fetchImpl(url);+ } catch (error) {+ throw new NativesError(`fetch ${url} failed: ${describeCause(error)}`);+ }+ if (!response.ok) {+ throw new NativesError(`fetch ${url} → HTTP ${response.status} ${response.statusText}`);+ }+ let bytes;+ try {+ bytes = Buffer.from(await response.arrayBuffer());+ } catch (error) {+ throw new NativesError(`fetch ${url} body read failed: ${describeCause(error)}`);+ }+ const actual = `sha512-${createHash('sha512').update(bytes).digest('base64')}`;+ if (actual !== integrity) {+ throw new NativesError(+ `${pkgName}@${version}: sha512 hash mismatch, expected ${integrity}, got ${actual}`,);- process.exit(1);}+ return bytes;+}- const res = await fetch(tarballUrl);- if (!res.ok) {- console.error(- `[prepare-platform-natives] fetch ${tarballUrl} → ${res.status} ${res.statusText}`,- );- process.exit(1);+function defaultTarBin(platform) {+ return platform === 'win32'+ ? join(process.env.SystemRoot || 'C:\\Windows', 'System32', 'tar.exe')+ : 'tar';+}++export function extractWithTar(tarBin) {+ return (archivePath, destination) => {+ execFileSync(tarBin, ['-xzf', archivePath, '-C', destination, '--strip-components=1'], {+ stdio: 'inherit',+ });+ };+}++function removeIfPresent(path, log) {+ try {+ rmSync(path, { recursive: true, force: true });+ } catch (error) {+ log(`${LOG_PREFIX} could not remove ${path}: ${error.code}`);}- await pipeline(Readable.fromWeb(res.body), createWriteStream(tmpTarball));+}+function publish({ stagingDir, targetDir, markerPath, marker }) {try {- const actual = `sha512-${createHash('sha512').update(readFileSync(tmpTarball)).digest('base64')}`;- if (actual !== expectedIntegrity) {- console.error(- `[prepare-platform-natives] integrity mismatch for ${pkgName}@${version}: expected ${expectedIntegrity}, got ${actual}`,+ rmSync(targetDir, { recursive: true, force: true });+ } catch (error) {+ throw new NativesError(`publish could not remove the previous target: ${error.message}`);+ }+ try {+ renameSync(stagingDir, targetDir);+ } catch (error) {+ throw new NativesError(`publish rename failed: ${error.message}`);+ }+ try {+ mkdirSync(dirname(markerPath), { recursive: true });+ writeFileSync(markerPath, `${JSON.stringify(marker, null, 2)}\n`);+ } catch (error) {+ throw new NativesError(`publish marker write failed: ${error.message}`);+ }+}++async function prepareOne({+ repoRoot,+ napiDir,+ lockfile,+ suffix,+ version,+ registry,+ fetchImpl,+ extract,+ log,+}) {+ const pkgName = `@napi-rs/keyring-${suffix}`;+ const targetDir = join(napiDir, `keyring-${suffix}`);+ const integrity = lockfileIntegrityFor(lockfile, pkgName, version);+ const state = installedState({ repoRoot, targetDir, pkgName, version, integrity });+ if (state.valid) {+ log(`${LOG_PREFIX} ${pkgName}@${version} present (${state.reason}) — skip`);+ return;+ }+ if (!integrity) {+ throw new NativesError(+ `${pkgName}@${version} has no integrity entry in pnpm-lock.yaml — refusing an unpinned registry fetch.`,+ );+ }+ log(`${LOG_PREFIX} ${pkgName}@${version} ${state.reason} — fetching`);+ const url = new URL(`${pkgName}/-/keyring-${suffix}-${version}.tgz`, registry).href;+ const bytes = await fetchVerified({ fetchImpl, url, pkgName, version, integrity });+ let tar;+ try {+ tar = gunzipSync(bytes);+ } catch (error) {+ throw new NativesError(`archive rejected: gunzip ${error.code ?? 'failed'}: ${error.message}`);+ }+ const expected = readTarEntries(tar);+ const stagingDir = join(napiDir, `${STAGING_PREFIX}${suffix}-${process.pid}`);+ const archivePath = join(tmpdir(), `keyring-${suffix}-${version}-${process.pid}.tgz`);+ rmSync(stagingDir, { recursive: true, force: true });+ mkdirSync(stagingDir, { recursive: true });+ try {+ writeFileSync(archivePath, bytes);+ try {+ extract(archivePath, stagingDir);+ } catch (error) {+ throw new NativesError(`${pkgName}@${version}: extraction failed: ${error.message}`);+ }+ const mismatch = sameFiles(treeFiles(stagingDir), expected);+ if (mismatch) {+ throw new NativesError(+ `${pkgName}@${version}: extraction does not match the verified archive: ${mismatch}`,);- process.exit(1);}-- mkdirSync(targetDir, { recursive: true });- const tarBin =- process.platform === 'win32'- ? join(process.env.SystemRoot || 'C:\\Windows', 'System32', 'tar.exe')- : 'tar';- execFileSync(tarBin, ['-xzf', tmpTarball, '-C', targetDir, '--strip-components=1'], {- stdio: 'inherit',+ publish({+ stagingDir,+ targetDir,+ markerPath: markerPathFor(repoRoot, pkgName),+ marker: { package: pkgName, version, integrity, files: Object.fromEntries(expected) },});} finally {- rmSync(tmpTarball, { force: true });+ removeIfPresent(archivePath, log);+ removeIfPresent(stagingDir, log);}+ log(`${LOG_PREFIX} extracted ${pkgName}@${version} → ${targetDir}`);+}- console.log(`[prepare-platform-natives] extracted ${pkgName}@${version} → ${targetDir}`);+export async function prepareNatives({+ repoRoot,+ platform = process.platform,+ registry = process.env.npm_config_registry || DEFAULT_REGISTRY,+ fetchImpl = fetch,+ extract = extractWithTar(defaultTarBin(platform)),+ checkOnly = false,+ log = console.log,+}) {+ const suffixes = PLATFORM_PACKAGES[platform];+ if (!suffixes) {+ log(+ `${LOG_PREFIX} platform=${platform} — no-op (win32/linux only; darwin uses prepare-universal.mjs).`,+ );+ return;+ }+ const napiDir = join(repoRoot, 'node_modules', '@napi-rs');+ const wrapperPkgJson = join(napiDir, 'keyring', 'package.json');+ if (!existsSync(wrapperPkgJson)) {+ throw new NativesError(+ `@napi-rs/keyring not present at ${wrapperPkgJson}. Run \`pnpm install\` first.`,+ );+ }+ const version = JSON.parse(readFileSync(wrapperPkgJson, 'utf8')).version;+ const lockfile = readFileSync(join(repoRoot, 'pnpm-lock.yaml'), 'utf8');+ log(`${LOG_PREFIX} target version: @napi-rs/keyring-* v${version}`);+ log(`${LOG_PREFIX} @napi-rs root: ${napiDir}`);+ for (const leftover of readdirSync(napiDir).filter((name) => name.startsWith(STAGING_PREFIX))) {+ removeIfPresent(join(napiDir, leftover), log);+ }+ const registryBase = registry.endsWith('/') ? registry : `${registry}/`;+ const invalid = [];+ for (const suffix of suffixes) {+ if (checkOnly) {+ const pkgName = `@napi-rs/keyring-${suffix}`;+ const integrity = lockfileIntegrityFor(lockfile, pkgName, version);+ const state = installedState({+ repoRoot,+ targetDir: join(napiDir, `keyring-${suffix}`),+ pkgName,+ version,+ integrity,+ });+ log(+ `${LOG_PREFIX} ${pkgName}@${version}: ${state.valid ? 'verified' : 'NOT verified'} (${state.reason})`,+ );+ if (!state.valid) invalid.push(`${pkgName}@${version} (${state.reason})`);+ continue;+ }+ await prepareOne({+ repoRoot,+ napiDir,+ lockfile,+ suffix,+ version,+ registry: registryBase,+ fetchImpl,+ extract,+ log,+ });+ }+ if (invalid.length > 0) {+ throw new NativesError(`keyring prebuilds not verified: ${invalid.join('; ')}`);+ }+ log(`${LOG_PREFIX} all target-platform keyring prebuilds present.`);}-console.log('[prepare-platform-natives] all target-platform keyring prebuilds present.');+if (import.meta.main) {+ try {+ await prepareNatives({+ repoRoot: resolve(dirname(fileURLToPath(import.meta.url)), '..', '..', '..'),+ checkOnly: process.argv.includes('--check'),+ });+ } catch (error) {+ console.error(`${LOG_PREFIX} ${error instanceof NativesError ? error.message : error.stack}`);+ process.exitCode = 1;+ }+}diff --git a/packages/desktop/scripts/prepare-platform-natives.test.mjs b/packages/desktop/scripts/prepare-platform-natives.test.mjsnew file mode 100644index 000000000..7458823e4--- /dev/null+++ b/packages/desktop/scripts/prepare-platform-natives.test.mjs@@ -0,0 +1,482 @@+import { createHash } from 'node:crypto';+import {+ chmodSync,+ existsSync,+ mkdirSync,+ mkdtempSync,+ readdirSync,+ readFileSync,+ rmSync,+ symlinkSync,+ writeFileSync,+} from 'node:fs';+import { createServer } from 'node:http';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { gzipSync } from 'node:zlib';+import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest';+import {+ extractWithTar,+ installedState,+ NativesError,+ prepareNatives,+ readTarEntries,+} from './prepare-platform-natives.mjs';++const VERSION = '1.3.0';+const ARM64 = '@napi-rs/keyring-win32-arm64-msvc';+const X64 = '@napi-rs/keyring-win32-x64-msvc';++function tarHeader(name, size, type = '0') {+ const header = Buffer.alloc(512, 0);+ header.write(name, 0, 100, 'utf8');+ header.write('0000644\0', 100, 8, 'latin1');+ header.write('0000000\0', 108, 8, 'latin1');+ header.write('0000000\0', 116, 8, 'latin1');+ header.write(`${size.toString(8).padStart(11, '0')}\0`, 124, 12, 'latin1');+ header.write('00000000000\0', 136, 12, 'latin1');+ header.write(' ', 148, 8, 'latin1');+ header.write(type, 156, 1, 'latin1');+ header.write('ustar\0', 257, 6, 'latin1');+ header.write('00', 263, 2, 'latin1');+ let sum = 0;+ for (const byte of header) sum += byte;+ header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 8, 'latin1');+ return header;+}++function tarBytes(entries) {+ const parts = [];+ for (const { name, data = Buffer.alloc(0), type = '0' } of entries) {+ parts.push(tarHeader(name, data.length, type), data);+ parts.push(Buffer.alloc((512 - (data.length % 512)) % 512));+ }+ parts.push(Buffer.alloc(1024));+ return Buffer.concat(parts);+}++const nativeBytes = Buffer.alloc(40_000, 7);+const manifest = Buffer.from(JSON.stringify({ name: ARM64, version: VERSION }));+const archive = gzipSync(+ tarBytes([+ { name: 'package/', type: '5' },+ { name: 'package/package.json', data: manifest },+ { name: 'package/README.md', data: Buffer.from('readme') },+ { name: 'package/keyring.win32-arm64-msvc.node', data: nativeBytes },+ ]),+);+const integrity = `sha512-${createHash('sha512').update(archive).digest('base64')}`;++let server;+let registry;+let mode = 'ok';+let hits = 0;++beforeAll(async () => {+ server = createServer((request, response) => {+ hits += 1;+ if (mode === 'drop') {+ request.socket.destroy();+ return;+ }+ if (mode === 'unavailable') {+ response.writeHead(503, 'Service Unavailable').end();+ return;+ }+ if (!request.url.endsWith(`/keyring-win32-arm64-msvc-${VERSION}.tgz`)) {+ response.writeHead(404, 'Not Found').end();+ return;+ }+ response.writeHead(200, { 'content-type': 'application/octet-stream' });+ response.end(mode === 'tampered' ? gzipSync(tarBytes([])) : archive);+ });+ await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));+ registry = `http://127.0.0.1:${server.address().port}/`;+});++afterAll(() => new Promise((resolve) => server.close(resolve)));++const roots = [];+afterEach(() => {+ mode = 'ok';+ for (const root of roots.splice(0)) {+ chmodSync(join(root, 'node_modules', '@napi-rs'), 0o755);+ rmSync(root, { recursive: true, force: true });+ }+});++function fixtureRoot() {+ const root = mkdtempSync(join(tmpdir(), 'prepare-platform-natives-'));+ roots.push(root);+ const napi = join(root, 'node_modules', '@napi-rs');+ mkdirSync(join(napi, 'keyring'), { recursive: true });+ writeFileSync(join(napi, 'keyring', 'package.json'), JSON.stringify({ version: VERSION }));+ const store = join(root, 'node_modules', '.pnpm', `@napi-rs+keyring-win32-x64-msvc@${VERSION}`);+ const hostPackage = join(store, 'node_modules', '@napi-rs', 'keyring-win32-x64-msvc');+ mkdirSync(hostPackage, { recursive: true });+ writeFileSync(join(hostPackage, 'package.json'), JSON.stringify({ name: X64, version: VERSION }));+ symlinkSync(hostPackage, join(napi, 'keyring-win32-x64-msvc'));+ writeFileSync(+ join(root, 'pnpm-lock.yaml'),+ `packages:\n\n '${ARM64}@${VERSION}':\n resolution: {integrity: ${integrity}}\n`,+ );+ return root;+}++const run = (root, overrides = {}) => {+ const lines = [];+ return prepareNatives({+ repoRoot: root,+ platform: 'win32',+ registry,+ extract: extractWithTar('tar'),+ log: (line) => lines.push(line),+ ...overrides,+ }).then(() => lines);+};++const arm64Dir = (root) => join(root, 'node_modules', '@napi-rs', 'keyring-win32-arm64-msvc');+const hostManifest = (root) =>+ join(root, 'node_modules', '@napi-rs', 'keyring-win32-x64-msvc', 'package.json');+const parseFailure = (text) => {+ try {+ JSON.parse(text);+ } catch (error) {+ return error.message;+ }+};+const leftovers = (root) =>+ readdirSync(join(root, 'node_modules', '@napi-rs')).filter((name) => name.startsWith('.staging'));++describe('prepare-platform-natives', () => {+ test('publishes a verified prebuild and records its completion marker', async () => {+ const root = fixtureRoot();+ const lines = await run(root);+ expect(readFileSync(join(arm64Dir(root), 'keyring.win32-arm64-msvc.node'))).toEqual(+ nativeBytes,+ );+ expect(lines.join('\n')).toContain(`${X64}@${VERSION} present (pnpm-managed) — skip`);+ expect(+ installedState({+ repoRoot: root,+ targetDir: arm64Dir(root),+ pkgName: ARM64,+ version: VERSION,+ integrity,+ }),+ ).toEqual({ valid: true, reason: 'verified' });+ expect(leftovers(root)).toEqual([]);+ });++ test('skips a verified install without touching the registry', async () => {+ const root = fixtureRoot();+ await run(root);+ const before = hits;+ const lines = await run(root);+ expect(hits).toBe(before);+ expect(lines.join('\n')).toContain(`${ARM64}@${VERSION} present (verified) — skip`);+ });++ test('refuses bytes that do not match the lockfile as a hash mismatch', async () => {+ const root = fixtureRoot();+ mode = 'tampered';+ await expect(run(root)).rejects.toThrow(/sha512 hash mismatch, expected sha512-/);+ expect(existsSync(arm64Dir(root))).toBe(false);+ expect(leftovers(root)).toEqual([]);+ });++ test('names an HTTP failure by its status', async () => {+ const root = fixtureRoot();+ mode = 'unavailable';+ await expect(run(root)).rejects.toThrow(/→ HTTP 503 Service Unavailable/);+ expect(existsSync(arm64Dir(root))).toBe(false);+ });++ test('names a dropped connection by its cause code', async () => {+ const root = fixtureRoot();+ mode = 'drop';+ await expect(run(root)).rejects.toThrow(/failed: (UND_ERR_SOCKET|ECONNRESET)/);+ });++ test('rebuilds a partial install from verified bytes', async () => {+ const root = fixtureRoot();+ mkdirSync(arm64Dir(root), { recursive: true });+ writeFileSync(join(arm64Dir(root), 'package.json'), manifest);+ writeFileSync(+ join(arm64Dir(root), 'keyring.win32-arm64-msvc.node'),+ nativeBytes.subarray(0, 100),+ );+ const lines = await run(root);+ expect(lines.join('\n')).toContain('no completion marker — fetching');+ expect(readFileSync(join(arm64Dir(root), 'keyring.win32-arm64-msvc.node'))).toEqual(+ nativeBytes,+ );+ });++ test('keeps a partial install unverified when the bytes cannot be fetched', async () => {+ const root = fixtureRoot();+ mkdirSync(arm64Dir(root), { recursive: true });+ writeFileSync(join(arm64Dir(root), 'package.json'), manifest);+ writeFileSync(+ join(arm64Dir(root), 'keyring.win32-arm64-msvc.node'),+ nativeBytes.subarray(0, 100),+ );+ mode = 'unavailable';+ await expect(run(root)).rejects.toThrow(/HTTP 503/);+ await expect(run(root, { checkOnly: true })).rejects.toThrow(+ new RegExp(`${ARM64}@${VERSION} \\(no completion marker\\)`),+ );+ });++ test('catches a truncated file behind a valid marker', async () => {+ const root = fixtureRoot();+ await run(root);+ writeFileSync(+ join(arm64Dir(root), 'keyring.win32-arm64-msvc.node'),+ nativeBytes.subarray(0, 100),+ );+ await expect(run(root, { checkOnly: true })).rejects.toThrow(+ /differs from the verified archive/,+ );+ await run(root);+ expect(readFileSync(join(arm64Dir(root), 'keyring.win32-arm64-msvc.node'))).toEqual(+ nativeBytes,+ );+ });++ test('publishes nothing when extraction does not reproduce the verified archive', async () => {+ const root = fixtureRoot();+ const partialExtract = (_archivePath, destination) => {+ writeFileSync(join(destination, 'package.json'), manifest);+ };+ await expect(run(root, { extract: partialExtract })).rejects.toThrow(+ /extraction does not match the verified archive/,+ );+ expect(existsSync(arm64Dir(root))).toBe(false);+ expect(leftovers(root)).toEqual([]);+ });++ test.skipIf(process.getuid?.() === 0)('names a failed publish rename by its code', async () => {+ const root = fixtureRoot();+ const napi = join(root, 'node_modules', '@napi-rs');+ const lockParent = (archivePath, destination) => {+ extractWithTar('tar')(archivePath, destination);+ chmodSync(napi, 0o555);+ };+ await expect(run(root, { extract: lockParent })).rejects.toThrow(/publish rename failed: E/);+ });++ test('names a completion marker it cannot write by its code', async () => {+ const root = fixtureRoot();+ writeFileSync(join(root, 'node_modules', '.cache'), 'not a directory');+ await expect(run(root)).rejects.toThrow(/publish marker write failed: E/);+ });++ test.skipIf(process.getuid?.() === 0)(+ 'logs a staging directory it cannot remove after a failed publish',+ async () => {+ const root = fixtureRoot();+ const napi = join(root, 'node_modules', '@napi-rs');+ const staging = join(napi, `.staging-keyring-win32-arm64-msvc-${process.pid}`);+ const lockParent = (archivePath, destination) => {+ extractWithTar('tar')(archivePath, destination);+ chmodSync(napi, 0o555);+ };+ const lines = [];+ await expect(+ run(root, { extract: lockParent, log: (line) => lines.push(line) }),+ ).rejects.toThrow(/publish rename failed: E/);+ expect(lines.filter((line) => line.includes(staging))).toEqual([+ expect.stringMatching(/^\[prepare-platform-natives\] .*: E[A-Z]+$/),+ ]);+ },+ );++ test.skipIf(process.getuid?.() === 0)(+ 'logs a leftover staging directory it cannot remove and still publishes',+ async () => {+ const root = fixtureRoot();+ const leftover = join(+ root,+ 'node_modules',+ '@napi-rs',+ '.staging-keyring-win32-arm64-msvc-earlier',+ );+ const locked = join(leftover, 'locked');+ mkdirSync(locked, { recursive: true });+ writeFileSync(join(locked, 'keyring.win32-arm64-msvc.node'), 'partial');+ chmodSync(locked, 0o555);+ let lines;+ try {+ lines = await run(root);+ } finally {+ chmodSync(locked, 0o755);+ }+ expect(lines.filter((line) => line.includes(leftover))).toEqual([+ expect.stringMatching(/^\[prepare-platform-natives\] .*: E[A-Z]+$/),+ ]);+ expect(readFileSync(join(arm64Dir(root), 'keyring.win32-arm64-msvc.node'))).toEqual(+ nativeBytes,+ );+ },+ );++ test('refetches when the lockfile pins another archive', async () => {+ const root = fixtureRoot();+ await run(root);+ const other = `sha512-${createHash('sha512').update('other').digest('base64')}`;+ writeFileSync(+ join(root, 'pnpm-lock.yaml'),+ `packages:\n\n '${ARM64}@${VERSION}':\n resolution: {integrity: ${other}}\n`,+ );+ await expect(run(root, { checkOnly: true })).rejects.toThrow(+ `${ARM64}@${VERSION} (completion marker is for another version or archive)`,+ );+ const before = hits;+ await expect(run(root)).rejects.toThrow(/sha512 hash mismatch/);+ expect(hits).toBe(before + 1);+ });++ test('refuses a marker once the wrapper moves to another version', async () => {+ const root = fixtureRoot();+ await run(root);+ writeFileSync(+ join(root, 'node_modules', '@napi-rs', 'keyring', 'package.json'),+ JSON.stringify({ version: '1.4.0' }),+ );+ writeFileSync(+ join(root, 'pnpm-lock.yaml'),+ `packages:\n\n '${ARM64}@1.4.0':\n resolution: {integrity: ${integrity}}\n`,+ );+ await expect(run(root, { checkOnly: true })).rejects.toThrow(+ `${ARM64}@1.4.0 (completion marker is for another version or archive)`,+ );+ });++ test('rebuilds a published tree that gained an extra file', async () => {+ const root = fixtureRoot();+ await run(root);+ writeFileSync(join(arm64Dir(root), 'extra.node'), 'stray');+ await expect(run(root, { checkOnly: true })).rejects.toThrow(+ /4 file\(s\) on disk, 3 in the verified archive/,+ );+ await run(root);+ expect(existsSync(join(arm64Dir(root), 'extra.node'))).toBe(false);+ });++ test('does not trust a host link that leaves the pnpm store', async () => {+ const root = fixtureRoot();+ await run(root);+ const elsewhere = join(root, 'elsewhere', 'keyring-win32-x64-msvc');+ mkdirSync(elsewhere, { recursive: true });+ writeFileSync(join(elsewhere, 'package.json'), JSON.stringify({ name: X64, version: VERSION }));+ const link = join(root, 'node_modules', '@napi-rs', 'keyring-win32-x64-msvc');+ rmSync(link);+ symlinkSync(elsewhere, link);+ await expect(run(root, { checkOnly: true })).rejects.toThrow(+ `${X64}@${VERSION} (a link outside the pnpm store or at another version)`,+ );+ });++ test('names a corrupt store manifest behind a host link by its parse error', async () => {+ const root = fixtureRoot();+ await run(root);+ const corrupt = `{"version": "${VERSION}"`;+ writeFileSync(hostManifest(root), corrupt);+ const failure = await run(root, { checkOnly: true }).catch((error) => error);+ expect(failure).toBeInstanceOf(NativesError);+ expect(failure.message).toContain(`${X64}@${VERSION} (`);+ expect(failure.message).toContain(parseFailure(corrupt));+ expect(failure.message).not.toContain('a link outside the pnpm store or at another version');+ });++ test.skipIf(process.getuid?.() === 0)(+ 'names an unreadable store manifest behind a host link by its code',+ async () => {+ const root = fixtureRoot();+ await run(root);+ chmodSync(hostManifest(root), 0o000);+ const failure = await run(root, { checkOnly: true }).catch((error) => error);+ expect(failure).toBeInstanceOf(NativesError);+ expect(failure.message).toContain(`${X64}@${VERSION} (`);+ expect(failure.message).toMatch(/EACCES/);+ expect(failure.message).not.toContain('a link outside the pnpm store or at another version');+ },+ );++ test.skipIf(process.getuid?.() === 0)(+ 'names a previous target it cannot remove before publishing',+ async () => {+ const root = fixtureRoot();+ await run(root);+ writeFileSync(+ join(arm64Dir(root), 'keyring.win32-arm64-msvc.node'),+ nativeBytes.subarray(0, 100),+ );+ chmodSync(arm64Dir(root), 0o555);+ try {+ await expect(run(root)).rejects.toThrow(+ /publish could not remove the previous target: E[A-Z]+/,+ );+ } finally {+ chmodSync(arm64Dir(root), 0o755);+ }+ },+ );++ test('check mode verifies without fetching', async () => {+ const root = fixtureRoot();+ await run(root);+ const before = hits;+ const lines = await run(root, { checkOnly: true });+ expect(hits).toBe(before);+ expect(lines.join('\n')).toContain(`${ARM64}@${VERSION}: verified (verified)`);+ });++ test('refuses an unpinned package', async () => {+ const root = fixtureRoot();+ writeFileSync(join(root, 'pnpm-lock.yaml'), 'packages:\n');+ await expect(run(root)).rejects.toThrow(/no integrity entry in pnpm-lock.yaml/);+ });++ test('is a no-op on a platform it does not stage', async () => {+ const root = fixtureRoot();+ const lines = await run(root, { platform: 'darwin' });+ expect(lines.join('\n')).toContain('no-op');+ });+});++describe('readTarEntries', () => {+ test('hashes regular files under the stripped package root and honours pax paths', () => {+ const pax = Buffer.from('30 path=package/long-name.txt\n');+ const entries = readTarEntries(+ tarBytes([+ { name: 'package/a.txt', data: Buffer.from('a') },+ { name: 'PaxHeader', data: pax, type: 'x' },+ { name: 'package/short', data: Buffer.from('b') },+ ]),+ );+ expect([...entries.keys()]).toEqual(['a.txt', 'long-name.txt']);+ });++ test.each([+ ['a link entry', [{ name: 'package/link', type: '2' }], /unsupported entry type "2"/],+ ['an unsafe path', [{ name: '../escape', data: Buffer.from('x') }], /unsafe path/],+ [+ 'a duplicate entry',+ [+ { name: 'package/a', data: Buffer.from('1') },+ { name: 'package/a', data: Buffer.from('2') },+ ],+ /duplicate entry/,+ ],+ ])('rejects %s', (_name, entries, message) => {+ expect(() => readTarEntries(tarBytes(entries))).toThrow(message);+ });++ test('rejects a corrupted header', () => {+ const bytes = tarBytes([{ name: 'package/a', data: Buffer.from('1') }]);+ bytes[10] ^= 0xff;+ expect(() => readTarEntries(bytes)).toThrow(NativesError);+ });+});