Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Check OK core's build output against a manifest before readers (#5574)

Nick Gomez8bf281e49913
diff --git a/.github/workflows/desktop-build-win-linux.yml b/.github/workflows/desktop-build-win-linux.yml
index 41b7b9676..4141ab3dc 100644
--- a/.github/workflows/desktop-build-win-linux.yml
+++ b/.github/workflows/desktop-build-win-linux.yml
@@ -185,6 +185,7 @@ jobs:
packages/native-config/*.node
retention-days: 3
if-no-files-found: error
+ include-hidden-files: true
build-windows:
if: inputs.platforms == 'both' || inputs.platforms == 'windows'
diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml
index b4214d545..98f3827fc 100644
--- a/.github/workflows/desktop-release.yml
+++ b/.github/workflows/desktop-release.yml
@@ -475,6 +475,7 @@ jobs:
packages/native-config/*.node
retention-days: 3
if-no-files-found: error
+ include-hidden-files: true
# Deliberately does NOT `needs: prepare` — the mac job is self-contained
# (own workspace build, no bundle consumption) and is the longest job in
diff --git a/packages/core/package.json b/packages/core/package.json
index fb22ca7ae..8915a9286 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -156,7 +156,7 @@
"zod": "^4.3.6"
},
"scripts": {
- "build": "tsdown && pnpm run verify:exports",
+ "build": "node scripts/build-output.mjs build",
"verify:exports": "node scripts/verify-exports-targets.mjs",
"typecheck": "tsc --noEmit && tsc -p tsconfig.perf.json --noEmit",
"test": "vitest run src/ tests/contract/ scripts/",
diff --git a/packages/core/scripts/build-output.mjs b/packages/core/scripts/build-output.mjs
new file mode 100644
index 000000000..39069d477
--- /dev/null
+++ b/packages/core/scripts/build-output.mjs
@@ -0,0 +1,225 @@
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { lstatSync, readdirSync, readFileSync, readlinkSync, rmSync, writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ExportsTargetsError, verifyExportsTargets } from './verify-exports-targets.mjs';
+
+export const MANIFEST_NAME = 'output-manifest.json';
+const MANIFEST_VERSION = 1;
+const PRODUCER_TASK = 'build:output';
+const LISTED_PATHS = 10;
+
+const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const TURBO_ROOT = resolve(PACKAGE_ROOT, '../..');
+const OUTPUT_ROOT = join(PACKAGE_ROOT, 'dist');
+const MANIFEST_PATH = join(OUTPUT_ROOT, MANIFEST_NAME);
+
+class BuildOutputError extends Error {}
+
+const packageName = () => JSON.parse(readFileSync(join(PACKAGE_ROOT, 'package.json'), 'utf8')).name;
+
+const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
+
+const byPath = (left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0);
+
+function entryType(entry) {
+ if (entry.isSymbolicLink()) return 'symlink';
+ if (entry.isDirectory()) return 'directory';
+ if (entry.isFile()) return 'file';
+ return 'other';
+}
+
+function inventory() {
+ if (!lstatSync(OUTPUT_ROOT).isDirectory()) throw new BuildOutputError('dist is not a directory');
+ const members = [];
+ const visit = (relative) => {
+ for (const entry of readdirSync(join(OUTPUT_ROOT, relative), { withFileTypes: true })) {
+ const path = relative ? `${relative}/${entry.name}` : entry.name;
+ if (path === MANIFEST_NAME) continue;
+ const type = entryType(entry);
+ if (type === 'directory') {
+ members.push({ path, type });
+ visit(path);
+ } else if (type === 'other') {
+ members.push({ path, type });
+ } else {
+ const absolute = join(OUTPUT_ROOT, path);
+ const bytes =
+ type === 'file' ? readFileSync(absolute) : Buffer.from(readlinkSync(absolute));
+ members.push({ path, type, size: bytes.length, sha256: sha256(bytes) });
+ }
+ }
+ };
+ visit('');
+ return members.sort(byPath);
+}
+
+function readManifest() {
+ let manifest;
+ try {
+ manifest = JSON.parse(readFileSync(MANIFEST_PATH, 'utf8'));
+ } catch (error) {
+ return {
+ problem:
+ error.code === 'ENOENT'
+ ? `dist/${MANIFEST_NAME} is missing`
+ : `dist/${MANIFEST_NAME} is unreadable (${error.message})`,
+ };
+ }
+ if (
+ manifest?.version !== MANIFEST_VERSION ||
+ !Array.isArray(manifest.members) ||
+ !manifest.members.every((member) => typeof member?.path === 'string')
+ ) {
+ return { problem: `dist/${MANIFEST_NAME} is not a version ${MANIFEST_VERSION} manifest` };
+ }
+ return { members: new Map(manifest.members.map((member) => [member.path, member])) };
+}
+
+function inspect() {
+ const manifest = readManifest();
+ if (manifest.problem)
+ return { problem: manifest.problem, missing: [], changed: [], unexpected: [] };
+ let actual;
+ try {
+ actual = inventory();
+ } catch (error) {
+ const where = error.path ? ` at ${error.path}` : '';
+ return {
+ problem: `dist could not be read (${error.code ?? error.message}${where})`,
+ missing: [],
+ changed: [],
+ unexpected: [],
+ };
+ }
+ const found = new Map(actual.map((member) => [member.path, member]));
+ const missing = [];
+ const changed = [];
+ for (const [path, expected] of manifest.members) {
+ const member = found.get(path);
+ if (!member) missing.push(path);
+ else if (
+ member.type !== expected.type ||
+ member.size !== expected.size ||
+ member.sha256 !== expected.sha256
+ )
+ changed.push(path);
+ }
+ const unexpected = actual
+ .filter((member) => !manifest.members.has(member.path))
+ .map((member) => member.path);
+ return { problem: undefined, missing, changed, unexpected };
+}
+
+const differs = (state) =>
+ state.problem !== undefined ||
+ state.missing.length + state.changed.length + state.unexpected.length > 0;
+
+function listed(paths) {
+ const shown = paths.slice(0, LISTED_PATHS).join(', ');
+ return paths.length > LISTED_PATHS ? `${shown} and ${paths.length - LISTED_PATHS} more` : shown;
+}
+
+function describe(state) {
+ if (state.problem) return state.problem;
+ return [
+ ['missing', state.missing],
+ ['changed', state.changed],
+ ['not in the manifest', state.unexpected],
+ ]
+ .filter(([, paths]) => paths.length > 0)
+ .map(([label, paths]) => `${label}: ${listed(paths)}`)
+ .join('; ');
+}
+
+function removeUnexpected(paths) {
+ const removed = [];
+ for (const path of [...paths].sort()) {
+ if (removed.some((parent) => path.startsWith(`${parent}/`))) continue;
+ rmSync(join(OUTPUT_ROOT, ...path.split('/')), { recursive: true, force: true });
+ removed.push(path);
+ }
+ return removed;
+}
+
+function run(label, args, cwd) {
+ const result = spawnSync(process.execPath, args, { cwd, stdio: 'inherit' });
+ if (result.error) throw result.error;
+ if (result.status !== 0)
+ throw new BuildOutputError(`${label} exited ${result.status ?? result.signal}`);
+}
+
+function produce() {
+ rmSync(OUTPUT_ROOT, { recursive: true, force: true });
+ run('tsdown', [fileURLToPath(import.meta.resolve('tsdown/run'))], PACKAGE_ROOT);
+ verifyExportsTargets(PACKAGE_ROOT);
+ writeFileSync(
+ MANIFEST_PATH,
+ `${JSON.stringify({ version: MANIFEST_VERSION, members: inventory() })}\n`,
+ );
+}
+
+function rebuildWithoutCacheReads(name) {
+ const turbo = createRequire(import.meta.url).resolve('turbo/bin/turbo');
+ const args = [turbo, 'run', PRODUCER_TASK, `--filter=${name}`, '--cache=local:w'];
+ if (process.env.TURBO_CACHE_DIR)
+ args.push(`--cache-dir=${resolve(TURBO_ROOT, process.env.TURBO_CACHE_DIR)}`);
+ run(`turbo run ${PRODUCER_TASK}`, args, TURBO_ROOT);
+}
+
+function complete() {
+ const name = packageName();
+ let state = inspect();
+ if (state.problem === undefined && state.unexpected.length > 0) {
+ const removed = removeUnexpected(state.unexpected);
+ process.stderr.write(
+ `${name}: removed ${listed(removed)} from dist; its manifest does not list them\n`,
+ );
+ state = { ...state, unexpected: [] };
+ }
+ if (!differs(state)) return;
+ process.stderr.write(
+ `${name}: dist differs from its manifest (${describe(state)}); rebuilding it once without reading the cache\n`,
+ );
+ rebuildWithoutCacheReads(name);
+ state = inspect();
+ if (differs(state))
+ throw new BuildOutputError(
+ `${name}: dist still differs from its manifest after one rebuild (${describe(state)})`,
+ );
+ process.stderr.write(`${name}: rebuilt dist matches its manifest\n`);
+}
+
+function verify() {
+ const state = inspect();
+ if (!differs(state)) return;
+ const name = packageName();
+ throw new BuildOutputError(
+ `${name}: dist differs from the manifest its producer wrote (${describe(state)}). Rebuild it through Turbo (\`turbo run build --filter=${name}\` from public/open-knowledge), or re-download it from the job that built it.`,
+ );
+}
+
+const MODES = { build: produce, produce, complete, verify };
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
+ const mode = process.argv[2];
+ if (!Object.hasOwn(MODES, mode)) {
+ process.stderr.write(
+ `usage: node scripts/build-output.mjs <${Object.keys(MODES).join('|')}>\n`,
+ );
+ process.exitCode = 2;
+ } else {
+ try {
+ MODES[mode]();
+ } catch (error) {
+ process.stderr.write(
+ error instanceof BuildOutputError || error instanceof ExportsTargetsError
+ ? `${error.message}\n`
+ : `build-output.mjs ${mode} failed: ${error instanceof Error ? error.stack : String(error)}\n`,
+ );
+ process.exitCode = 1;
+ }
+ }
+}
diff --git a/packages/core/scripts/build-output.uncached.test.ts b/packages/core/scripts/build-output.uncached.test.ts
new file mode 100644
index 000000000..864810f66
--- /dev/null
+++ b/packages/core/scripts/build-output.uncached.test.ts
@@ -0,0 +1,541 @@
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import {
+ copyFileSync,
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ symlinkSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { delimiter, dirname, join, relative } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { zstdCompressSync, zstdDecompressSync } from 'node:zlib';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+import { isTestOnlySourceFile } from '../../../test-support/test-only-source-file.mjs';
+import { withoutTurboAgentDetection } from '../../../test-support/turbo-agent-env.test-helper.mjs';
+import { MANIFEST_NAME } from './build-output.mjs';
+
+const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const OK_ROOT = join(PACKAGE_ROOT, '..', '..');
+const TURBO_BIN = join(OK_ROOT, 'node_modules', 'turbo', 'bin', 'turbo');
+const CORE_MANIFEST = JSON.parse(readFileSync(join(PACKAGE_ROOT, 'package.json'), 'utf8'));
+const OK_MANIFEST = JSON.parse(readFileSync(join(OK_ROOT, 'package.json'), 'utf8'));
+const CORE = CORE_MANIFEST.name as string;
+const REAL_TURBO = JSON.parse(readFileSync(join(OK_ROOT, 'turbo.json'), 'utf8'));
+const PRODUCER_TASK_ID = `${CORE}#build:output`;
+const COMPLETION_TASK_ID = `${CORE}#build`;
+const RUN_TIMEOUT_MS = 60_000;
+const ENV_CACHE_DIR = '.cache-from-env';
+const CHUNK_MEMBER = 'packages/core/dist/chunk.mjs';
+const MANIFEST_MEMBER = `packages/core/dist/${MANIFEST_NAME}`;
+const UNRELATED = 'packages/core/dist-unrelated/keep.txt';
+
+const FIXTURE_COMPILER = `import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+appendFileSync('../../compiles.log', 'compile\\n');
+if (existsSync('../../compile-fails')) {
+ process.stderr.write('fixture compiler: told to fail\\n');
+ process.exit(1);
+}
+const source = readFileSync('src/index.ts', 'utf8');
+mkdirSync('dist/types', { recursive: true });
+writeFileSync('dist/index.mjs', "import { chunk } from './chunk.mjs';\\nexport const source = " + JSON.stringify(source) + ';\\nexport { chunk };\\n');
+writeFileSync('dist/chunk.mjs', 'export const chunk = ' + JSON.stringify('x'.repeat(2048)) + ';\\n');
+writeFileSync('dist/index.d.mts', 'export declare const source: string;\\nexport declare const chunk: string;\\n');
+writeFileSync('dist/types/index.d.mts', 'export {};\\n');
+`;
+
+const FIXTURE_READER = `import { createHash } from 'node:crypto';
+import { appendFileSync, readdirSync, readFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+const root = '../core/dist';
+const tree = readdirSync(root, { recursive: true, withFileTypes: true })
+ .map((entry) => {
+ const path = relative(root, join(entry.parentPath, entry.name)).split('\\\\').join('/');
+ if (!entry.isFile()) return { path, type: entry.isDirectory() ? 'directory' : 'other' };
+ return { path, type: 'file', sha256: createHash('sha256').update(readFileSync(join(root, path))).digest('hex') };
+ })
+ .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+appendFileSync('../../reads.jsonl', JSON.stringify(tree) + '\\n');
+`;
+
+type TreeEntry = { path: string; type: string; sha256?: string };
+type TaskSummary = { taskId: string; hash: string; cache: { status: string; source?: string } };
+type Run = {
+ status: number | null;
+ output: string;
+ tasks: Map<string, TaskSummary>;
+ reads: TreeEntry[][];
+ compiles: number;
+};
+
+const workspaces: string[] = [];
+
+afterAll(() => {
+ for (const root of workspaces) rmSync(root, { recursive: true, force: true, maxRetries: 3 });
+});
+
+function writeJson(path: string, value: unknown) {
+ mkdirSync(dirname(path), { recursive: true });
+ writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function createWorkspace(): string {
+ const root = mkdtempSync(join(tmpdir(), 'ok-core-build-output-'));
+ workspaces.push(root);
+ const tasks: Record<string, unknown> = { build: REAL_TURBO.tasks.build };
+ for (const [key, definition] of Object.entries(REAL_TURBO.tasks)) {
+ if (key.startsWith(`${CORE}#`)) tasks[key] = definition;
+ }
+ tasks['reader#read'] = { dependsOn: ['^build'], cache: false, command: ['node', 'read.mjs'] };
+ const topLevel = Object.fromEntries(
+ Object.entries(REAL_TURBO).filter(
+ ([key]) => !['$schema', 'globalDependencies', 'tasks'].includes(key),
+ ),
+ );
+ writeJson(join(root, 'turbo.json'), { ...topLevel, tasks });
+ writeJson(join(root, 'package.json'), {
+ name: 'core-build-output-fixture',
+ private: true,
+ packageManager: OK_MANIFEST.packageManager,
+ });
+ writeFileSync(join(root, 'pnpm-workspace.yaml'), "packages:\n - 'packages/*'\n");
+ writeFileSync(
+ join(root, '.gitignore'),
+ ['node_modules', 'dist', 'dist-unrelated', '.turbo', ENV_CACHE_DIR, ''].join('\n'),
+ );
+ mkdirSync(join(root, 'node_modules'), { recursive: true });
+ symlinkSync(
+ join(OK_ROOT, 'node_modules', 'turbo'),
+ join(root, 'node_modules', 'turbo'),
+ 'junction',
+ );
+
+ const core = join(root, 'packages', 'core');
+ writeJson(join(core, 'package.json'), {
+ name: CORE,
+ private: true,
+ type: 'module',
+ exports: { '.': { types: './dist/index.d.mts', default: './dist/index.mjs' } },
+ scripts: { build: CORE_MANIFEST.scripts.build },
+ });
+ mkdirSync(join(core, 'scripts'), { recursive: true });
+ for (const name of readdirSync(join(PACKAGE_ROOT, 'scripts'))) {
+ if (name.endsWith('.mjs') && !isTestOnlySourceFile(name)) {
+ copyFileSync(join(PACKAGE_ROOT, 'scripts', name), join(core, 'scripts', name));
+ }
+ }
+ mkdirSync(join(core, 'src'), { recursive: true });
+ writeFileSync(join(core, 'src', 'index.ts'), 'export const value = 1;\n');
+ writeJson(join(core, 'node_modules', 'tsdown', 'package.json'), {
+ name: 'tsdown',
+ type: 'module',
+ exports: { './run': './run.mjs' },
+ });
+ writeFileSync(join(core, 'node_modules', 'tsdown', 'run.mjs'), FIXTURE_COMPILER);
+ mkdirSync(dirname(join(root, UNRELATED)), { recursive: true });
+ writeFileSync(join(root, UNRELATED), 'not core output\n');
+
+ writeJson(join(root, 'packages', 'reader', 'package.json'), {
+ name: 'reader',
+ private: true,
+ dependencies: { [CORE]: 'workspace:*' },
+ });
+ writeFileSync(join(root, 'packages', 'reader', 'read.mjs'), FIXTURE_READER);
+ return root;
+}
+
+function turboEnv(extra: Record<string, string>) {
+ const env = Object.fromEntries(
+ Object.entries(withoutTurboAgentDetection(process.env)).filter(
+ ([key]) => !key.startsWith('TURBO_') && !key.startsWith('GIT_'),
+ ),
+ );
+ return { ...env, TURBO_TELEMETRY_DISABLED: '1', TURBO_NO_UPDATE_NOTIFIER: '1', ...extra };
+}
+
+function lines(path: string): string[] {
+ return existsSync(path) ? readFileSync(path, 'utf8').split('\n').filter(Boolean) : [];
+}
+
+function readerRun(
+ root: string,
+ env: Record<string, string> = { TURBO_CACHE_DIR: ENV_CACHE_DIR },
+): Run {
+ rmSync(join(root, '.turbo', 'runs'), { recursive: true, force: true });
+ rmSync(join(root, 'reads.jsonl'), { force: true });
+ const compilesBefore = lines(join(root, 'compiles.log')).length;
+ const result = spawnSync(
+ process.execPath,
+ [
+ TURBO_BIN,
+ 'run',
+ 'read',
+ '--filter=reader',
+ '--summarize',
+ '--output-logs=full',
+ '--log-prefix=task',
+ '--concurrency=1',
+ ],
+ {
+ cwd: root,
+ env: turboEnv(env),
+ encoding: 'utf8',
+ timeout: RUN_TIMEOUT_MS,
+ maxBuffer: 16 * 1024 * 1024,
+ },
+ );
+ if (result.error) throw result.error;
+ const output = `${result.stdout}${result.stderr}`;
+ const runsDir = join(root, '.turbo', 'runs');
+ const summaries = existsSync(runsDir)
+ ? readdirSync(runsDir).filter((name) => name.endsWith('.json'))
+ : [];
+ if (summaries.length !== 1)
+ throw new Error(`expected one run summary, found ${summaries.length}: ${output}`);
+ const summary = JSON.parse(readFileSync(join(runsDir, summaries[0]), 'utf8'));
+ return {
+ status: result.status,
+ output,
+ tasks: new Map((summary.tasks as TaskSummary[]).map((task) => [task.taskId, task])),
+ reads: lines(join(root, 'reads.jsonl')).map((line) => JSON.parse(line)),
+ compiles: lines(join(root, 'compiles.log')).length - compilesBefore,
+ };
+}
+
+function completionLines(run: Run): string[] {
+ return run.output
+ .split('\n')
+ .filter((line) => line.startsWith(`${COMPLETION_TASK_ID.replace('#', ':')}: `))
+ .filter((line) => !line.includes('cache bypass, force executing'));
+}
+
+const completionLine = (message: string) => `${CORE}:build: ${CORE}: ${message}`;
+
+function producerCache(run: Run) {
+ const producer = run.tasks.get(PRODUCER_TASK_ID);
+ if (!producer) throw new Error(`no ${PRODUCER_TASK_ID} in the run summary: ${run.output}`);
+ return { status: producer.cache.status, source: producer.cache.source };
+}
+
+function distTree(root: string): TreeEntry[] {
+ const dist = join(root, 'packages', 'core', 'dist');
+ return readdirSync(dist, { recursive: true, withFileTypes: true })
+ .map((entry) => {
+ const path = relative(dist, join(entry.parentPath, entry.name)).split('\\').join('/');
+ if (!entry.isFile()) return { path, type: entry.isDirectory() ? 'directory' : 'other' };
+ return {
+ path,
+ type: 'file',
+ sha256: createHash('sha256')
+ .update(readFileSync(join(dist, path)))
+ .digest('hex'),
+ };
+ })
+ .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+}
+
+type TarEntry = { header: Buffer; data: Buffer };
+
+function archivePath(root: string, cacheDir: string, hash: string) {
+ return join(root, cacheDir, `${hash}.tar.zst`);
+}
+
+function readArchive(path: string) {
+ const tar = zstdDecompressSync(readFileSync(path));
+ const entries: { name: string; entry: TarEntry }[] = [];
+ let offset = 0;
+ while (
+ offset + 512 <= tar.length &&
+ tar.subarray(offset, offset + 512).some((byte) => byte !== 0)
+ ) {
+ const header = Buffer.from(tar.subarray(offset, offset + 512));
+ const field = (start: number, length: number) =>
+ header
+ .subarray(start, start + length)
+ .toString('utf8')
+ .replace(/\0.*$/s, '');
+ const type = field(156, 1);
+ if (!['', '0', '2', '5'].includes(type))
+ throw new Error(`unsupported tar entry type ${JSON.stringify(type)} in ${path}`);
+ const prefix =
+ header.subarray(257, 263).toString('latin1') === 'ustar\0' ? field(345, 155) : '';
+ const name = prefix ? `${prefix}/${field(0, 100)}` : field(0, 100);
+ const size = Number.parseInt(field(124, 12).trim() || '0', 8);
+ const data = Buffer.from(tar.subarray(offset + 512, offset + 512 + size));
+ entries.push({ name: name.replace(/\/$/, ''), entry: { header, data } });
+ offset += 512 + Math.ceil(size / 512) * 512;
+ }
+ return entries;
+}
+
+function writeArchive(path: string, entries: { entry: TarEntry }[]) {
+ const blocks: Buffer[] = [];
+ for (const { entry } of entries) {
+ const header = Buffer.from(entry.header);
+ header.write(`${entry.data.length.toString(8).padStart(11, '0')}\0`, 124, 'ascii');
+ header.fill(0x20, 148, 156);
+ const checksum = header.reduce((sum, byte) => sum + byte, 0);
+ header.write(`${checksum.toString(8).padStart(6, '0')}\0 `, 148, 'ascii');
+ blocks.push(header, entry.data, Buffer.alloc((512 - (entry.data.length % 512)) % 512));
+ }
+ blocks.push(Buffer.alloc(1024));
+ writeFileSync(path, zstdCompressSync(Buffer.concat(blocks)));
+}
+
+type Damage = 'missing' | 'truncated' | 'same-length' | { replace: string };
+
+function damageArchive(path: string, damage: Damage, member = CHUNK_MEMBER) {
+ const entries = readArchive(path);
+ const index = entries.findIndex(({ name }) => name === member);
+ if (index === -1)
+ throw new Error(`${member} is not in ${path}: ${entries.map(({ name }) => name).join(', ')}`);
+ const { entry } = entries[index];
+ if (damage === 'missing') entries.splice(index, 1);
+ else if (damage === 'truncated') entry.data = entry.data.subarray(0, 7);
+ else if (damage === 'same-length')
+ entry.data = Buffer.concat([Buffer.from('/'), entry.data.subarray(1)]);
+ else entry.data = Buffer.from(damage.replace);
+ writeArchive(path, entries);
+}
+
+function removeDist(root: string) {
+ rmSync(join(root, 'packages', 'core', 'dist'), { recursive: true, force: true });
+}
+
+describe('core build output reaches its readers exactly as the producer made it', () => {
+ let root: string;
+ let golden: TreeEntry[];
+ let producerHash: string;
+
+ const expectUnrelatedFileSurvives = () =>
+ expect(readFileSync(join(root, UNRELATED), 'utf8')).toBe('not core output\n');
+
+ const expectQuietHitNext = () => {
+ const next = readerRun(root);
+ expect(next.status, next.output).toBe(0);
+ expect(producerCache(next)).toEqual({ status: 'HIT', source: 'LOCAL' });
+ expect(next.compiles).toBe(0);
+ expect(next.reads).toEqual([golden]);
+ };
+
+ beforeAll(() => {
+ root = createWorkspace();
+ const cold = readerRun(root);
+ expect(cold.status, cold.output).toBe(0);
+ expect(producerCache(cold).status).toBe('MISS');
+ expect(cold.compiles).toBe(1);
+ golden = distTree(root);
+ expect(golden.map(({ path }) => path)).toContain(MANIFEST_NAME);
+ expect(cold.reads).toEqual([golden]);
+ producerHash = cold.tasks.get(PRODUCER_TASK_ID)?.hash ?? '';
+ expect(existsSync(archivePath(root, ENV_CACHE_DIR, producerHash))).toBe(true);
+ }, RUN_TIMEOUT_MS * 2);
+
+ test(
+ 'an unchanged valid hit runs no producer and prints nothing',
+ () => {
+ const run = readerRun(root);
+ expect(run.status, run.output).toBe(0);
+ expect(producerCache(run)).toEqual({ status: 'HIT', source: 'LOCAL' });
+ expect(run.compiles).toBe(0);
+ expect(completionLines(run)).toEqual([]);
+ expect(run.reads).toEqual([golden]);
+ expectUnrelatedFileSurvives();
+ },
+ RUN_TIMEOUT_MS,
+ );
+
+ test(
+ 'extra files beside a valid hit are removed before the reader runs, without a rebuild',
+ () => {
+ const dist = join(root, 'packages', 'core', 'dist');
+ writeFileSync(join(dist, 'planted.mjs'), 'export const planted = true;\n');
+ mkdirSync(join(dist, 'planted-dir'), { recursive: true });
+ writeFileSync(join(dist, 'planted-dir', 'inner.mjs'), 'export {};\n');
+ const run = readerRun(root);
+ expect(run.reads).toEqual([golden]);
+ expect(run.status, run.output).toBe(0);
+ expect(producerCache(run)).toEqual({ status: 'HIT', source: 'LOCAL' });
+ expect(run.compiles).toBe(0);
+ expect(completionLines(run)).toEqual([
+ completionLine(
+ 'removed planted-dir, planted.mjs from dist; its manifest does not list them',
+ ),
+ ]);
+ expectUnrelatedFileSurvives();
+ },
+ RUN_TIMEOUT_MS,
+ );
+
+ test.each([
+ ['missing member', CHUNK_MEMBER, 'missing', 'missing: chunk.mjs'],
+ ['truncated member', CHUNK_MEMBER, 'truncated', 'changed: chunk.mjs'],
+ ['same-length member', CHUNK_MEMBER, 'same-length', 'changed: chunk.mjs'],
+ ['missing manifest', MANIFEST_MEMBER, 'missing', `dist/${MANIFEST_NAME} is missing`],
+ ['truncated manifest', MANIFEST_MEMBER, 'truncated', `dist/${MANIFEST_NAME} is unreadable (`],
+ [
+ 'manifest with a non-object member',
+ MANIFEST_MEMBER,
+ { replace: '{"version":1,"members":[null]}\n' },
+ `dist/${MANIFEST_NAME} is not a version 1 manifest`,
+ ],
+ ] as const)(
+ 'a %s restored from the local cache is rebuilt once before the reader runs, and the cache entry is replaced',
+ (_label, member, damage, difference) => {
+ const archive = archivePath(root, ENV_CACHE_DIR, producerHash);
+ const intact = readFileSync(archive);
+ damageArchive(archive, damage, member);
+ removeDist(root);
+ try {
+ const run = readerRun(root);
+ expect(run.reads).toEqual([golden]);
+ expect(run.status, run.output).toBe(0);
+ expect(producerCache(run)).toEqual({ status: 'HIT', source: 'LOCAL' });
+ expect(run.compiles).toBe(1);
+ expect(completionLines(run)).toContainEqual(
+ expect.stringContaining(completionLine(`dist differs from its manifest (${difference}`)),
+ );
+ expect(completionLines(run)).toContain(completionLine('rebuilt dist matches its manifest'));
+ expectUnrelatedFileSurvives();
+ expectQuietHitNext();
+ } finally {
+ writeFileSync(archive, intact);
+ removeDist(root);
+ }
+ },
+ RUN_TIMEOUT_MS * 2,
+ );
+
+ test(
+ 'a failed rebuild blocks the reader',
+ () => {
+ const archive = archivePath(root, ENV_CACHE_DIR, producerHash);
+ const intact = readFileSync(archive);
+ damageArchive(archive, 'same-length');
+ removeDist(root);
+ writeFileSync(join(root, 'compile-fails'), '');
+ try {
+ const run = readerRun(root);
+ expect(run.reads).toEqual([]);
+ expect(run.status, run.output).not.toBe(0);
+ expect(producerCache(run)).toEqual({ status: 'HIT', source: 'LOCAL' });
+ expect(run.compiles).toBe(1);
+ expectUnrelatedFileSurvives();
+ } finally {
+ rmSync(join(root, 'compile-fails'), { force: true });
+ writeFileSync(archive, intact);
+ removeDist(root);
+ }
+ expectQuietHitNext();
+ },
+ RUN_TIMEOUT_MS * 2,
+ );
+
+ test.each([
+ ['missing member', (dist: string) => rmSync(join(dist, 'chunk.mjs')), 'missing: chunk.mjs'],
+ [
+ 'same-length change',
+ (dist: string) => {
+ const bytes = readFileSync(join(dist, 'chunk.mjs'));
+ bytes[0] = 0x2f;
+ writeFileSync(join(dist, 'chunk.mjs'), bytes);
+ },
+ 'changed: chunk.mjs',
+ ],
+ [
+ 'file its manifest does not list',
+ (dist: string) => writeFileSync(join(dist, 'planted.mjs'), 'export const planted = true;\n'),
+ 'not in the manifest: planted.mjs',
+ ],
+ [
+ 'missing manifest',
+ (dist: string) => rmSync(join(dist, MANIFEST_NAME)),
+ `dist/${MANIFEST_NAME} is missing`,
+ ],
+ ] as const)(
+ 'verify rejects a dist with a %s, names it, and changes nothing',
+ (_label, damage, difference) => {
+ expectQuietHitNext();
+ const core = join(root, 'packages', 'core');
+ damage(join(core, 'dist'));
+ try {
+ const damaged = distTree(root);
+ expect(damaged).not.toEqual(golden);
+ const compilesBefore = lines(join(root, 'compiles.log')).length;
+ const verify = spawnSync(process.execPath, ['scripts/build-output.mjs', 'verify'], {
+ cwd: core,
+ env: turboEnv({}),
+ encoding: 'utf8',
+ });
+ expect(verify.status, `${verify.stdout}${verify.stderr}`).toBe(1);
+ expect(verify.stderr).toContain(
+ `${CORE}: dist differs from the manifest its producer wrote (${difference}). Rebuild it through Turbo`,
+ );
+ expect(distTree(root)).toEqual(damaged);
+ expect(lines(join(root, 'compiles.log')).length).toBe(compilesBefore);
+ expectUnrelatedFileSurvives();
+ } finally {
+ removeDist(root);
+ }
+ },
+ RUN_TIMEOUT_MS,
+ );
+
+ test(
+ 'a repair rewrites the cache entry in the configured cache directory when no TURBO_CACHE_DIR is set',
+ () => {
+ const configured = REAL_TURBO.cacheDir as string;
+ const configRun = () => readerRun(root, {});
+ const cold = configRun();
+ expect(cold.status, cold.output).toBe(0);
+ const archive = archivePath(root, configured, cold.tasks.get(PRODUCER_TASK_ID)?.hash ?? '');
+ damageArchive(archive, 'truncated');
+ removeDist(root);
+ const run = configRun();
+ expect(run.reads).toEqual([golden]);
+ expect(run.status, run.output).toBe(0);
+ expect(producerCache(run)).toEqual({ status: 'HIT', source: 'LOCAL' });
+ expect(run.compiles).toBe(1);
+ const next = configRun();
+ expect(producerCache(next)).toEqual({ status: 'HIT', source: 'LOCAL' });
+ expect(next.compiles).toBe(0);
+ expect(next.reads).toEqual([golden]);
+ },
+ RUN_TIMEOUT_MS * 3,
+ );
+
+ test(
+ "core's own build script rebuilds from source when run directly",
+ () => {
+ const core = join(root, 'packages', 'core');
+ writeFileSync(join(core, 'src', 'index.ts'), 'export const value = 2;\n');
+ const env = turboEnv({
+ PATH: `${join(core, 'node_modules', '.bin')}${delimiter}${process.env.PATH ?? ''}`,
+ });
+ const direct = spawnSync(CORE_MANIFEST.scripts.build, {
+ cwd: core,
+ env,
+ shell: true,
+ encoding: 'utf8',
+ timeout: RUN_TIMEOUT_MS,
+ });
+ expect(direct.status, `${direct.stdout}${direct.stderr}`).toBe(0);
+ expect(readFileSync(join(core, 'dist', 'index.mjs'), 'utf8')).toContain(
+ 'export const value = 2;',
+ );
+ const verify = spawnSync(process.execPath, ['scripts/build-output.mjs', 'verify'], {
+ cwd: core,
+ env,
+ encoding: 'utf8',
+ });
+ expect(verify.status, `${verify.stdout}${verify.stderr}`).toBe(0);
+ expectUnrelatedFileSurvives();
+ },
+ RUN_TIMEOUT_MS,
+ );
+});
diff --git a/packages/core/scripts/verify-exports-targets.mjs b/packages/core/scripts/verify-exports-targets.mjs
index 869e3bf18..2f406c32b 100644
--- a/packages/core/scripts/verify-exports-targets.mjs
+++ b/packages/core/scripts/verify-exports-targets.mjs
@@ -6,6 +6,8 @@ const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
export const REQUIRED_CONDITIONS = ['types', 'default'];
+export class ExportsTargetsError extends Error {}
+
const RELATIVE_IMPORT = /\bfrom\s*["'](\.\.?\/[^"']+)["']|\bimport\s*["'](\.\.?\/[^"']+)["']/g;
export function nestedRelativeImports(source) {
@@ -117,7 +119,7 @@ export function verifyExportsTargets(packageRoot = PACKAGE_ROOT) {
}
if (errors.length > 0) {
- throw new Error(
+ throw new ExportsTargetsError(
`${packageJson.name} exports targets are not all on disk or not flat. Run \`pnpm --filter ${packageJson.name} build\` and check tsdown's entry map against package.json exports.\n- ${errors.join('\n- ')}`,
);
}
diff --git a/packages/desktop/package.json b/packages/desktop/package.json
index 913c976fa..20d005bfa 100644
--- a/packages/desktop/package.json
+++ b/packages/desktop/package.json
@@ -18,7 +18,7 @@
"scripts": {
"dev": "turbo run dev:electron --filter=@inkeep/open-knowledge-desktop",
"dev:electron": "electron-vite dev --watch",
- "build:desktop": "electron-vite build",
+ "build:desktop": "node ../core/scripts/build-output.mjs verify && electron-vite build",
"build:dir": "pnpm run build:desktop && node scripts/run-electron-builder.mjs --dir --publish never",
"build:mac": "pnpm run build:desktop && node scripts/prepare-universal.mjs && node scripts/stage-parcel-watcher.mjs && node scripts/run-electron-builder.mjs --mac --publish never",
"build:mac:unsigned": "pnpm run build:desktop && node scripts/prepare-universal.mjs && node scripts/stage-parcel-watcher.mjs && CSC_IDENTITY_AUTO_DISCOVERY=false node scripts/run-electron-builder.mjs --mac --publish never -c.mac.identity=null",
diff --git a/turbo.json b/turbo.json
index 09e48e825..a0a9aea5a 100644
--- a/turbo.json
+++ b/turbo.json
@@ -2,6 +2,9 @@
"$schema": "https://turbo.build/schema.json",
"cacheDir": ".turbo/cache",
"agentGuidance": false,
+ "futureFlags": {
+ "experimentalTaskCommand": true
+ },
"remoteCache": {
"signature": true
},
@@ -19,6 +22,16 @@
"dependsOn": ["^build"],
"outputs": ["dist/**", ".next/**", "!.next/cache/**", "!.next/dev/**"]
},
+ "@inkeep/open-knowledge-core#build:output": {
+ "dependsOn": ["^build"],
+ "outputs": ["dist/**"],
+ "command": ["node", "scripts/build-output.mjs", "produce"]
+ },
+ "@inkeep/open-knowledge-core#build": {
+ "dependsOn": ["build:output"],
+ "cache": false,
+ "command": ["node", "scripts/build-output.mjs", "complete"]
+ },
"build:desktop": {
"dependsOn": ["^build", "@inkeep/open-knowledge#build"],
"outputs": ["out/**"]