Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

perf(scripts): the knowledge and release checks read a commit's files in one git process (#9889)

Christopher Chedeaub93ee919ae53
diff --git a/scripts/check-knowledge.mjs b/scripts/check-knowledge.mjs
index d486d78f7f..ade3989a15 100644
--- a/scripts/check-knowledge.mjs
+++ b/scripts/check-knowledge.mjs
@@ -15,6 +15,7 @@ import {createHash} from 'node:crypto';
import {createRequire} from 'node:module';
import {fileURLToPath} from 'node:url';
import {collectThemingTargets} from '../packages/cli/foundation/discovery/theming-targets.mjs';
+import {readObjects} from './lib/git-objects.mjs';
const require = createRequire(import.meta.url);
const {
@@ -792,21 +793,36 @@ export function readKnowledgeRecordsAtAuthority(
authorityCommit,
reviewHead,
) {
- const resolvedCommit = execFileSync(
+ // One process resolves both commits, and fails when either does not.
+ const resolved = execFileSync(
'git',
- ['-C', root, 'rev-parse', '--verify', `${authorityCommit}^{commit}`],
+ [
+ '-C',
+ root,
+ 'rev-parse',
+ `${authorityCommit}^{commit}`,
+ `${reviewHead}^{commit}`,
+ ],
{encoding: 'utf8'},
- ).trim();
+ )
+ .trim()
+ .split('\n');
+ // Without --verify, rev-parse echoes an option and expands a range, so
+ // only two object ids count as a resolution.
+ if (
+ resolved.length !== 2 ||
+ !resolved.every(line => /^[0-9a-f]{40,64}$/.test(line))
+ ) {
+ throw new Error(
+ `cannot resolve ${authorityCommit} and ${reviewHead} to two commits.`,
+ );
+ }
+ const [resolvedCommit, resolvedHead] = resolved;
if (resolvedCommit !== authorityCommit) {
throw new Error(
`authority commit must be the full resolved commit ${resolvedCommit}.`,
);
}
- const resolvedHead = execFileSync(
- 'git',
- ['-C', root, 'rev-parse', '--verify', `${reviewHead}^{commit}`],
- {encoding: 'utf8'},
- ).trim();
if (resolvedHead !== reviewHead) {
throw new Error(
`review head must be the full resolved commit ${resolvedHead}.`,
@@ -845,14 +861,17 @@ export function readKnowledgeRecordsAtAuthority(
)
.split(/\r?\n/)
.filter(isKnowledgeRecordPath);
- const records = paths.map(filePath => ({
- path: filePath,
- content: execFileSync(
- 'git',
- ['-C', root, 'show', `${resolvedCommit}:${filePath}`],
- {encoding: 'utf8'},
- ),
- }));
+ const objects = readObjects(
+ root,
+ paths.map(filePath => `${resolvedCommit}:${filePath}`),
+ );
+ const records = paths.map(filePath => {
+ const content = objects.get(`${resolvedCommit}:${filePath}`);
+ if (!content) {
+ throw new Error(`cannot read ${filePath} at ${resolvedCommit}.`);
+ }
+ return {path: filePath, content: content.toString('utf8')};
+ });
return {records, authority: resolvedCommit};
}
diff --git a/scripts/lib/git-objects.mjs b/scripts/lib/git-objects.mjs
new file mode 100644
index 0000000000..a21a2d08f5
--- /dev/null
+++ b/scripts/lib/git-objects.mjs
@@ -0,0 +1,46 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Many files out of git in one process.
+ * @input A repository and object names such as `<commit>:<path>`.
+ * @output Each name's bytes, or null when git holds no such object.
+ * @position check-knowledge reads every knowledge record at a review's
+ * authority commit through it: one `git cat-file --batch` instead of one
+ * `git show` per record. Each git command is a process launch, which a
+ * loaded machine makes slow.
+ */
+import {execFileSync} from 'node:child_process';
+
+// A found object's header line: its id, type and size in bytes.
+const FOUND = /^[0-9a-f]{40,64} [a-z]+ (\d+)$/;
+
+/**
+ * Reads every object `names` lists with one `git cat-file --batch`.
+ * @param {string} root
+ * @param {string[]} names
+ * @returns {Map<string, Buffer | null>}
+ */
+export function readObjects(root, names) {
+ /** @type {Map<string, Buffer | null>} */
+ const objects = new Map();
+ if (names.length === 0) return objects;
+ const out = execFileSync('git', ['-C', root, 'cat-file', '--batch'], {
+ input: names.map(name => `${name}\n`).join(''),
+ maxBuffer: 1024 * 1024 * 1024,
+ });
+ let at = 0;
+ for (const name of names) {
+ const end = out.indexOf(0x0a, at);
+ const found = FOUND.exec(out.toString('utf8', at, end));
+ at = end + 1;
+ if (found === null) {
+ objects.set(name, null);
+ continue;
+ }
+ const size = Number(found[1]);
+ objects.set(name, out.subarray(at, at + size));
+ // A line feed follows each object's bytes.
+ at += size + 1;
+ }
+ return objects;
+}
diff --git a/scripts/lib/git-objects.test.mjs b/scripts/lib/git-objects.test.mjs
new file mode 100644
index 0000000000..2a2d67d488
--- /dev/null
+++ b/scripts/lib/git-objects.test.mjs
@@ -0,0 +1,124 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {execFileSync} from 'node:child_process';
+import {createHash} from 'node:crypto';
+import fs from 'node:fs';
+import {createRequire} from 'node:module';
+import os from 'node:os';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {afterEach, describe, expect, it} from 'vitest';
+import {readObjects} from './git-objects.mjs';
+import {filesAtRef} from '../release/active-release.mjs';
+
+const require = createRequire(import.meta.url);
+const {
+ isKnowledgeRecordPath,
+} = require('../../.github/scripts/knowledge-paths.cjs');
+const REPO_ROOT = path.resolve(
+ path.dirname(fileURLToPath(import.meta.url)),
+ '../..',
+);
+
+/** @type {string[]} */
+const roots = [];
+afterEach(() => {
+ for (const root of roots.splice(0))
+ fs.rmSync(root, {recursive: true, force: true});
+});
+
+/** @param {string} root @param {string[]} args */
+const git = (root, ...args) =>
+ execFileSync('git', ['-C', root, ...args], {maxBuffer: 1024 * 1024 * 1024});
+
+/**
+ * The id git names a blob by: the hash of its header and bytes.
+ * @param {string} format the repository's object format
+ * @param {Buffer} content
+ */
+function blobId(format, content) {
+ return createHash(format === 'sha256' ? 'sha256' : 'sha1')
+ .update(`blob ${content.length}\0`)
+ .update(content)
+ .digest('hex');
+}
+
+// The two batched readers: check-knowledge's and the release check's own.
+const READERS = {
+ readObjects: (root, ref, files) => {
+ const objects = readObjects(
+ root,
+ files.map(file => `${ref}:${file}`),
+ );
+ return new Map(files.map(file => [file, objects.get(`${ref}:${file}`)]));
+ },
+ filesAtRef,
+};
+
+describe.each(Object.entries(READERS))('%s', (_, read) => {
+ it('reads each file as `git show` reads it, and null where git has none', () => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'git-objects-'));
+ roots.push(root);
+ const files = {
+ 'a.txt': Buffer.from('one\ntwo\n'),
+ 'my dir/space name.md': Buffer.from('# Ünïcode — ok\n'),
+ 'empty.txt': Buffer.alloc(0),
+ 'binary.dat': Buffer.from([0, 10, 255, 10, 0, 1]),
+ };
+ for (const [file, content] of Object.entries(files)) {
+ fs.mkdirSync(path.dirname(path.join(root, file)), {recursive: true});
+ fs.writeFileSync(path.join(root, file), content);
+ }
+ git(root, 'init', '-q');
+ git(root, 'add', '-A');
+ git(
+ root,
+ '-c',
+ 'user.name=T',
+ '-c',
+ 'commit',
+ '-qm',
+ 'files',
+ );
+
+ const read1 = read(root, 'HEAD', [...Object.keys(files), 'missing.txt']);
+ for (const file of Object.keys(files)) {
+ expect(read1.get(file), file).toEqual(git(root, 'show', `HEAD:${file}`));
+ }
+ expect(read1.get('missing.txt')).toBeNull();
+ expect(read(root, 'no-such-ref', ['a.txt']).get('a.txt')).toBeNull();
+ });
+
+ it('reads every knowledge record of this repository at HEAD byte for byte', () => {
+ const listing = git(
+ REPO_ROOT,
+ 'ls-tree',
+ '-r',
+ 'HEAD',
+ '--',
+ 'docs',
+ 'packages',
+ )
+ .toString()
+ .split('\n')
+ .filter(Boolean)
+ .map(line => {
+ const [meta, file] = line.split('\t');
+ return {id: meta.split(' ')[2], file};
+ })
+ .filter(({file}) => isKnowledgeRecordPath(file));
+ expect(listing.length).toBeGreaterThan(100);
+ const format = git(REPO_ROOT, 'rev-parse', '--show-object-format')
+ .toString()
+ .trim();
+ const contents = read(
+ REPO_ROOT,
+ 'HEAD',
+ listing.map(({file}) => file),
+ );
+ for (const {id, file} of listing) {
+ expect(blobId(format, contents.get(file)), file).toBe(id);
+ }
+ });
+});
diff --git a/scripts/lib/review-diff.test.mjs b/scripts/lib/review-diff.test.mjs
index ec7f6a7e1f..aeac7c4c92 100644
--- a/scripts/lib/review-diff.test.mjs
+++ b/scripts/lib/review-diff.test.mjs
@@ -8,6 +8,7 @@ import {afterEach, describe, expect, it} from 'vitest';
import {readDiff} from './review-diff.mjs';
import {collectConcernMatches} from '../check-knowledge.mjs';
import {loadKnowledgeIndex} from '../specs-for-file.mjs';
+import {routeReviewBaselines} from '../review-global-baselines.mjs';
const HERE = import.meta.dirname;
const ROOT = path.resolve(HERE, '../..');
@@ -321,12 +322,17 @@ describe('review-global-baselines.mjs', () => {
);
}
+ // The router runs in this process; the case below runs the command once.
it("routes a concern the head's added lines match, beside the requested triggers", () => {
- const result = route(
- repository(
- 'export function A() {\n useEffect(() => () => {}, []);\n}\n',
- ),
+ const {root, base, head} = repository(
+ 'export function A() {\n useEffect(() => () => {}, []);\n}\n',
);
+ const result = routeReviewBaselines({
+ root,
+ authorityCommit: base,
+ reviewHead: head,
+ requestedTriggers: ['styling'],
+ });
expect(result.matches.map(match => match.claimId)).toEqual(['INV2']);
expect(result.concernMatches).toEqual([
expect.objectContaining({
diff --git a/scripts/release/active-release.mjs b/scripts/release/active-release.mjs
index ce7cd34efa..9d8963441e 100644
--- a/scripts/release/active-release.mjs
+++ b/scripts/release/active-release.mjs
@@ -375,9 +375,57 @@ function latestStableVersion(root) {
return versions.at(-1) ?? null;
}
-function fileDigestAtRef(root, ref, file) {
+// A found object's header line in `git cat-file --batch` output: its id,
+// type and size in bytes.
+const FOUND_OBJECT = /^[0-9a-f]{40,64} [a-z]+ (\d+)$/;
+
+/**
+ * Each file's bytes at `ref`, or null when the ref lacks it, read with one
+ * `git cat-file --batch` instead of one `git show` per file. This module
+ * imports nothing beyond its own lib (see the header), so it carries its own
+ * reader.
+ * @param {string} root
+ * @param {string} ref
+ * @param {string[]} files
+ * @returns {Map<string, Buffer | null>}
+ */
+function filesAtRef(root, ref, files) {
+ const contents = new Map();
+ if (files.length === 0) return contents;
+ const out = execFileSync('git', ['cat-file', '--batch'], {
+ cwd: root,
+ input: files.map(file => `${ref}:${file}\n`).join(''),
+ maxBuffer: 1024 * 1024 * 1024,
+ });
+ let at = 0;
+ for (const file of files) {
+ const end = out.indexOf(0x0a, at);
+ const found = FOUND_OBJECT.exec(out.toString('utf8', at, end));
+ at = end + 1;
+ if (found === null) {
+ contents.set(file, null);
+ continue;
+ }
+ const size = Number(found[1]);
+ contents.set(file, out.subarray(at, at + size));
+ at += size + 1;
+ }
+ return contents;
+}
+
+function fileDigestsAtRef(root, ref, files) {
+ const contents = filesAtRef(root, ref, files);
+ return new Map(
+ files.map(file => {
+ const content = contents.get(file);
+ return [file, content ? sha256(content) : null];
+ }),
+ );
+}
+
+function parseJson(content) {
try {
- return sha256(execFileSync('git', ['show', `${ref}:${file}`], {cwd: root}));
+ return content ? JSON.parse(content.toString('utf8')) : null;
} catch {
return null;
}
@@ -392,7 +440,9 @@ function changesetMapAtRef(root, ref) {
'--',
'.changeset',
]);
- return new Map(
+ return fileDigestsAtRef(
+ root,
+ ref,
files
.split('\n')
.filter(
@@ -400,8 +450,7 @@ function changesetMapAtRef(root, ref) {
file.startsWith('.changeset/') &&
file.endsWith('.md') &&
file !== '.changeset/README.md',
- )
- .map(file => [file, fileDigestAtRef(root, ref, file)]),
+ ),
);
}
@@ -416,11 +465,12 @@ function releaseOutputPathsAtRef(root, plan, releaseRef) {
}
function releaseOutputMapAtRef(root, plan, releaseRef) {
- return new Map(
+ return fileDigestsAtRef(
+ root,
+ releaseRef,
releaseOutputPathsAtRef(root, plan, releaseRef)
.filter(isReleaseOutputPath)
- .filter(file => !isPackageManifestPath(file))
- .map(file => [file, fileDigestAtRef(root, releaseRef, file)]),
+ .filter(file => !isPackageManifestPath(file)),
);
}
@@ -439,7 +489,8 @@ function jsonAtRef(root, ref, file) {
}
function manifestMapAtRef(root, ref, files) {
- return new Map([...files].map(file => [file, jsonAtRef(root, ref, file)]));
+ const contents = filesAtRef(root, ref, [...files]);
+ return new Map([...files].map(file => [file, parseJson(contents.get(file))]));
}
function currentManifestMap(root, files) {
@@ -946,7 +997,12 @@ function versionTagExists(root, version) {
* @returns {{version: string|null, errors: string[]}}
*/
function declaredVersionAtRef(root, ref) {
- const config = jsonAtRef(root, ref, '.changeset/config.json');
+ const configFile = '.changeset/config.json';
+ const files = expandWorkspaceDirs(root).map(dir =>
+ path.relative(root, path.join(dir, 'package.json')),
+ );
+ const contents = filesAtRef(root, ref, [configFile, ...files]);
+ const config = parseJson(contents.get(configFile));
if (!config)
return {
version: null,
@@ -954,9 +1010,8 @@ function declaredVersionAtRef(root, ref) {
};
const fixed = new Set((config.fixed ?? []).flat());
const versions = new Map();
- for (const dir of expandWorkspaceDirs(root)) {
- const file = path.relative(root, path.join(dir, 'package.json'));
- const manifest = jsonAtRef(root, ref, file);
+ for (const file of files) {
+ const manifest = parseJson(contents.get(file));
if (!manifest || !fixed.has(manifest.name)) continue;
if (!isStableReleasePackage(manifest)) continue;
if (!versions.has(manifest.version)) versions.set(manifest.version, []);
@@ -1298,6 +1353,7 @@ export {
compareVersions,
computePlanDigest,
declaredVersionAtRef,
+ filesAtRef,
nextPlannedVersion,
isStableReleasePackage,
latestStableVersion,
diff --git a/scripts/review-global-baselines.mjs b/scripts/review-global-baselines.mjs
index a122f97ed3..abed22c36b 100644
--- a/scripts/review-global-baselines.mjs
+++ b/scripts/review-global-baselines.mjs
@@ -24,81 +24,102 @@ function valueFor(flag) {
return value;
}
-const root = path.resolve(valueFor('--root') ?? DEFAULT_ROOT);
-const authorityCommit = valueFor('--authority-commit');
-const reviewHead = valueFor('--review-head');
-const triggerValue = valueFor('--triggers');
-
-if (!authorityCommit) {
- throw new Error(
- '--authority-commit requires the full base authority commit.',
+/**
+ * The claims a review routes: those the requested triggers name at the
+ * authority commit, and those whose concerns the head's added lines match.
+ * @param {{root: string, authorityCommit: string, reviewHead: string,
+ * requestedTriggers: string[]}} options
+ */
+export function routeReviewBaselines({
+ root,
+ authorityCommit,
+ reviewHead,
+ requestedTriggers,
+}) {
+ const {records, authority} = readKnowledgeRecordsAtAuthority(
+ root,
+ authorityCommit,
+ reviewHead,
);
-}
-if (!/^[0-9a-f]{40}$/.test(authorityCommit)) {
- throw new Error(
- '--authority-commit must be a full 40-character commit hash.',
+ const matches = collectGlobalReviewMatches(
+ records,
+ requestedTriggers,
+ authority,
);
-}
-if (!reviewHead) {
- throw new Error('--review-head requires the full reviewed commit.');
-}
-if (!/^[0-9a-f]{40}$/.test(reviewHead)) {
- throw new Error('--review-head must be a full 40-character commit hash.');
-}
-if (!triggerValue) {
- throw new Error('--triggers requires a comma-separated trigger list.');
+ // The reviewed change's own added lines route the claims of every concern
+ // they re-implement (architecture:knowledge-contracts/INV20), whatever the
+ // caller's triggers or the component record's review_triggers.
+ const {addedLines} = readDiff(
+ execFileSync(
+ 'git',
+ [
+ '-C',
+ root,
+ 'diff',
+ '--no-color',
+ '--no-ext-diff',
+ '--src-prefix=a/',
+ '--dst-prefix=b/',
+ authority,
+ reviewHead,
+ ],
+ {encoding: 'utf8', maxBuffer: 256 * 1024 * 1024},
+ ),
+ );
+ return {
+ schemaVersion: 1,
+ authorityCommit,
+ reviewHead,
+ requestedTriggers: [...new Set(requestedTriggers)].sort(),
+ matches,
+ concernMatches: collectConcernMatches(records, addedLines, authority),
+ };
}
-const requestedTriggers = triggerValue
- .split(',')
- .map(value => value.trim())
- .filter(Boolean);
-if (requestedTriggers.length === 0) {
- throw new Error('--triggers requires at least one trigger.');
-}
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const root = path.resolve(valueFor('--root') ?? DEFAULT_ROOT);
+ const authorityCommit = valueFor('--authority-commit');
+ const reviewHead = valueFor('--review-head');
+ const triggerValue = valueFor('--triggers');
-const {records, authority} = readKnowledgeRecordsAtAuthority(
- root,
- authorityCommit,
- reviewHead,
-);
-const matches = collectGlobalReviewMatches(
- records,
- requestedTriggers,
- authority,
-);
-// The reviewed change's own added lines route the claims of every concern
-// they re-implement (architecture:knowledge-contracts/INV20), whatever the
-// caller's triggers or the component record's review_triggers.
-const {addedLines} = readDiff(
- execFileSync(
- 'git',
- [
- '-C',
- root,
- 'diff',
- '--no-color',
- '--no-ext-diff',
- '--src-prefix=a/',
- '--dst-prefix=b/',
- authority,
- reviewHead,
- ],
- {encoding: 'utf8', maxBuffer: 256 * 1024 * 1024},
- ),
-);
-const concernMatches = collectConcernMatches(records, addedLines, authority);
-console.log(
- JSON.stringify(
- {
- schemaVersion: 1,
- authorityCommit,
- reviewHead,
- requestedTriggers: [...new Set(requestedTriggers)].sort(),
- matches,
- concernMatches,
- },
- null,
- 2,
- ),
-);
+ if (!authorityCommit) {
+ throw new Error(
+ '--authority-commit requires the full base authority commit.',
+ );
+ }
+ if (!/^[0-9a-f]{40}$/.test(authorityCommit)) {
+ throw new Error(
+ '--authority-commit must be a full 40-character commit hash.',
+ );
+ }
+ if (!reviewHead) {
+ throw new Error('--review-head requires the full reviewed commit.');
+ }
+ if (!/^[0-9a-f]{40}$/.test(reviewHead)) {
+ throw new Error('--review-head must be a full 40-character commit hash.');
+ }
+ if (!triggerValue) {
+ throw new Error('--triggers requires a comma-separated trigger list.');
+ }
+
+ const requestedTriggers = triggerValue
+ .split(',')
+ .map(value => value.trim())
+ .filter(Boolean);
+ if (requestedTriggers.length === 0) {
+ throw new Error('--triggers requires at least one trigger.');
+ }
+
+ console.log(
+ JSON.stringify(
+ routeReviewBaselines({
+ root,
+ authorityCommit,
+ reviewHead,
+ requestedTriggers,
+ }),
+ null,
+ 2,
+ ),
+ );
+}