An open source design system that's fully customizable and agent ready
TypeScript
perf(scripts): the knowledge and release checks read a commit's files in one git process (#9889)
Christopher Chedeaub93ee919ae53
diff --git a/scripts/check-knowledge.mjs b/scripts/check-knowledge.mjsindex d486d78f7f..ade3989a15 100644--- a/scripts/check-knowledge.mjs+++ b/scripts/check-knowledge.mjs@@ -15,6 +15,7 @@ import {createHash} from 'node:crypto';import {createRequire} from 'node:module';import {fileURLToPath} from 'node:url';import {collectThemingTargets} from '../packages/cli/foundation/discovery/theming-targets.mjs';+import {readObjects} from './lib/git-objects.mjs';const require = createRequire(import.meta.url);const {@@ -792,21 +793,36 @@ export function readKnowledgeRecordsAtAuthority(authorityCommit,reviewHead,) {- const resolvedCommit = execFileSync(+ // One process resolves both commits, and fails when either does not.+ const resolved = execFileSync('git',- ['-C', root, 'rev-parse', '--verify', `${authorityCommit}^{commit}`],+ [+ '-C',+ root,+ 'rev-parse',+ `${authorityCommit}^{commit}`,+ `${reviewHead}^{commit}`,+ ],{encoding: 'utf8'},- ).trim();+ )+ .trim()+ .split('\n');+ // Without --verify, rev-parse echoes an option and expands a range, so+ // only two object ids count as a resolution.+ if (+ resolved.length !== 2 ||+ !resolved.every(line => /^[0-9a-f]{40,64}$/.test(line))+ ) {+ throw new Error(+ `cannot resolve ${authorityCommit} and ${reviewHead} to two commits.`,+ );+ }+ const [resolvedCommit, resolvedHead] = resolved;if (resolvedCommit !== authorityCommit) {throw new Error(`authority commit must be the full resolved commit ${resolvedCommit}.`,);}- const resolvedHead = execFileSync(- 'git',- ['-C', root, 'rev-parse', '--verify', `${reviewHead}^{commit}`],- {encoding: 'utf8'},- ).trim();if (resolvedHead !== reviewHead) {throw new Error(`review head must be the full resolved commit ${resolvedHead}.`,@@ -845,14 +861,17 @@ export function readKnowledgeRecordsAtAuthority().split(/\r?\n/).filter(isKnowledgeRecordPath);- const records = paths.map(filePath => ({- path: filePath,- content: execFileSync(- 'git',- ['-C', root, 'show', `${resolvedCommit}:${filePath}`],- {encoding: 'utf8'},- ),- }));+ const objects = readObjects(+ root,+ paths.map(filePath => `${resolvedCommit}:${filePath}`),+ );+ const records = paths.map(filePath => {+ const content = objects.get(`${resolvedCommit}:${filePath}`);+ if (!content) {+ throw new Error(`cannot read ${filePath} at ${resolvedCommit}.`);+ }+ return {path: filePath, content: content.toString('utf8')};+ });return {records, authority: resolvedCommit};}diff --git a/scripts/lib/git-objects.mjs b/scripts/lib/git-objects.mjsnew file mode 100644index 0000000000..a21a2d08f5--- /dev/null+++ b/scripts/lib/git-objects.mjs@@ -0,0 +1,46 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Many files out of git in one process.+ * @input A repository and object names such as `<commit>:<path>`.+ * @output Each name's bytes, or null when git holds no such object.+ * @position check-knowledge reads every knowledge record at a review's+ * authority commit through it: one `git cat-file --batch` instead of one+ * `git show` per record. Each git command is a process launch, which a+ * loaded machine makes slow.+ */+import {execFileSync} from 'node:child_process';++// A found object's header line: its id, type and size in bytes.+const FOUND = /^[0-9a-f]{40,64} [a-z]+ (\d+)$/;++/**+ * Reads every object `names` lists with one `git cat-file --batch`.+ * @param {string} root+ * @param {string[]} names+ * @returns {Map<string, Buffer | null>}+ */+export function readObjects(root, names) {+ /** @type {Map<string, Buffer | null>} */+ const objects = new Map();+ if (names.length === 0) return objects;+ const out = execFileSync('git', ['-C', root, 'cat-file', '--batch'], {+ input: names.map(name => `${name}\n`).join(''),+ maxBuffer: 1024 * 1024 * 1024,+ });+ let at = 0;+ for (const name of names) {+ const end = out.indexOf(0x0a, at);+ const found = FOUND.exec(out.toString('utf8', at, end));+ at = end + 1;+ if (found === null) {+ objects.set(name, null);+ continue;+ }+ const size = Number(found[1]);+ objects.set(name, out.subarray(at, at + size));+ // A line feed follows each object's bytes.+ at += size + 1;+ }+ return objects;+}diff --git a/scripts/lib/git-objects.test.mjs b/scripts/lib/git-objects.test.mjsnew file mode 100644index 0000000000..2a2d67d488--- /dev/null+++ b/scripts/lib/git-objects.test.mjs@@ -0,0 +1,124 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {execFileSync} from 'node:child_process';+import {createHash} from 'node:crypto';+import fs from 'node:fs';+import {createRequire} from 'node:module';+import os from 'node:os';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {afterEach, describe, expect, it} from 'vitest';+import {readObjects} from './git-objects.mjs';+import {filesAtRef} from '../release/active-release.mjs';++const require = createRequire(import.meta.url);+const {+ isKnowledgeRecordPath,+} = require('../../.github/scripts/knowledge-paths.cjs');+const REPO_ROOT = path.resolve(+ path.dirname(fileURLToPath(import.meta.url)),+ '../..',+);++/** @type {string[]} */+const roots = [];+afterEach(() => {+ for (const root of roots.splice(0))+ fs.rmSync(root, {recursive: true, force: true});+});++/** @param {string} root @param {string[]} args */+const git = (root, ...args) =>+ execFileSync('git', ['-C', root, ...args], {maxBuffer: 1024 * 1024 * 1024});++/**+ * The id git names a blob by: the hash of its header and bytes.+ * @param {string} format the repository's object format+ * @param {Buffer} content+ */+function blobId(format, content) {+ return createHash(format === 'sha256' ? 'sha256' : 'sha1')+ .update(`blob ${content.length}\0`)+ .update(content)+ .digest('hex');+}++// The two batched readers: check-knowledge's and the release check's own.+const READERS = {+ readObjects: (root, ref, files) => {+ const objects = readObjects(+ root,+ files.map(file => `${ref}:${file}`),+ );+ return new Map(files.map(file => [file, objects.get(`${ref}:${file}`)]));+ },+ filesAtRef,+};++describe.each(Object.entries(READERS))('%s', (_, read) => {+ it('reads each file as `git show` reads it, and null where git has none', () => {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'git-objects-'));+ roots.push(root);+ const files = {+ 'a.txt': Buffer.from('one\ntwo\n'),+ 'my dir/space name.md': Buffer.from('# Ünïcode — ok\n'),+ 'empty.txt': Buffer.alloc(0),+ 'binary.dat': Buffer.from([0, 10, 255, 10, 0, 1]),+ };+ for (const [file, content] of Object.entries(files)) {+ fs.mkdirSync(path.dirname(path.join(root, file)), {recursive: true});+ fs.writeFileSync(path.join(root, file), content);+ }+ git(root, 'init', '-q');+ git(root, 'add', '-A');+ git(+ root,+ '-c',+ 'user.name=T',+ '-c',+ '[email protected]',+ 'commit',+ '-qm',+ 'files',+ );++ const read1 = read(root, 'HEAD', [...Object.keys(files), 'missing.txt']);+ for (const file of Object.keys(files)) {+ expect(read1.get(file), file).toEqual(git(root, 'show', `HEAD:${file}`));+ }+ expect(read1.get('missing.txt')).toBeNull();+ expect(read(root, 'no-such-ref', ['a.txt']).get('a.txt')).toBeNull();+ });++ it('reads every knowledge record of this repository at HEAD byte for byte', () => {+ const listing = git(+ REPO_ROOT,+ 'ls-tree',+ '-r',+ 'HEAD',+ '--',+ 'docs',+ 'packages',+ )+ .toString()+ .split('\n')+ .filter(Boolean)+ .map(line => {+ const [meta, file] = line.split('\t');+ return {id: meta.split(' ')[2], file};+ })+ .filter(({file}) => isKnowledgeRecordPath(file));+ expect(listing.length).toBeGreaterThan(100);+ const format = git(REPO_ROOT, 'rev-parse', '--show-object-format')+ .toString()+ .trim();+ const contents = read(+ REPO_ROOT,+ 'HEAD',+ listing.map(({file}) => file),+ );+ for (const {id, file} of listing) {+ expect(blobId(format, contents.get(file)), file).toBe(id);+ }+ });+});diff --git a/scripts/lib/review-diff.test.mjs b/scripts/lib/review-diff.test.mjsindex ec7f6a7e1f..aeac7c4c92 100644--- a/scripts/lib/review-diff.test.mjs+++ b/scripts/lib/review-diff.test.mjs@@ -8,6 +8,7 @@ import {afterEach, describe, expect, it} from 'vitest';import {readDiff} from './review-diff.mjs';import {collectConcernMatches} from '../check-knowledge.mjs';import {loadKnowledgeIndex} from '../specs-for-file.mjs';+import {routeReviewBaselines} from '../review-global-baselines.mjs';const HERE = import.meta.dirname;const ROOT = path.resolve(HERE, '../..');@@ -321,12 +322,17 @@ describe('review-global-baselines.mjs', () => {);}+ // The router runs in this process; the case below runs the command once.it("routes a concern the head's added lines match, beside the requested triggers", () => {- const result = route(- repository(- 'export function A() {\n useEffect(() => () => {}, []);\n}\n',- ),+ const {root, base, head} = repository(+ 'export function A() {\n useEffect(() => () => {}, []);\n}\n',);+ const result = routeReviewBaselines({+ root,+ authorityCommit: base,+ reviewHead: head,+ requestedTriggers: ['styling'],+ });expect(result.matches.map(match => match.claimId)).toEqual(['INV2']);expect(result.concernMatches).toEqual([expect.objectContaining({diff --git a/scripts/release/active-release.mjs b/scripts/release/active-release.mjsindex ce7cd34efa..9d8963441e 100644--- a/scripts/release/active-release.mjs+++ b/scripts/release/active-release.mjs@@ -375,9 +375,57 @@ function latestStableVersion(root) {return versions.at(-1) ?? null;}-function fileDigestAtRef(root, ref, file) {+// A found object's header line in `git cat-file --batch` output: its id,+// type and size in bytes.+const FOUND_OBJECT = /^[0-9a-f]{40,64} [a-z]+ (\d+)$/;++/**+ * Each file's bytes at `ref`, or null when the ref lacks it, read with one+ * `git cat-file --batch` instead of one `git show` per file. This module+ * imports nothing beyond its own lib (see the header), so it carries its own+ * reader.+ * @param {string} root+ * @param {string} ref+ * @param {string[]} files+ * @returns {Map<string, Buffer | null>}+ */+function filesAtRef(root, ref, files) {+ const contents = new Map();+ if (files.length === 0) return contents;+ const out = execFileSync('git', ['cat-file', '--batch'], {+ cwd: root,+ input: files.map(file => `${ref}:${file}\n`).join(''),+ maxBuffer: 1024 * 1024 * 1024,+ });+ let at = 0;+ for (const file of files) {+ const end = out.indexOf(0x0a, at);+ const found = FOUND_OBJECT.exec(out.toString('utf8', at, end));+ at = end + 1;+ if (found === null) {+ contents.set(file, null);+ continue;+ }+ const size = Number(found[1]);+ contents.set(file, out.subarray(at, at + size));+ at += size + 1;+ }+ return contents;+}++function fileDigestsAtRef(root, ref, files) {+ const contents = filesAtRef(root, ref, files);+ return new Map(+ files.map(file => {+ const content = contents.get(file);+ return [file, content ? sha256(content) : null];+ }),+ );+}++function parseJson(content) {try {- return sha256(execFileSync('git', ['show', `${ref}:${file}`], {cwd: root}));+ return content ? JSON.parse(content.toString('utf8')) : null;} catch {return null;}@@ -392,7 +440,9 @@ function changesetMapAtRef(root, ref) {'--','.changeset',]);- return new Map(+ return fileDigestsAtRef(+ root,+ ref,files.split('\n').filter(@@ -400,8 +450,7 @@ function changesetMapAtRef(root, ref) {file.startsWith('.changeset/') &&file.endsWith('.md') &&file !== '.changeset/README.md',- )- .map(file => [file, fileDigestAtRef(root, ref, file)]),+ ),);}@@ -416,11 +465,12 @@ function releaseOutputPathsAtRef(root, plan, releaseRef) {}function releaseOutputMapAtRef(root, plan, releaseRef) {- return new Map(+ return fileDigestsAtRef(+ root,+ releaseRef,releaseOutputPathsAtRef(root, plan, releaseRef).filter(isReleaseOutputPath)- .filter(file => !isPackageManifestPath(file))- .map(file => [file, fileDigestAtRef(root, releaseRef, file)]),+ .filter(file => !isPackageManifestPath(file)),);}@@ -439,7 +489,8 @@ function jsonAtRef(root, ref, file) {}function manifestMapAtRef(root, ref, files) {- return new Map([...files].map(file => [file, jsonAtRef(root, ref, file)]));+ const contents = filesAtRef(root, ref, [...files]);+ return new Map([...files].map(file => [file, parseJson(contents.get(file))]));}function currentManifestMap(root, files) {@@ -946,7 +997,12 @@ function versionTagExists(root, version) {* @returns {{version: string|null, errors: string[]}}*/function declaredVersionAtRef(root, ref) {- const config = jsonAtRef(root, ref, '.changeset/config.json');+ const configFile = '.changeset/config.json';+ const files = expandWorkspaceDirs(root).map(dir =>+ path.relative(root, path.join(dir, 'package.json')),+ );+ const contents = filesAtRef(root, ref, [configFile, ...files]);+ const config = parseJson(contents.get(configFile));if (!config)return {version: null,@@ -954,9 +1010,8 @@ function declaredVersionAtRef(root, ref) {};const fixed = new Set((config.fixed ?? []).flat());const versions = new Map();- for (const dir of expandWorkspaceDirs(root)) {- const file = path.relative(root, path.join(dir, 'package.json'));- const manifest = jsonAtRef(root, ref, file);+ for (const file of files) {+ const manifest = parseJson(contents.get(file));if (!manifest || !fixed.has(manifest.name)) continue;if (!isStableReleasePackage(manifest)) continue;if (!versions.has(manifest.version)) versions.set(manifest.version, []);@@ -1298,6 +1353,7 @@ export {compareVersions,computePlanDigest,declaredVersionAtRef,+ filesAtRef,nextPlannedVersion,isStableReleasePackage,latestStableVersion,diff --git a/scripts/review-global-baselines.mjs b/scripts/review-global-baselines.mjsindex a122f97ed3..abed22c36b 100644--- a/scripts/review-global-baselines.mjs+++ b/scripts/review-global-baselines.mjs@@ -24,81 +24,102 @@ function valueFor(flag) {return value;}-const root = path.resolve(valueFor('--root') ?? DEFAULT_ROOT);-const authorityCommit = valueFor('--authority-commit');-const reviewHead = valueFor('--review-head');-const triggerValue = valueFor('--triggers');--if (!authorityCommit) {- throw new Error(- '--authority-commit requires the full base authority commit.',+/**+ * The claims a review routes: those the requested triggers name at the+ * authority commit, and those whose concerns the head's added lines match.+ * @param {{root: string, authorityCommit: string, reviewHead: string,+ * requestedTriggers: string[]}} options+ */+export function routeReviewBaselines({+ root,+ authorityCommit,+ reviewHead,+ requestedTriggers,+}) {+ const {records, authority} = readKnowledgeRecordsAtAuthority(+ root,+ authorityCommit,+ reviewHead,);-}-if (!/^[0-9a-f]{40}$/.test(authorityCommit)) {- throw new Error(- '--authority-commit must be a full 40-character commit hash.',+ const matches = collectGlobalReviewMatches(+ records,+ requestedTriggers,+ authority,);-}-if (!reviewHead) {- throw new Error('--review-head requires the full reviewed commit.');-}-if (!/^[0-9a-f]{40}$/.test(reviewHead)) {- throw new Error('--review-head must be a full 40-character commit hash.');-}-if (!triggerValue) {- throw new Error('--triggers requires a comma-separated trigger list.');+ // The reviewed change's own added lines route the claims of every concern+ // they re-implement (architecture:knowledge-contracts/INV20), whatever the+ // caller's triggers or the component record's review_triggers.+ const {addedLines} = readDiff(+ execFileSync(+ 'git',+ [+ '-C',+ root,+ 'diff',+ '--no-color',+ '--no-ext-diff',+ '--src-prefix=a/',+ '--dst-prefix=b/',+ authority,+ reviewHead,+ ],+ {encoding: 'utf8', maxBuffer: 256 * 1024 * 1024},+ ),+ );+ return {+ schemaVersion: 1,+ authorityCommit,+ reviewHead,+ requestedTriggers: [...new Set(requestedTriggers)].sort(),+ matches,+ concernMatches: collectConcernMatches(records, addedLines, authority),+ };}-const requestedTriggers = triggerValue- .split(',')- .map(value => value.trim())- .filter(Boolean);-if (requestedTriggers.length === 0) {- throw new Error('--triggers requires at least one trigger.');-}+if (process.argv[1] === fileURLToPath(import.meta.url)) {+ const root = path.resolve(valueFor('--root') ?? DEFAULT_ROOT);+ const authorityCommit = valueFor('--authority-commit');+ const reviewHead = valueFor('--review-head');+ const triggerValue = valueFor('--triggers');-const {records, authority} = readKnowledgeRecordsAtAuthority(- root,- authorityCommit,- reviewHead,-);-const matches = collectGlobalReviewMatches(- records,- requestedTriggers,- authority,-);-// The reviewed change's own added lines route the claims of every concern-// they re-implement (architecture:knowledge-contracts/INV20), whatever the-// caller's triggers or the component record's review_triggers.-const {addedLines} = readDiff(- execFileSync(- 'git',- [- '-C',- root,- 'diff',- '--no-color',- '--no-ext-diff',- '--src-prefix=a/',- '--dst-prefix=b/',- authority,- reviewHead,- ],- {encoding: 'utf8', maxBuffer: 256 * 1024 * 1024},- ),-);-const concernMatches = collectConcernMatches(records, addedLines, authority);-console.log(- JSON.stringify(- {- schemaVersion: 1,- authorityCommit,- reviewHead,- requestedTriggers: [...new Set(requestedTriggers)].sort(),- matches,- concernMatches,- },- null,- 2,- ),-);+ if (!authorityCommit) {+ throw new Error(+ '--authority-commit requires the full base authority commit.',+ );+ }+ if (!/^[0-9a-f]{40}$/.test(authorityCommit)) {+ throw new Error(+ '--authority-commit must be a full 40-character commit hash.',+ );+ }+ if (!reviewHead) {+ throw new Error('--review-head requires the full reviewed commit.');+ }+ if (!/^[0-9a-f]{40}$/.test(reviewHead)) {+ throw new Error('--review-head must be a full 40-character commit hash.');+ }+ if (!triggerValue) {+ throw new Error('--triggers requires a comma-separated trigger list.');+ }++ const requestedTriggers = triggerValue+ .split(',')+ .map(value => value.trim())+ .filter(Boolean);+ if (requestedTriggers.length === 0) {+ throw new Error('--triggers requires at least one trigger.');+ }++ console.log(+ JSON.stringify(+ routeReviewBaselines({+ root,+ authorityCommit,+ reviewHead,+ requestedTriggers,+ }),+ null,+ 2,+ ),+ );+}