An open source design system that's fully customizable and agent ready
TypeScript
ci: Lint checks a pull request's public text for internal identifiers, hosts and names (#9839)
Cindy Zhang0512d0df4bf3
diff --git a/.github/public-text-hashes.json b/.github/public-text-hashes.jsonnew file mode 100644index 0000000000..8a274181ab--- /dev/null+++ b/.github/public-text-hashes.json@@ -0,0 +1,13 @@+[+ "098bb76ab47a047141f2dd0a9de195eb3e25ba8ac6e98dc98fa7422c4bed1f9c",+ "2bf54fbb6097fed0b9ef7d4bf54160204724bd1751055b60abf8e433dc36b841",+ "35cc4085dee1aae0ba469eb18c8422e2e713f1948dbe1e194173f1a04c41e0c1",+ "42f048ff49f97a14cf36121cba33c6afe5f41e7e7a0f2406d607a7577ce29854",+ "75a500bf74b6f465e3f81113f601e066646991448c3ec5da8a473d07b2a1c1a1",+ "a5a3c217a4ec5f107d268aa8342c5a861e62f20cf1c599696aafd5ec208c1077",+ "b3d09297a144fc9e9841d9dce9dec6884213d3434f5ce6545896fe8f9f399808",+ "c90148620507a5bcb5067118487f5677f83fafb116af83b47a4fa398a8dc6312",+ "ce62d23575c30719d014e0c383aaa065cf185fe084dfb8af259af66aa3c5a0dd",+ "dea48037919e6537fa1f7cf8a5b94428f43a763fb33ee1a9d2f01597eae27eda",+ "e7f329b5f5d4636f720cd850972db90f751ba12153faa33425dc010ebf871bd2"+]diff --git a/.github/scripts/check-public-text.mjs b/.github/scripts/check-public-text.mjsnew file mode 100644index 0000000000..cde4257364--- /dev/null+++ b/.github/scripts/check-public-text.mjs@@ -0,0 +1,238 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file check-public-text.mjs+ * @input `--base <sha> --head <sha> --hashes <file> [--pr-json <file>]`, run+ * in a clone that holds the range. `--pr-json` is the pull request as the+ * REST API or the event payload gives it, for its title and description.+ * @output One error line for each finding: where it is and what kind it is,+ * never the text it matched. Exit 0 with no finding, 1 with a finding, 2 when+ * the range cannot be read.+ * @position Lint runs the base branch's copy on every pull request, over the+ * commit messages and added lines between the merge base and the head+ * (spec:AST-030 FR27).+ *+ * Two kinds of check. A shape check finds an internal identifier by its form.+ * A hash check compares each word, and each host's last two labels, with+ * salted SHA-256 digests, so the repository holds no internal name.+ */++import {execFileSync} from 'node:child_process';+import {createHash} from 'node:crypto';+import fs from 'node:fs';+import {pathToFileURL} from 'node:url';++import {readDiff} from '../../scripts/lib/review-diff.mjs';++export const SALT = 'astryx-public-text:v1:';++export const POLICY =+ 'docs/contributing/pull-requests.md#public-repository-boundary';++export const SHAPES = Object.freeze([+ {kind: 'an internal issue id', pattern: /\bkt-[a-z0-9]{4}\b/i},+ // Not after `#`: an upper-case hex colour after `#` is no id.+ {kind: 'an internal change or task id', pattern: /(?<!#)\b[DT][0-9]{6,}\b/},+ {kind: 'an internal paste id', pattern: /\bP[0-9]{9,}\b/},+ {kind: 'an internal host', pattern: /\binternal[a-z]*\.(?:com|net)\b/i},+]);++const WORD = /[A-Za-z0-9][A-Za-z0-9-]*/g;+const HOST = /\b(?:[a-z0-9-]+\.)+[a-z][a-z0-9-]*\b/g;+// Between a lower-case letter or digit and an upper-case one, and before the+// last capital of a run that starts a word: `useNameSession`, `HTTPServer`.+const CAMEL_PART = /(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/;++/** The salted digest of one lowercase token. */+export function digest(token) {+ return createHash('sha256')+ .update(SALT + token.toLowerCase())+ .digest('hex');+}++/**+ * The tokens of `text` that the hash check reads: each word, each part of a+ * hyphenated or camel-case word, and each host's last two labels.+ */+export function tokens(text) {+ const found = new Set();+ for (const word of text.match(WORD) ?? []) {+ found.add(word.toLowerCase());+ for (const part of word.split('-')) {+ if (part === '') continue;+ found.add(part.toLowerCase());+ for (const piece of part.split(CAMEL_PART))+ found.add(piece.toLowerCase());+ }+ }+ for (const host of text.toLowerCase().match(HOST) ?? []) {+ found.add(host.split('.').slice(-2).join('.'));+ }+ return found;+}++/**+ * The kinds of finding in one piece of text, without the text that matched.+ *+ * @param {string} text+ * @param {Set<string>} digests+ * @returns {string[]}+ */+export function findKinds(text, digests) {+ const kinds = SHAPES.filter(({pattern}) => pattern.test(text)).map(+ ({kind}) => kind,+ );+ for (const token of tokens(text)) {+ if (digests.has(digest(token))) {+ kinds.push('an internal name');+ break;+ }+ }+ return kinds;+}++/** The added lines of a unified diff, as `{file, line, text}`. */+export function addedLines(diff) {+ return readDiff(diff).everyAddedLine;+}++/**+ * Every finding in the pull request's public text.+ *+ * @param {{commits: {sha: string, message: string}[], diff: string,+ * title?: string, body?: string, digests: Set<string>,+ * skipFiles?: string[]}} input+ */+export function check({commits, diff, title, body, digests, skipFiles = []}) {+ const findings = [];+ for (const {sha, message} of commits) {+ for (const kind of findKinds(message, digests))+ findings.push({where: `commit ${sha.slice(0, 7)} message`, kind});+ }+ for (const [where, text] of [+ ['pull request title', title],+ ['pull request description', body],+ ]) {+ if (typeof text !== 'string') continue;+ for (const kind of findKinds(text, digests)) findings.push({where, kind});+ }+ for (const {file, line, text} of addedLines(diff)) {+ if (skipFiles.includes(file)) continue;+ for (const kind of findKinds(text, digests))+ findings.push({where: `${file}:${line}`, file, line, kind});+ }+ return findings;+}++function git(args) {+ return execFileSync('git', args, {+ encoding: 'utf8',+ maxBuffer: 256 * 1024 * 1024,+ });+}++function readArgs(argv) {+ const args = {};+ for (let i = 0; i < argv.length; i += 2) {+ const key = argv[i]?.replace(/^--/, '');+ args[key] = argv[i + 1];+ }+ return args;+}++/** Reads the range with git. Throws when it cannot be read whole. */+export function readRange(base, head) {+ if (+ !/^[0-9a-f]{7,64}$/i.test(base ?? '') ||+ !/^[0-9a-f]{7,64}$/i.test(head ?? '')+ )+ throw new Error('--base and --head must be commit shas');+ const log = git(['log', '--format=%H%x00%B%x1e', `${base}..${head}`]);+ const commits = log+ .split('\x1e')+ .map(entry => entry.replace(/^\n/, ''))+ .filter(Boolean)+ .map(entry => {+ const [sha, message = ''] = entry.split('\x00');+ return {sha, message};+ });+ if (commits.length === 0) throw new Error('the range has no commits');+ // `--text` and `--no-textconv`: a path marked binary in `.gitattributes`+ // shows its added lines too. Renames are found, so a moved file shows only+ // the lines its move changed.+ const diff = git([+ // Paths come unquoted, so a non-ASCII path reads as it is.+ '-c',+ 'core.quotePath=false',+ 'diff',+ '--text',+ '--no-textconv',+ '--unified=0',+ '--no-color',+ '--no-ext-diff',+ '--find-renames',+ base,+ head,+ ]);+ if (readDiff(diff).unreadable)+ throw new Error('a changed file has a path the check cannot read');+ return {commits, diff};+}++/**+ * The title and description from the pull request JSON: the REST API's pull+ * request, or an event payload that holds one under `pull_request`.+ */+export function readPullRequest(text) {+ const json = JSON.parse(text);+ const pr = json.pull_request ?? json;+ if (typeof pr.title !== 'string')+ throw new Error('the pull request JSON has no title');+ return {title: pr.title, body: typeof pr.body === 'string' ? pr.body : ''};+}++function main() {+ const args = readArgs(process.argv.slice(2));+ let digests;+ let range;+ let pr = {};+ try {+ digests = new Set(JSON.parse(fs.readFileSync(args.hashes, 'utf8')));+ range = readRange(args.base, args.head);+ if (args['pr-json'] != null)+ pr = readPullRequest(fs.readFileSync(args['pr-json'], 'utf8'));+ } catch (error) {+ console.log(+ `::error title=Public text check::The check could not read the pull request's commits, the digests or the description (${error.message.split('\n')[0]}). It does not pass what it cannot read.`,+ );+ process.exit(2);+ }+ const findings = check({+ ...range,+ title: pr.title,+ body: pr.body,+ digests,+ // The digests are hex, and the lockfile's integrity hashes are base64:+ // neither is text a person writes.+ skipFiles: ['.github/public-text-hashes.json', 'pnpm-lock.yaml'],+ });+ for (const {where, file, line, kind} of findings) {+ const at = file ? ` file=${file},line=${line},` : ' ';+ console.log(+ `::error${at}title=Public text check::${where} has ${kind}. Remove the reference there: public text must not carry internal identifiers, hosts or names (${POLICY}). If it is not clear what to remove, ask a maintainer.`,+ );+ }+ if (findings.length > 0) {+ console.log(+ `${findings.length} finding(s). Edit the file or the pull request description, or reword the commit, then push again.`,+ );+ process.exit(1);+ }+ console.log(+ `Checked ${range.commits.length} commit message(s) and the added lines: no internal identifiers.`,+ );+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)+ main();diff --git a/.github/scripts/check-public-text.test.mjs b/.github/scripts/check-public-text.test.mjsnew file mode 100644index 0000000000..29f98c9f6b--- /dev/null+++ b/.github/scripts/check-public-text.test.mjs@@ -0,0 +1,385 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {execFileSync, spawnSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {afterEach, describe, expect, it} from 'vitest';+import YAML from 'yaml';++import {+ addedLines,+ check,+ digest,+ findKinds,+ tokens,+} from './check-public-text.mjs';++const here = path.dirname(fileURLToPath(import.meta.url));+const root = path.resolve(here, '../..');+const SCRIPT = path.join(here, 'check-public-text.mjs');++// A made-up internal name and host, hashed here, so no test names a real one.+const FAKE_NAME = ['zebra', 'frob'].join('');+const FAKE_HOST = `${FAKE_NAME}.dev`;+const digests = new Set([digest(FAKE_NAME), digest(FAKE_HOST)]);++// Identifier shapes, assembled so this file holds no complete one.+const changeId = ['D', '12345678'].join('');+const taskId = ['T', '1234567'].join('');+const pasteId = ['P', '123456789'].join('');+const issueId = ['kt', 'ab12'].join('-');+const internalHost = ['internalexample', 'com'].join('.');++describe('findKinds', () => {+ it.each([+ [`see ${changeId} for it`, 'an internal change or task id'],+ [`tracked in ${taskId}.`, 'an internal change or task id'],+ [`paste ${pasteId}`, 'an internal paste id'],+ [`issue ${issueId}`, 'an internal issue id'],+ [`https://${internalHost}/x`, 'an internal host'],+ ])('finds %j as %s', (text, kind) => {+ expect(findKinds(text, digests)).toEqual([kind]);+ });++ it('finds a hashed name in any case, and a hashed host by its last two labels', () => {+ expect(findKinds(`uses ${FAKE_NAME.toUpperCase()} here`, digests)).toEqual([+ 'an internal name',+ ]);+ expect(findKinds(`https://build.${FAKE_HOST}/job`, digests)).toEqual([+ 'an internal name',+ ]);+ });++ it('passes ordinary text, short ids, timestamps and other hosts', () => {+ for (const text of [+ 'Fix the Drawer focus order (#9785).',+ 'D123 and T42 are short; P12345678 is too short for a paste.',+ '2026-11-03T120000Z',+ 'https://tinyurl.com/abc and https://example.com',+ `${FAKE_NAME}s is a different word`,+ ])+ expect(findKinds(text, digests), text).toEqual([]);+ });++ it('finds a hashed name inside a hyphenated or camel-case word', () => {+ const capital = FAKE_NAME[0].toUpperCase() + FAKE_NAME.slice(1);+ for (const text of [+ `${FAKE_NAME}-cli`,+ `tools-${FAKE_NAME}`,+ `use${capital}Session`,+ `${FAKE_NAME.toUpperCase()}_DIR`,+ `a/${FAKE_NAME}/b`,+ ])+ expect(findKinds(text, digests), text).toEqual(['an internal name']);+ });++ it('passes an upper-case hex colour that looks like an id', () => {+ // Assembled so this file holds no complete id shape.+ const colour = ['#D', '4485150'].join('');+ expect(findKinds(`color: ${colour};`, digests)).toEqual([]);+ });++ it('reads words and two-label hosts as tokens', () => {+ expect([...tokens('A b-c a.b.example.com')]).toEqual(+ expect.arrayContaining(['a', 'b-c', 'example.com']),+ );+ });+});++describe('addedLines', () => {+ it('reads only added lines, with their file and line', () => {+ const diff = [+ 'diff --git a/x.md b/x.md',+ '--- a/x.md',+ '+++ b/x.md',+ '@@ -3,0 +4,2 @@',+ '+one',+ '+two',+ '@@ -9 +11 @@',+ '-old',+ '+new',+ 'diff --git a/gone.md b/gone.md',+ '--- a/gone.md',+ '+++ /dev/null',+ '@@ -1 +0,0 @@',+ '-removed',+ ].join('\n');+ expect(addedLines(diff)).toEqual([+ {file: 'x.md', line: 4, text: 'one'},+ {file: 'x.md', line: 5, text: 'two'},+ {file: 'x.md', line: 11, text: 'new'},+ ]);+ });++ it('reads an added line that starts with ++ as content, not a file header', () => {+ const diff = [+ 'diff --git a/x.md b/x.md',+ '--- a/x.md',+ '+++ b/x.md',+ '@@ -1,0 +2,2 @@',+ `+++ see ${changeId}`,+ '+next',+ ].join('\n');+ expect(addedLines(diff)).toEqual([+ {file: 'x.md', line: 2, text: `++ see ${changeId}`},+ {file: 'x.md', line: 3, text: 'next'},+ ]);+ expect(findKinds(addedLines(diff)[0].text, digests)).toEqual([+ 'an internal change or task id',+ ]);+ });+});++describe('check', () => {+ const diff = (file, text) =>+ `diff --git a/${file} b/${file}\n--- /dev/null\n+++ b/${file}\n@@ -0,0 +1 @@\n+${text}\n`;++ it('reports where each finding is and its kind, never the text', () => {+ const findings = check({+ commits: [{sha: 'abcdef1234', message: `fix: per ${changeId}`}],+ diff: diff('docs/a.md', `see ${FAKE_NAME}`),+ title: 'A clean title',+ body: `Body names ${internalHost}`,+ digests,+ });+ expect(findings.map(({where, kind}) => `${where}: ${kind}`)).toEqual([+ 'commit abcdef1 message: an internal change or task id',+ 'pull request description: an internal host',+ 'docs/a.md:1: an internal name',+ ]);+ expect(JSON.stringify(findings)).not.toContain(FAKE_NAME);+ expect(JSON.stringify(findings)).not.toContain(changeId);+ });++ it('skips the files it is told to skip', () => {+ expect(+ check({+ commits: [{sha: 'a'.repeat(40), message: 'chore: deps'}],+ diff: diff('pnpm-lock.yaml', `integrity: sha512-x+${changeId}/y`),+ digests,+ skipFiles: ['pnpm-lock.yaml'],+ }),+ ).toEqual([]);+ });+});++describe('the script on a real repository', () => {+ const dirs = [];+ afterEach(() => {+ for (const dir of dirs.splice(0))+ fs.rmSync(dir, {recursive: true, force: true});+ });++ const repo = () => {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'public-text-'));+ dirs.push(dir);+ const git = (...args) =>+ execFileSync('git', args, {+ cwd: dir,+ encoding: 'utf8',+ env: {+ ...process.env,+ GIT_AUTHOR_NAME: 't',+ GIT_AUTHOR_EMAIL: '[email protected]',+ GIT_COMMITTER_NAME: 't',+ GIT_COMMITTER_EMAIL: '[email protected]',+ },+ }).trim();+ git('init', '-q', '-b', 'main');+ const commit = (file, text, message) => {+ fs.mkdirSync(path.dirname(path.join(dir, file)), {recursive: true});+ fs.writeFileSync(path.join(dir, file), text);+ git('add', file);+ git('commit', '-q', '-m', message);+ return git('rev-parse', 'HEAD');+ };+ const hashes = path.join(dir, 'hashes.json');+ fs.writeFileSync(hashes, JSON.stringify([...digests]));+ const run = (base, head, pr = null) => {+ const args = [SCRIPT, '--base', base, '--head', head, '--hashes', hashes];+ if (pr != null) {+ const file = path.join(dir, 'pull-request.json');+ fs.writeFileSync(file, JSON.stringify(pr));+ args.push('--pr-json', file);+ }+ return spawnSync(process.execPath, args, {cwd: dir, encoding: 'utf8'});+ };+ return {commit, run, hashes, git, dir};+ };++ it('fails on a finding and prints no matched text', () => {+ const {commit, run} = repo();+ const base = commit('a.md', 'a\n', 'base');+ const head = commit('b.md', `uses ${FAKE_NAME}\n`, `add b (${changeId})`);+ const result = run(base, head, {+ pull_request: {title: 'A clean title', body: `see ${FAKE_HOST}`},+ });+ expect(result.status).toBe(1);+ expect(result.stdout).toContain(+ 'message has an internal change or task id',+ );+ expect(result.stdout).toContain('file=b.md,line=1,');+ expect(result.stdout).toContain(+ 'pull request description has an internal name',+ );+ expect(result.stdout).toContain('Remove the reference there');+ expect(result.stdout).toContain('ask a maintainer');+ for (const value of [FAKE_NAME, changeId]) {+ expect(result.stdout).not.toContain(value);+ expect(result.stderr).not.toContain(value);+ }+ });++ it.each(['My notes.md', 'café.md'])(+ 'reads %j when it comes first in the diff',+ name => {+ const {commit, run} = repo();+ const base = commit('z.md', 'z\n', 'base');+ const head = commit(name, `uses ${FAKE_NAME}\n`, 'add notes');+ const result = run(base, head);+ expect(result.status).toBe(1);+ expect(result.stdout).toContain(`file=${name},line=1,`);+ },+ );++ it('reads a path marked binary as text', () => {+ const {commit, run} = repo();+ commit('.gitattributes', '*.dat binary\n', 'mark dat files binary');+ const base = commit('a.md', 'a\n', 'base');+ const head = commit('notes.dat', `uses ${FAKE_NAME}\n`, 'add notes');+ const result = run(base, head);+ expect(result.status).toBe(1);+ expect(result.stdout).toContain('file=notes.dat,line=1,');+ });++ it('passes a moved file whose text was already on the base branch', () => {+ const {commit, run, git, dir} = repo();+ commit('old/notes.md', `from ${changeId} long ago\n`, 'old notes');+ const base = commit('a.md', 'a\n', 'base');+ fs.mkdirSync(path.join(dir, 'new'));+ git('mv', 'old/notes.md', 'new/notes.md');+ git('commit', '-q', '-m', 'move the notes');+ expect(run(base, git('rev-parse', 'HEAD')).status).toBe(0);+ });++ it('reads the title and description from the REST API shape too', () => {+ const {commit, run} = repo();+ const base = commit('a.md', 'a\n', 'base');+ const head = commit('b.md', 'b\n', 'add b');+ const result = run(base, head, {title: `fix ${taskId}`, body: null});+ expect(result.status).toBe(1);+ expect(result.stdout).toContain(+ 'pull request title has an internal change or task id',+ );+ });++ it('passes clean commits', () => {+ const {commit, run} = repo();+ const base = commit('a.md', 'a\n', 'base');+ const head = commit('b.md', 'b\n', 'add b');+ expect(run(base, head).status).toBe(0);+ });++ it('ignores a match already on the base branch before the merge base', () => {+ const {commit, run} = repo();+ commit('old.md', `old ${FAKE_NAME}\n`, `old (${changeId})`);+ const base = commit('a.md', 'a\n', 'base');+ const head = commit('b.md', 'b\n', 'add b');+ expect(run(base, head).status).toBe(0);+ });++ it('fails closed when the range is empty, unreadable or not a sha', () => {+ const {commit, run, hashes} = repo();+ const base = commit('a.md', 'a\n', 'base');+ expect(run(base, base).status).toBe(2);+ expect(run(base, 'f'.repeat(40)).status).toBe(2);+ expect(run('main', base).status).toBe(2);+ fs.rmSync(hashes);+ const head = commit('b.md', 'b\n', 'add b');+ expect(run(base, head).status).toBe(2);+ });+});++describe('the committed digests', () => {+ it('hold only salted SHA-256 hex digests', () => {+ const list = JSON.parse(+ fs.readFileSync(+ path.join(root, '.github/public-text-hashes.json'),+ 'utf8',+ ),+ );+ expect(list.length).toBeGreaterThan(0);+ for (const entry of list) expect(entry).toMatch(/^[0-9a-f]{64}$/);+ });+});++describe('Lint runs the check (spec:AST-030 FR27)', () => {+ const workflow = YAML.parse(+ fs.readFileSync(path.join(root, '.github/workflows/lint.yml'), 'utf8'),+ );+ const steps = workflow.jobs.lint.steps;+ const index = steps.findIndex(step => step.name === 'Check public text');+ const step = steps[index];++ it('runs on every pull request, before the slower checks', () => {+ expect(step.if).toBe("github.event_name == 'pull_request'");+ expect(index).toBeLessThan(+ steps.findIndex(s => s.name === 'Lint changed workflows (actionlint)'),+ );+ });++ it("runs the base branch's copy of the check, its diff reader and digests", () => {+ for (const file of [+ '.github/scripts/check-public-text.mjs',+ 'scripts/lib/review-diff.mjs',+ '.github/public-text-hashes.json',+ ])+ expect(step.run).toContain(`git show "origin/$BASE_REF:${file}"`);+ expect(step.run).toContain(+ 'node "$CHECK_DIR/.github/scripts/check-public-text.mjs"',+ );+ expect(step.run).not.toMatch(/node \.github\/scripts\/check-public-text/);+ });++ it('skips only when the base branch has no check, never for missing digests', () => {+ const exits = step.run.match(/exit 0/g) ?? [];+ expect(exits).toHaveLength(1);+ const skip = step.run.slice(0, step.run.indexOf('exit 0'));+ expect(skip).toContain('check-public-text.mjs');+ expect(skip).not.toContain('public-text-hashes.json');+ // A missing digest file is removed, so the check itself fails on it.+ expect(step.run).toContain('|| rm -f "$CHECK_DIR/public-text-hashes.json"');+ });++ it('deepens only the base branch and the pull request head before any whole fetch', () => {+ const deepen = step.run.indexOf(+ 'git fetch --no-tags --deepen=500 origin "$BASE_REF" "refs/pull/$PR_NUMBER/head"',+ );+ const whole = step.run.indexOf(+ 'git fetch --no-tags --unshallow origin "$BASE_REF" "refs/pull/$PR_NUMBER/head"',+ );+ expect(deepen).toBeGreaterThan(-1);+ expect(whole).toBeGreaterThan(deepen);+ expect(step.run).not.toMatch(/--unshallow origin >/);+ });++ it('checks the pull request head from its merge base, and fails without one', () => {+ expect(step.run).toContain('HEAD_SHA=$(git rev-parse HEAD^2)');+ expect(step.run).toContain('git merge-base "$BASE_TIP" "$HEAD_SHA"');+ expect(step.run).toContain('--base "$MERGE_BASE" --head "$HEAD_SHA"');+ expect(step.run).toMatch(/merge base was not found[^\n]*\n\s*exit 1/);+ });++ it('reads the current title and description through the API, and never prints them', () => {+ expect(step.run).toContain(+ 'gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" > "$CHECK_DIR/pull-request.json"',+ );+ expect(step.run).toContain('--pr-json "$CHECK_DIR/pull-request.json"');+ // A step's env is printed in the log, so the text never goes through it.+ expect(JSON.stringify(step.env)).not.toMatch(/title|body/);+ expect(step.run).not.toMatch(/cat [^\n]*pull-request\.json/);+ expect(workflow.jobs.lint.permissions['pull-requests']).toBe('read');+ });+});diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.ymlindex 5a4f81bc84..d3f877c7d8 100644--- a/.github/workflows/lint.yml+++ b/.github/workflows/lint.yml@@ -23,6 +23,8 @@ jobs:runs-on: ubuntu-24.04-armpermissions:contents: read+ # The public text check reads the current title and description.+ pull-requests: readsteps:- name: Require a branch in this repository or the ci:run label@@ -66,6 +68,56 @@ jobs:>> "$GITHUB_OUTPUT"fi+ # Public text carries no internal identifier, host or name: each commit+ # message and added line from the merge base to the head, and the title+ # and description as they read when the check runs, so a re-run reads an+ # edit (spec:AST-030 FR27). The base branch's copy of the check runs, so+ # a pull request cannot change its own check.+ - name: Check public text+ if: github.event_name == 'pull_request'+ env:+ BASE_REF: ${{ github.base_ref }}+ PR_NUMBER: ${{ github.event.pull_request.number }}+ GH_TOKEN: ${{ github.token }}+ run: |+ set -u+ CHECK_DIR="$RUNNER_TEMP/public-text"+ mkdir -p "$CHECK_DIR/.github/scripts" "$CHECK_DIR/scripts/lib"+ if ! git show "origin/$BASE_REF:.github/scripts/check-public-text.mjs" > "$CHECK_DIR/.github/scripts/check-public-text.mjs" 2>/dev/null; then+ echo "::notice::The base branch has no public text check yet. It runs once the check is on the base branch."+ exit 0+ fi+ # Without the shared diff reader or the digests, the check fails.+ git show "origin/$BASE_REF:scripts/lib/review-diff.mjs" > "$CHECK_DIR/scripts/lib/review-diff.mjs" 2>/dev/null \+ || rm -f "$CHECK_DIR/scripts/lib/review-diff.mjs"+ git show "origin/$BASE_REF:.github/public-text-hashes.json" > "$CHECK_DIR/public-text-hashes.json" 2>/dev/null \+ || rm -f "$CHECK_DIR/public-text-hashes.json"+ if ! gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" > "$CHECK_DIR/pull-request.json" 2>/dev/null; then+ echo "::notice::The current title and description could not be read, so the check reads them as of this run's event."+ cp "$GITHUB_EVENT_PATH" "$CHECK_DIR/pull-request.json"+ fi+ HEAD_SHA=$(git rev-parse HEAD^2)+ BASE_TIP=$(git rev-parse HEAD^1)+ MERGE_BASE=$(git merge-base "$BASE_TIP" "$HEAD_SHA" 2>/dev/null || true)+ # A branch far behind its base: deepen only the base branch and the+ # pull request head, then, rarely, take the whole history.+ if [ -z "$MERGE_BASE" ]; then+ git fetch --no-tags --deepen=500 origin "$BASE_REF" "refs/pull/$PR_NUMBER/head" > /dev/null 2>&1 || true+ MERGE_BASE=$(git merge-base "$BASE_TIP" "$HEAD_SHA" 2>/dev/null || true)+ fi+ if [ -z "$MERGE_BASE" ]; then+ git fetch --no-tags --unshallow origin "$BASE_REF" "refs/pull/$PR_NUMBER/head" > /dev/null 2>&1 || true+ MERGE_BASE=$(git merge-base "$BASE_TIP" "$HEAD_SHA" 2>/dev/null || true)+ fi+ if [ -z "$MERGE_BASE" ]; then+ echo "::error title=Public text check::The merge base was not found, so the commits cannot be checked."+ exit 1+ fi+ node "$CHECK_DIR/.github/scripts/check-public-text.mjs" \+ --base "$MERGE_BASE" --head "$HEAD_SHA" \+ --hashes "$CHECK_DIR/public-text-hashes.json" \+ --pr-json "$CHECK_DIR/pull-request.json"+# A change to a workflow, a composite action or this config runs# actionlint over the workflows, with shellcheck on their run: scripts,# so a broken expression, an unknown runner label or a script that cannotdiff --git a/docs/specs/AST-030/spec.md b/docs/specs/AST-030/spec.mdindex b6df6c5b9a..abc2b14483 100644--- a/docs/specs/AST-030/spec.md+++ b/docs/specs/AST-030/spec.md@@ -43,7 +43,9 @@ affects_consumer_docs: []".github/scripts/ci-pr-gate.*",".github/scripts/perf-*","apps/sandbox/perf/**",- "playwright.perf.config.ts"+ "playwright.perf.config.ts",+ ".github/scripts/check-public-text.*",+ ".github/public-text-hashes.json"]}```@@ -538,6 +540,20 @@ they are separately named.classifier has no Lab-only lane. An outsidecontributor's pull request still needs `ci:run` (FR17), and `main` and therelease-check cover Lab as before (DEC-21).+- **FR27 — Lint fails a pull request whose public text carries an internal+ reference.** Public text is each commit message and added line from the merge+ base to the pull request head, and the pull request's title and description+ as they read when the check runs. Generated text that no person writes, the+ lockfile and the digest list, is not read.+ The required Lint job runs the base branch's copy of the check, so a pull+ request cannot change the check that judges it. The check matches generic+ shapes of internal identifiers and hosts, and salted digests of internal+ names; the repository holds no internal name in plain text. A failure names+ the location and the kind of reference, never the matched text, and tells+ the contributor to remove the reference at that location, with a maintainer's+ help when the reference is unclear. A range it cannot read fails the check. A+ maintainer handles a false positive through the normal fix-forward route,+ with no bypass.### Platform support@@ -601,6 +617,7 @@ The nightly perf lane is the one scheduled measurement outside them (FR18).| FR23 | `release-check-shape.test.mjs`, `a11y-contract-shard.test.mjs` and the release-check workflow contract | release-check and pull-request legs; a contract file longer than a leg's share, one it cannot name exactly, and a serial one; the recorded suite; a re-check from an admin, a maintainer and a writer, from another branch, with a short SHA and with an older commit; failed-only beside a passed, a running and a missing run | a narrower release-check, a held or over-budget leg, a test selected by no leg or by two, a re-check that a writer can start, that waits for main's running check, or that re-runs jobs of another commit fails || FR24 | `ci-heavy-lanes.test.mjs` cadence cases | release-check 170 and 176 minutes old; the unit lane hourly beside it; weekday 06:37 Pacific inside the window in standard and daylight time; the pre-cut check already started; the tip already checked; a weekend morning; before 06:30 and after 07:30 | a release-check starts inside the 175-minute window, the pre-cut check is skipped or starts twice, or it runs on a weekend || FR26 | `ci-lab-only-routing.test.mjs`, `change-scope.test.mjs` and `test-scope-action.test.mjs` Lab-only cases | a Lab-only change, with and without a Changeset; Lab beside core, charts, a Storybook story, the docsite, the Sandbox, a workflow, the lockfile, root configuration, internal tooling or a spec record; a rename into or out of Lab; an incomplete list; a base classifier without the lane; each required context, its owners, the UI legs' Lab tests, the UI and Node guard tests, the repository guards, the Storybook build and Lab typecheck, the heavy suites with and without a label, lint and an agent push | a mixed change takes the Lab-only lane; a required context skips or fails, or passes without its notice; a Lab test, a guard test, a repository guard, the Lab typecheck or the Storybook build does not run; an unlabelled heavy suite, the docsite or the full typecheck runs, or a label's suite cannot audit Lab; lint is skipped |+| FR27 | `check-public-text.test.mjs` over the check and the Lint workflow step | each identifier shape; a hashed name and host made up in the test; ordinary text; a finding in a commit message, an added line and the description; a match before the merge base; an empty range, a bad sha, a missing commit and a missing digest file | a reference passes; the output holds the matched text; a match on the base branch fails a later pull request; an unreadable range passes; the step runs the pull request's own copy |## Decision logdiff --git a/scripts/lib/review-diff.mjs b/scripts/lib/review-diff.mjsindex d0d548d599..d5ff32b97b 100644--- a/scripts/lib/review-diff.mjs+++ b/scripts/lib/review-diff.mjs@@ -4,13 +4,16 @@* @file Read the shipped source lines a change adds.* @input A unified diff (`git diff` output).* @output Each added line of shipped source with its file and new line number,- * every changed path, every added line of each path, and the count of added- * and removed lines of each path.- * @position Shared by `pnpm specs:for --diff` and- * `scripts/review-global-baselines.mjs`, which match these lines against the+ * every added line of any file with its file and new line number, every+ * changed path, every added line of each path, and the count of added and+ * removed lines of each path.+ * @position Shared by `pnpm specs:for --diff`, the public text check in Lint+ * and `scripts/review-global-baselines.mjs`, which match these lines against the* `concerns` the owning records declare (`architecture:knowledge-contracts/INV20`).*/+import {Buffer} from 'node:buffer';+/** Source files whose added lines can change what a product runs. */const SOURCE_FILE = /\.(?:[cm]?[jt]sx?)$/;@@ -27,38 +30,91 @@ export function isShippedSource(file) {/*** Parse a unified diff into its changed paths and added lines. `addedLines`- * keeps shipped source only; `addedByFile` keeps every added line of each path;+ * keeps shipped source only; `everyAddedLine` keeps every added line of any+ * file, with its new line number; `addedByFile` keeps every added line of each path;* `changedCountByFile` counts the added and removed lines of each path.*/+/**+ * The path in a `--- a/…` or `+++ b/…` header. Git quotes a path with special+ * bytes (`"b/caf\303\251.md"`); the quotes and octal escapes are undone.+ */+function headerPath(rest, side) {+ // Git ends a header path that holds a space with a tab.+ let value = rest.replace(/\t$/, '');+ if (value.startsWith('"') && value.endsWith('"')) {+ const bytes = [];+ const inner = value.slice(1, -1);+ for (let i = 0; i < inner.length; i += 1) {+ if (inner[i] === '\\' && /[0-7]{3}/.test(inner.slice(i + 1, i + 4))) {+ bytes.push(parseInt(inner.slice(i + 1, i + 4), 8));+ i += 3;+ } else if (inner[i] === '\\' && i + 1 < inner.length) {+ const escaped = {n: '\n', t: '\t', '"': '"', '\\': '\\'}[inner[i + 1]];+ bytes.push(...Buffer.from(escaped ?? inner[i + 1]));+ i += 1;+ } else {+ bytes.push(...Buffer.from(inner[i]));+ }+ }+ value = Buffer.from(bytes).toString('utf8');+ }+ return value.startsWith(`${side}/`) ? value.slice(2) : null;+}+export function readDiff(diffText) {const files = [];const addedLines = [];+ const everyAddedLine = [];const addedByFile = {};const changedCountByFile = {};+ // True when a hunk comes before its file's path could be read.+ let unreadable = false;let file = null;+ let oldFile = null;let newLine = 0;let inHunk = false;+ const open = name => {+ file = name;+ if (!files.includes(name)) files.push(name);+ addedByFile[name] ??= [];+ changedCountByFile[name] ??= 0;+ };for (const line of diffText.split('\n')) {- const header = /^diff --git a\/\S+ b\/(\S+)/.exec(line);- if (header) {- file = header[1];- files.push(file);- addedByFile[file] ??= [];- changedCountByFile[file] ??= 0;+ // Every `diff --git` line starts a file. Its path comes from the `+++`+ // header (or `---` for a deleted file); the `diff --git` line itself+ // cannot be split when a path holds a space. A diff with no headers (a+ // pure rename, a mode or binary change) keeps the simple-path reading.+ if (line.startsWith('diff --git ')) {+ file = null;+ oldFile = null;inHunk = false;+ const simple = /^diff --git a\/(\S+) b\/(\S+)$/.exec(line);+ if (simple) open(simple[2]);continue;}const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line);if (hunk) {newLine = Number(hunk[1]);inHunk = file != null;+ if (file == null) unreadable = true;continue;}// Before the first hunk, `---` and `+++` lines are file headers. Inside a// hunk, they are content.- if (!inHunk) continue;+ if (!inHunk) {+ if (line.startsWith('--- ')) oldFile = headerPath(line.slice(4), 'a');+ else if (line.startsWith('+++ ')) {+ const name =+ line.slice(4) === '/dev/null'+ ? oldFile+ : headerPath(line.slice(4), 'b');+ if (name != null) open(name);+ }+ continue;+ }if (line.startsWith('+')) {addedByFile[file].push(line.slice(1));+ everyAddedLine.push({file, line: newLine, text: line.slice(1)});changedCountByFile[file] += 1;if (isShippedSource(file)) {addedLines.push({file, line: newLine, text: line.slice(1)});@@ -70,5 +126,12 @@ export function readDiff(diffText) {newLine += 1;}}- return {files, addedLines, addedByFile, changedCountByFile};+ return {+ files,+ addedLines,+ everyAddedLine,+ addedByFile,+ changedCountByFile,+ unreadable,+ };}diff --git a/scripts/lib/review-diff.test.mjs b/scripts/lib/review-diff.test.mjsindex a31200908d..ec7f6a7e1f 100644--- a/scripts/lib/review-diff.test.mjs+++ b/scripts/lib/review-diff.test.mjs@@ -55,6 +55,68 @@ describe('readDiff', () => {expect(addedByFile).toEqual({'docs/a.md': ['added']});});+ it('keeps every added line of any file with its new line number', () => {+ const diff = [+ 'diff --git a/docs/a.md b/docs/a.md',+ '--- a/docs/a.md',+ '+++ b/docs/a.md',+ '@@ -3,0 +4,2 @@',+ '+one',+ '+++ two',+ '@@ -9 +11 @@',+ '-old',+ '+new',+ ].join('\n');+ expect(readDiff(diff).everyAddedLine).toEqual([+ {file: 'docs/a.md', line: 4, text: 'one'},+ {file: 'docs/a.md', line: 5, text: '++ two'},+ {file: 'docs/a.md', line: 11, text: 'new'},+ ]);+ });++ it('reads a path with a space, or a quoted non-ASCII path, from the +++ header', () => {+ const diff = [+ 'diff --git a/My notes.md b/My notes.md',+ 'new file mode 100644',+ '--- /dev/null',+ '+++ b/My notes.md\t',+ '@@ -0,0 +1 @@',+ '+first',+ 'diff --git "a/caf\\303\\251.md" "b/caf\\303\\251.md"',+ '--- "a/caf\\303\\251.md"',+ '+++ "b/caf\\303\\251.md"',+ '@@ -1 +1 @@',+ '-old',+ '+second',+ ].join('\n');+ const {files, everyAddedLine, unreadable} = readDiff(diff);+ expect(files).toEqual(['My notes.md', 'café.md']);+ expect(everyAddedLine).toEqual([+ {file: 'My notes.md', line: 1, text: 'first'},+ {file: 'café.md', line: 1, text: 'second'},+ ]);+ expect(unreadable).toBe(false);+ });++ it('reads a deleted file by its old path, and flags a hunk with no readable path', () => {+ expect(+ readDiff(+ [+ 'diff --git a/gone file.md b/gone file.md',+ '--- a/gone file.md',+ '+++ /dev/null',+ '@@ -1 +0,0 @@',+ '-removed',+ ].join('\n'),+ ).files,+ ).toEqual(['gone file.md']);+ const odd = readDiff(+ ['diff --git a/x y b/x y', '@@ -0,0 +1 @@', '+lost'].join('\n'),+ );+ expect(odd.unreadable).toBe(true);+ expect(odd.everyAddedLine).toEqual([]);+ });+it('reads added shipped-source lines with their new line numbers', () => {const {files, addedLines} = readDiff(fixture('pr-8381.diff'));expect(files).toEqual([