Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧪 test: Spec the First-Sentence Diagnostic Comparison in the Suppression Runner (#17165)

Marco Beretta0c637aa10a67
diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml
index 1ce8492813..70728b999b 100644
--- a/.github/workflows/static-checks.yml
+++ b/.github/workflows/static-checks.yml
@@ -97,6 +97,7 @@ jobs:
# count is measured against, so a change to it has to run against
# the record — the ESLint step above only ever sees sources.
- 'scripts/static-checks.mts'
+ - 'scripts/diagnostics*.mts'
- '.github/workflows/static-checks.yml'
- '!**.md'
# Stylesheets are outside the design lint, which reads JSX.
diff --git a/e2e/specs/mock/scenarios/suppression-baseline.spec.ts b/e2e/specs/mock/scenarios/suppression-baseline.spec.ts
index 903e5bdcd3..5ce9e231aa 100644
--- a/e2e/specs/mock/scenarios/suppression-baseline.spec.ts
+++ b/e2e/specs/mock/scenarios/suppression-baseline.spec.ts
@@ -628,6 +628,14 @@ test.describe('the recorded design-rule backlog', () => {
join(emptyRoot, 'scripts/static-checks.mts'),
);
copyFileSync(resolve(repoRoot, 'scripts/i18n.mts'), join(emptyRoot, 'scripts/i18n.mts'));
+ copyFileSync(
+ resolve(repoRoot, 'scripts/diagnostics.mts'),
+ join(emptyRoot, 'scripts/diagnostics.mts'),
+ );
+ copyFileSync(
+ resolve(repoRoot, 'scripts/diagnostics.test.mts'),
+ join(emptyRoot, 'scripts/diagnostics.test.mts'),
+ );
copyFileSync(
resolve(repoRoot, 'scripts/typecheck.mts'),
join(emptyRoot, 'scripts/typecheck.mts'),
@@ -1108,6 +1116,9 @@ function syntheticRoot(): string {
for (const file of [
'scripts/static-checks.mts',
+ 'scripts/diagnostics.mts',
+ /** The suppressions check runs the comparison's spec before it reads the baseline. */
+ 'scripts/diagnostics.test.mts',
'scripts/i18n.mts',
'scripts/typecheck.mts',
/** The local design rules the config registers and the runner validates against. */
diff --git a/scripts/diagnostics.mts b/scripts/diagnostics.mts
new file mode 100644
index 0000000000..efa26fc166
--- /dev/null
+++ b/scripts/diagnostics.mts
@@ -0,0 +1,105 @@
+/**
+ * The comparison behind the design-suppression swap check in
+ * `scripts/static-checks.mts`: given one changed file's head report, its base
+ * report and the record at both ends, which design-rule violations this change
+ * introduced beyond what the record grew to cover. Kept apart from the runner,
+ * which owns git and ESLint, so the comparison can be specified on its own.
+ */
+
+/**
+ * ESLint's on-disk suppressions shape, which the recorded baseline claims to be:
+ * a count per file, per rule. `validateSuppressions` is what checks the claim, so
+ * the leaf stays `unknown`: a string or a float there is one of the failures.
+ */
+export type SuppressionsFile = Record<string, Record<string, { count?: unknown }>>;
+
+/**
+ * One file's entry in ESLint's JSON report, as the checks here read it. A
+ * suppressed message carries its text too: the swap check compares messages,
+ * and what the baseline silences is exactly what it has to see.
+ */
+export type LintReport = {
+ filePath: string;
+ /** `message` also carries ESLint's unused-directive text, where `ruleId` is null. */
+ messages: { ruleId: string | null; message?: string }[];
+ suppressedMessages?: {
+ ruleId: string | null;
+ message?: string;
+ suppressions?: { kind: string; justification?: string }[];
+ }[];
+};
+
+/** The design rules are `@shadcn/lint`'s and the local `design` plugin's
+ * (`scripts/eslint/design.mjs`); both are recorded in the same backlog. */
+export const isDesignRule = (ruleId: string | null | undefined): boolean =>
+ ruleId?.startsWith('shadcn/') === true || ruleId?.startsWith('design/') === true;
+
+/** A diagnostic is compared by its first sentence, the violation itself. What follows is the
+ * remedy, and it depends on where the file sits: a client path is offered the app's own
+ * compositions (`PanelContent`), a package path is not, so the same violation reads differently
+ * on either side of a move. */
+export const diagnosis = (message: string): string =>
+ /^.*?\.(?=\s|$)/.exec(message)?.[0] ?? message;
+
+export const signatures = (reports: LintReport[]): string[] =>
+ reports
+ .flatMap((file) => [...file.messages, ...(file.suppressedMessages ?? [])])
+ .filter((message) => isDesignRule(message.ruleId))
+ .map((message) => `${message.ruleId}: ${diagnosis(message.message ?? '')}`);
+
+const countOf = (record: SuppressionsFile, file: string, rule: string): number => {
+ const entry = record[file]?.[rule]?.count;
+ return typeof entry === 'number' ? entry : 0;
+};
+
+export type ChangedFile = {
+ /** The file's path at the head, as the repository names it. */
+ path: string;
+ /** What ESLint reports for the head version. */
+ head: LintReport;
+ /** The base version linted under the path it had there; absent for a file this change adds. */
+ base?: { path: string; reports: LintReport[] };
+};
+
+/**
+ * The violations a changed file carries beyond what its base version held and
+ * beyond what its record entry grew by. A file the change adds owes everything
+ * in it: the record may not grow to cover a violation written in the same
+ * commit as its entry, or the rules would police nothing but the tree they
+ * landed on. A file that existed at the base, renamed or not, brings its
+ * allowance and may grow it, and a rename's allowance is the one recorded under
+ * the name it had, or the move itself would read as room to spend.
+ */
+export function introducedViolations(
+ file: ChangedFile,
+ recorded: SuppressionsFile,
+ recordedBefore: SuppressionsFile,
+): string[] {
+ const { path, base } = file;
+ const had = new Map<string, number>();
+ for (const signature of base ? signatures(base.reports) : []) {
+ had.set(signature, (had.get(signature) ?? 0) + 1);
+ }
+ const allowed = new Map<string, number>();
+ const problems: string[] = [];
+ for (const signature of signatures([file.head])) {
+ const remaining = had.get(signature) ?? 0;
+ if (remaining > 0) {
+ had.set(signature, remaining - 1);
+ continue;
+ }
+ const rule = signature.slice(0, signature.indexOf(':'));
+ const grew = countOf(recorded, path, rule) - countOf(recordedBefore, base?.path ?? path, rule);
+ const spent = allowed.get(rule) ?? 0;
+ if (base && spent < grew) {
+ allowed.set(rule, spent + 1);
+ continue;
+ }
+ problems.push(
+ base
+ ? `${path}: ${signature} is new here and the record did not grow to cover it; fix it, or record what this change owes with \`npm run lint:design:suppress\` so the count says so`
+ : `${path}: ${signature} is in a file this change adds; the backlog records what the tree owed when the rules landed, not what a new file brings, so fix it`,
+ );
+ }
+ return problems;
+}
diff --git a/scripts/diagnostics.test.mts b/scripts/diagnostics.test.mts
new file mode 100644
index 0000000000..05786de89a
--- /dev/null
+++ b/scripts/diagnostics.test.mts
@@ -0,0 +1,109 @@
+import { test } from 'node:test';
+import { strict as assert } from 'node:assert';
+import { diagnosis, introducedViolations } from './diagnostics.mts';
+import type { LintReport, SuppressionsFile } from './diagnostics.mts';
+
+const RULE = 'shadcn/no-restyle';
+const CLIENT_PATH = 'client/src/components/Chat/Probe.tsx';
+const PACKAGE_PATH = 'packages/chat/src/components/Probe.tsx';
+
+/** `shadcn/no-restyle`'s message for a gap class on `<Button>`, as ESLint reports it at each path:
+ * the violation reads the same, the remedy offers `PanelContent` only inside the app. */
+const restyle = (className: string, path: string): string =>
+ `"${className}" is not allowed on <Button>: <Button> owns its spacing. Use a size (default, roomy), or margin here${
+ path.startsWith('client/') ? ', gap on the parent, or PanelContent' : ' or gap on the parent'
+ } for space around it. Add a size in packages/client/dist/index.mjs only if the design explicitly calls for one. See CLAUDE.md, "Theming and styling".`;
+
+/** A report whose design diagnostics are all suppressed by the record, as the baseline leaves them. */
+const report = (path: string, ...classNames: string[]): LintReport => ({
+ filePath: path,
+ messages: [],
+ suppressedMessages: classNames.map((className) => ({
+ ruleId: RULE,
+ message: restyle(className, path),
+ suppressions: [{ kind: 'file' }],
+ })),
+});
+
+const recordOf = (path: string, count: number): SuppressionsFile => ({
+ [path]: { [RULE]: { count } },
+});
+
+test('the remedy differs across the move, the violation does not', () => {
+ assert.notEqual(restyle('gap-1', CLIENT_PATH), restyle('gap-1', PACKAGE_PATH));
+ assert.equal(
+ diagnosis(restyle('gap-1', CLIENT_PATH)),
+ '"gap-1" is not allowed on <Button>: <Button> owns its spacing.',
+ );
+ assert.equal(diagnosis(restyle('gap-1', CLIENT_PATH)), diagnosis(restyle('gap-1', PACKAGE_PATH)));
+});
+
+test('a file moved into a package with its violation re-recorded at the same count passes', () => {
+ const problems = introducedViolations(
+ {
+ path: PACKAGE_PATH,
+ head: report(PACKAGE_PATH, 'gap-1'),
+ base: { path: CLIENT_PATH, reports: [report(CLIENT_PATH, 'gap-1')] },
+ },
+ recordOf(PACKAGE_PATH, 1),
+ recordOf(CLIENT_PATH, 1),
+ );
+ assert.deepEqual(problems, []);
+});
+
+test('a same-count swap fails and names the new violation', () => {
+ const problems = introducedViolations(
+ {
+ path: CLIENT_PATH,
+ head: report(CLIENT_PATH, 'gap-2'),
+ base: { path: CLIENT_PATH, reports: [report(CLIENT_PATH, 'gap-1')] },
+ },
+ recordOf(CLIENT_PATH, 1),
+ recordOf(CLIENT_PATH, 1),
+ );
+ assert.equal(problems.length, 1);
+ assert.match(
+ problems[0],
+ /"gap-2" is not allowed on <Button>: <Button> owns its spacing\. is new here/,
+ );
+});
+
+test('a same-count swap across a move fails too', () => {
+ const problems = introducedViolations(
+ {
+ path: PACKAGE_PATH,
+ head: report(PACKAGE_PATH, 'gap-2'),
+ base: { path: CLIENT_PATH, reports: [report(CLIENT_PATH, 'gap-1')] },
+ },
+ recordOf(PACKAGE_PATH, 1),
+ recordOf(CLIENT_PATH, 1),
+ );
+ assert.equal(problems.length, 1);
+ assert.match(
+ problems[0],
+ /^packages\/chat\/src\/components\/Probe\.tsx: .*"gap-2".* is new here/,
+ );
+});
+
+test('a violation added with the count raised to cover it passes', () => {
+ const problems = introducedViolations(
+ {
+ path: CLIENT_PATH,
+ head: report(CLIENT_PATH, 'gap-1', 'gap-2'),
+ base: { path: CLIENT_PATH, reports: [report(CLIENT_PATH, 'gap-1')] },
+ },
+ recordOf(CLIENT_PATH, 2),
+ recordOf(CLIENT_PATH, 1),
+ );
+ assert.deepEqual(problems, []);
+});
+
+test('a file the change adds brings no allowance, whatever the record says', () => {
+ const problems = introducedViolations(
+ { path: PACKAGE_PATH, head: report(PACKAGE_PATH, 'gap-1') },
+ recordOf(PACKAGE_PATH, 1),
+ {},
+ );
+ assert.equal(problems.length, 1);
+ assert.match(problems[0], /is in a file this change adds/);
+});
diff --git a/scripts/static-checks.mts b/scripts/static-checks.mts
index 98959d98c7..6591a664db 100644
--- a/scripts/static-checks.mts
+++ b/scripts/static-checks.mts
@@ -57,7 +57,9 @@ import {
} from 'node:fs';
import type { Dirent } from 'node:fs';
+import type { LintReport, SuppressionsFile } from './diagnostics.mts';
import { isTranslationReferenced } from './i18n.mts';
+import { introducedViolations, isDesignRule } from './diagnostics.mts';
import { resolveTypecheck } from './typecheck.mts';
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
@@ -109,6 +111,7 @@ const FILTERS = {
/** This runner is the gate: `validateSuppressions` is what a recorded count
* is measured against, so a change to it has to run against the record. */
'scripts/static-checks.mts',
+ 'scripts/diagnostics*.mts',
'.github/workflows/static-checks.yml',
'!**.md',
],
@@ -262,29 +265,6 @@ const DESIGN_ROOTS = ['client/src', 'packages/client/src', 'packages/chat/src'];
*/
const GATE_SOURCE = 'scripts/static-checks.mts';
-/**
- * ESLint's on-disk suppressions shape, which the recorded baseline claims to be:
- * a count per file, per rule. `validateSuppressions` is what checks the claim, so
- * the leaf stays `unknown` — a string or a float there is one of the failures.
- */
-type SuppressionsFile = Record<string, Record<string, { count?: unknown }>>;
-
-/**
- * One file's entry in ESLint's JSON report, as the checks here read it. A
- * suppressed message carries its text too: the swap check compares messages,
- * and what the baseline silences is exactly what it has to see.
- */
-type LintReport = {
- filePath: string;
- /** `message` also carries ESLint's unused-directive text, where `ruleId` is null. */
- messages: { ruleId: string | null; message?: string }[];
- suppressedMessages?: {
- ruleId: string | null;
- message?: string;
- suppressions?: { kind: string; justification?: string }[];
- }[];
-};
-
/** A path in ESLint's report, as the repository names it. */
const reportedPath = (filePath: string): string =>
filePath.startsWith(ROOT)
@@ -299,11 +279,6 @@ type SuppressionsPlugin = { plugin: { rules: Record<string, unknown> } };
/** The local `design` plugin's rule map, recorded beside `@shadcn/lint`'s. */
type DesignPlugin = { default: { rules: Record<string, unknown> } };
-/** The design rules are `@shadcn/lint`'s and the local `design` plugin's
- * (`scripts/eslint/design.mjs`); both are recorded in the same backlog. */
-const isDesignRule = (ruleId: string | null | undefined): boolean =>
- ruleId?.startsWith('shadcn/') === true || ruleId?.startsWith('design/') === true;
-
const I18N_FILE = 'client/src/locales/en/translation.json';
const I18N_SOURCE_DIRS = [
'client/src',
@@ -876,6 +851,13 @@ async function validateSuppressions(context: CheckContext): Promise<CheckOutcome
const targets = [SUPPRESSIONS_FILE, ...named];
const problems: string[] = [];
+ /** The swap comparison is specified on its own: a regression in it lets a
+ * violation through, which reads here as a pass. */
+ const comparison = runCommand(
+ { command: process.execPath, args: ['--test', resolve(ROOT, 'scripts/diagnostics.test.mts')] },
+ [],
+ );
+ if (comparison.status !== 0) problems.push(comparison.output);
// Read the rule names from the plugin rather than listing them here, so a rule
// this stack has not enabled yet is still a valid key and a renamed one is not.
let known: Set<string>;
@@ -1487,17 +1469,6 @@ async function introducedWithinAllowance(
return [`the base versions could not be linted: ${(error as Error).message}`];
}
- /** A diagnostic is compared by its first sentence, the violation itself. What follows is the
- * remedy, and it depends on where the file sits: a client path is offered the app's own
- * compositions (`PanelContent`), a package path is not, so the same violation reads differently
- * on either side of a move. */
- const diagnosis = (message: string): string => /^.*?\.(?=\s|$)/.exec(message)?.[0] ?? message;
- const signatures = (reports: LintReport[]): string[] =>
- reports
- .flatMap((file) => [...file.messages, ...(file.suppressedMessages ?? [])])
- .filter((message) => isDesignRule(message.ruleId))
- .map((message) => `${message.ruleId}: ${diagnosis(message.message ?? '')}`);
-
/** The record as the base held it: how much an entry grew between there and
* here is what this change may add. A count that goes up is a line in the
* diff and the recipe CLAUDE.md documents; a count that stands still is the
@@ -1512,59 +1483,21 @@ async function introducedWithinAllowance(
recordedBefore = {};
}
}
- const countOf = (record: SuppressionsFile, file: string, rule: string): number => {
- const entry = record[file]?.[rule]?.count;
- return typeof entry === 'number' ? entry : 0;
- };
-
const problems: string[] = [];
for (const file of subjects) {
- const relative = reportedPath(file.filePath);
- const inheritedFrom = before.get(relative);
- /** A file the change adds owes everything in it: the record may not grow to
- * cover a violation written in the same commit as its entry, or the rules
- * would police nothing but the tree they landed on. A file that existed at
- * the base — renamed or not — brings its allowance and may grow it, and a
- * rename's allowance is the one recorded under the name it had. */
- const inherited = inheritedFrom !== undefined;
- const had = new Map<string, number>();
- if (inheritedFrom !== undefined) {
- /** Linted as the path it had, not the one it has: moving a component out
- * of the library and into the app enters a stricter scope, and reading
- * the old contents under the new path would let the destination's own
- * diagnostics appear on both sides and pass as inherited. */
- const was = resolve(ROOT, inheritedFrom.path);
- for (const signature of signatures(await lintText(inheritedFrom.source, { filePath: was }))) {
- had.set(signature, (had.get(signature) ?? 0) + 1);
- }
- }
- const allowed = new Map<string, number>();
- for (const signature of signatures([file])) {
- const remaining = had.get(signature) ?? 0;
- if (remaining > 0) {
- had.set(signature, remaining - 1);
- continue;
- }
- /** Beyond what the base held: chargeable to this change unless the entry
- * grew for that rule, and only as far as it grew. A renamed file's
- * allowance is the one recorded under the name it had, or the move
- * itself would read as room to spend. */
- const rule = signature.slice(0, signature.indexOf(':'));
- const grew =
- countOf(recorded, relative, rule) -
- countOf(recordedBefore, inheritedFrom?.path ?? relative, rule);
- const key = `${relative}\u0000${rule}`;
- const spent = allowed.get(key) ?? 0;
- if (inherited && spent < grew) {
- allowed.set(key, spent + 1);
- continue;
- }
- problems.push(
- inherited
- ? `${relative}: ${signature} is new here and the record did not grow to cover it; fix it, or record what this change owes with \`npm run lint:design:suppress\` so the count says so`
- : `${relative}: ${signature} is in a file this change adds; the backlog records what the tree owed when the rules landed, not what a new file brings, so fix it`,
- );
- }
+ const path = reportedPath(file.filePath);
+ const inheritedFrom = before.get(path);
+ /** Linted as the path it had, not the one it has: moving a component out
+ * of the library and into the app enters a stricter scope, and reading
+ * the old contents under the new path would let the destination's own
+ * diagnostics appear on both sides and pass as inherited. */
+ const base = inheritedFrom && {
+ path: inheritedFrom.path,
+ reports: await lintText(inheritedFrom.source, {
+ filePath: resolve(ROOT, inheritedFrom.path),
+ }),
+ };
+ problems.push(...introducedViolations({ path, head: file, base }, recorded, recordedBefore));
}
return problems;
}