Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Keep build outputs out of cached OK test discovery (#5296)

Nick Gomez858143750507
diff --git a/packages/app/tests/integration/mock-module-completeness.test.ts b/packages/app/tests/integration/mock-module-completeness.test.ts
index 7b1002e7e..5e6cac8d0 100644
--- a/packages/app/tests/integration/mock-module-completeness.test.ts
+++ b/packages/app/tests/integration/mock-module-completeness.test.ts
@@ -1,6 +1,15 @@
-import { readFileSync } from 'node:fs';
-import { join, relative } from 'node:path';
-import { describe, expect, test } from 'vitest';
+import fs, {
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, relative, sep } from 'node:path';
+import { afterAll, describe, expect, test, vi } from 'vitest';
+import { isTestOnlySourceFile } from '../../../../test-support/test-only-source-file.mjs';
const APP_ROOT = join(import.meta.dir, '..', '..');
@@ -11,21 +20,40 @@ const ALLOWLIST: Record<string, string> = {
'value export consumed by plain tests in this process.',
};
-function extractDoMockCalls(src: string): Array<{ specifier: string; factory: string }> {
- const calls: Array<{ specifier: string; factory: string }> = [];
- const re = /vi\.doMock\(\s*(['"])([^'"]+)\1\s*,/g;
- let m: RegExpExecArray | null = re.exec(src);
- while (m !== null) {
+interface DoMockCall {
+ specifier: string | null;
+ factory: string | null;
+}
+
+const DO_MOCK_SITE = /\b(?:vi|vitest)\s*\.\s*doMock\b/g;
+
+const DO_MOCK_HEAD =
+ /^\s*(?:<[^>]*>\s*)?\(\s*(?:(?:await\s+)?import\s*\(\s*(['"`])([^'"`$]+)\1\s*\)|(['"`])([^'"`$]+)\3)\s*([,)])/;
+
+function extractDoMockCalls(src: string): DoMockCall[] {
+ const calls: DoMockCall[] = [];
+ for (const site of src.matchAll(DO_MOCK_SITE)) {
+ const afterName = (site.index ?? 0) + site[0].length;
+ const head = DO_MOCK_HEAD.exec(src.slice(afterName));
+ if (head === null) {
+ calls.push({ specifier: null, factory: null });
+ continue;
+ }
+ const specifier = head[2] ?? head[4] ?? '';
+ if (head[5] === ')') {
+ calls.push({ specifier, factory: null });
+ continue;
+ }
+ const start = afterName + head[0].length;
let depth = 1;
- let i = re.lastIndex;
+ let i = start;
while (i < src.length && depth > 0) {
const ch = src[i];
if (ch === '(') depth++;
else if (ch === ')') depth--;
i++;
}
- calls.push({ specifier: m[2] ?? '', factory: src.slice(re.lastIndex, i) });
- m = re.exec(src);
+ calls.push({ specifier, factory: src.slice(start, i) });
}
return calls;
}
@@ -34,29 +62,192 @@ function factoryHasActualSpread(factory: string): boolean {
return /\.\.\.\s*actual[A-Za-z_$]?[\w$]*/.test(factory);
}
+const posixRelative = (from: string, to: string): string => relative(from, to).split(sep).join('/');
+
+interface FactoryScan {
+ scanned: string[];
+ factories: number;
+ exempted: string[];
+ violations: string[];
+}
+
+async function scanPlainTestFactories(
+ packageRoot: string,
+ allowlist: Record<string, string>,
+): Promise<FactoryScan> {
+ const sourceRoot = join(packageRoot, 'src');
+ const glob = new Bun.Glob('**/*.test.{ts,tsx}');
+ const scan: FactoryScan = { scanned: [], factories: 0, exempted: [], violations: [] };
+ for await (const file of glob.scan(sourceRoot)) {
+ if (file.includes('.dom.test.')) continue;
+ const abs = join(sourceRoot, file);
+ const rel = posixRelative(packageRoot, abs);
+ scan.scanned.push(rel);
+ for (const call of extractDoMockCalls(readFileSync(abs, 'utf-8'))) {
+ if (call.specifier === null) {
+ scan.violations.push(
+ `${rel} names vi.doMock where this guard cannot read a string-literal specifier. ` +
+ 'If it is a call, write the specifier as a string literal so its factory can be checked; ' +
+ 'if it is text (a test title, string or directive reason), reword it so vi.doMock does not appear.',
+ );
+ continue;
+ }
+ if (call.factory === null) continue;
+ scan.factories += 1;
+ if (factoryHasActualSpread(call.factory)) continue;
+ const allowKey = `${rel}::${call.specifier}`;
+ if (allowKey in allowlist) {
+ scan.exempted.push(allowKey);
+ continue;
+ }
+ scan.violations.push(
+ `${rel} mocks '${call.specifier}' with a partial factory (no \`...actual*\`-convention spread). ` +
+ `Spread the real module (static-import it as actual, then \`...actual\` first in the factory) ` +
+ `or add '${allowKey}' to ALLOWLIST with a safety rationale.`,
+ );
+ }
+ }
+ scan.scanned.sort();
+ return scan;
+}
+
+function plainTestFilesUnder(packageRoot: string): string[] {
+ return readdirSync(join(packageRoot, 'src'), { recursive: true, withFileTypes: true })
+ .filter((entry) => !entry.isDirectory() && isTestOnlySourceFile(entry.name, 'vitest'))
+ .map((entry) => posixRelative(packageRoot, join(entry.parentPath, entry.name)))
+ .filter((rel) => !rel.includes('.dom.test.'))
+ .filter((rel) => !rel.split('/').some((segment) => segment.startsWith('.')))
+ .sort();
+}
+
+const violatingFile = (violation: string) => violation.split(' ')[0];
+
describe('vi.doMock factory completeness', () => {
test('every plain-test factory spreads the real module or is allowlisted', async () => {
- const glob = new Bun.Glob('src/**/*.test.{ts,tsx}');
- const violations: string[] = [];
- for await (const file of glob.scan(APP_ROOT)) {
- if (file.includes('.dom.test.')) continue;
- const abs = join(APP_ROOT, file);
- const rel = relative(APP_ROOT, abs);
- const src = readFileSync(abs, 'utf-8');
- if (!src.includes('vi.doMock(')) continue;
- for (const call of extractDoMockCalls(src)) {
- const hasSpread = factoryHasActualSpread(call.factory);
- const allowKey = `${rel}::${call.specifier}`;
- if (!hasSpread && !(allowKey in ALLOWLIST)) {
- violations.push(
- `${rel} mocks '${call.specifier}' with a partial factory (no \`...actual*\`-convention spread). ` +
- `Spread the real module (static-import it as actual, then \`...actual\` first in the factory) ` +
- `or add '${allowKey}' to ALLOWLIST with a safety rationale.`,
- );
- }
- }
+ const scan = await scanPlainTestFactories(APP_ROOT, ALLOWLIST);
+ expect(scan.violations).toEqual([]);
+ });
+
+ test('the walk reads every plain source test file and checks real factories', async () => {
+ const scan = await scanPlainTestFactories(APP_ROOT, ALLOWLIST);
+ const expected = plainTestFilesUnder(APP_ROOT);
+ expect(
+ expected,
+ 'Found no plain test files under src/: the package root or the test-file pattern is wrong.',
+ ).not.toEqual([]);
+ expect(
+ scan.scanned,
+ "The walk must read exactly the plain test files under src/. Lines marked '-' are files it missed, " +
+ "lines marked '+' are files it read that are not plain src/ tests: fix scanPlainTestFactories' root or glob.",
+ ).toEqual(expected);
+ expect(
+ scan.factories,
+ 'No plain src/ test calls vi.doMock with a factory, so this guard checks nothing: fix the walk, ' +
+ 'or retire the guard if those mocks are gone.',
+ ).toBeGreaterThan(0);
+ });
+
+ test('every ALLOWLIST entry still exempts a partial factory', async () => {
+ const scan = await scanPlainTestFactories(APP_ROOT, ALLOWLIST);
+ const stale = Object.keys(ALLOWLIST).filter((key) => !scan.exempted.includes(key));
+ expect(
+ stale,
+ 'These ALLOWLIST entries exempt nothing: the factory now spreads ...actual*, or the file or ' +
+ 'specifier is gone. Delete them from ALLOWLIST.',
+ ).toEqual([]);
+ });
+});
+
+describe('the walk, on a fixture package', () => {
+ const partial = "vi.doMock('sonner', () => ({ toast: { error() {} } }));\n";
+ const spread = "vi.doMock('sonner', () => ({ ...actualSonner, toast: { error() {} } }));\n";
+ const fixtureRoots: string[] = [];
+ let shared = '';
+
+ const makeFixture = (files: Record<string, string>): string => {
+ const root = mkdtempSync(join(tmpdir(), 'ok-mock-completeness-'));
+ fixtureRoots.push(root);
+ for (const [rel, body] of Object.entries(files)) {
+ mkdirSync(dirname(join(root, rel)), { recursive: true });
+ writeFileSync(join(root, rel), body);
}
- expect(violations).toEqual([]);
+ return root;
+ };
+
+ const fixturePackage = (): string => {
+ shared ||= makeFixture({
+ 'src/root.test.ts': partial,
+ 'src/nested/deep/inner.test.tsx': partial,
+ 'src/compliant.test.ts': spread,
+ 'src/view.dom.test.tsx': partial,
+ 'src/.cache/hidden.test.ts': partial,
+ 'src/shared.test-helper.ts': partial,
+ 'dist/src/built.test.ts': partial,
+ 'tests/integration/outside.test.ts': partial,
+ });
+ return shared;
+ };
+
+ afterAll(() => {
+ for (const root of fixtureRoots) rmSync(root, { recursive: true, force: true });
+ });
+
+ test('fires on a partial factory at the source root and nested below it, and nowhere else', async () => {
+ const scan = await scanPlainTestFactories(fixturePackage(), {});
+ expect(scan.violations.map(violatingFile).sort()).toEqual([
+ 'src/nested/deep/inner.test.tsx',
+ 'src/root.test.ts',
+ ]);
+ expect(scan.scanned).toEqual([
+ 'src/compliant.test.ts',
+ 'src/nested/deep/inner.test.tsx',
+ 'src/root.test.ts',
+ ]);
+ expect(scan.scanned).toEqual(plainTestFilesUnder(fixturePackage()));
+ expect(scan.factories).toBe(3);
+ });
+
+ test('reads no directory outside the source root', async () => {
+ const root = fixturePackage();
+ const readdir = vi.spyOn(fs, 'readdirSync');
+ let read: string[] = [];
+ try {
+ await scanPlainTestFactories(root, {});
+ read = readdir.mock.calls
+ .map(([dir]) => posixRelative(root, String(dir)))
+ .filter((dir) => !dir.startsWith('..'));
+ } finally {
+ readdir.mockRestore();
+ }
+ expect(read).toContain('src');
+ expect(read.filter((dir) => dir !== 'src' && !dir.startsWith('src/'))).toEqual([]);
+ });
+
+ test('an ALLOWLIST entry exempts only its own file and specifier', async () => {
+ const exact = await scanPlainTestFactories(fixturePackage(), {
+ 'src/root.test.ts::sonner': 'fixture',
+ });
+ expect(exact.violations.map(violatingFile)).toEqual(['src/nested/deep/inner.test.tsx']);
+ expect(exact.exempted).toEqual(['src/root.test.ts::sonner']);
+
+ const otherSpecifier = await scanPlainTestFactories(fixturePackage(), {
+ 'src/root.test.ts::next-themes': 'fixture',
+ });
+ expect(otherSpecifier.violations.map(violatingFile).sort()).toEqual([
+ 'src/nested/deep/inner.test.tsx',
+ 'src/root.test.ts',
+ ]);
+ expect(otherSpecifier.exempted).toEqual([]);
+ });
+
+ test('fails on a vi.doMock specifier it cannot read, and leaves an automock alone', async () => {
+ const root = makeFixture({
+ 'src/forms.test.ts': "vi.doMock('@/lib/auto');\nvi.doMock(MODULE, () => ({}));\n",
+ });
+ const scan = await scanPlainTestFactories(root, {});
+ expect(scan.factories).toBe(0);
+ expect(scan.violations.map(violatingFile)).toEqual(['src/forms.test.ts']);
+ expect(scan.violations[0]).toContain('cannot read');
});
});
@@ -73,6 +264,35 @@ describe('guard self-test (bidirectional + planted-positive)', () => {
expect(extractDoMockCalls('const x = 1; await import("./y");')).toEqual([]);
});
+ test('extractDoMockCalls: reads every specifier form Vitest accepts, and marks the rest unreadable', () => {
+ const forms = [
+ "vi.doMock(import('sonner'), () => ({ toast }));",
+ "vi.doMock(await import('next-themes'), () => ({ useTheme }));",
+ 'vi.doMock(`@/lib/a`, () => ({ a }));',
+ "vitest.doMock('@/lib/b', () => ({ b }));",
+ "vi .doMock('@/lib/c', () => ({ c }));",
+ "vi.doMock<typeof import('@/lib/d')>('@/lib/d', () => ({ d }));",
+ "vi.doMock('@/lib/auto');",
+ 'vi.doMock(MODULE, () => ({}));',
+ `vi.doMock(\`@/lib/\${name}\`, () => ({}));`,
+ "test('resets vi.doMock state between cases', () => {});",
+ ].join('\n');
+ expect(
+ extractDoMockCalls(forms).map((c) => [c.specifier, c.factory === null ? 'none' : 'factory']),
+ ).toEqual([
+ ['sonner', 'factory'],
+ ['next-themes', 'factory'],
+ ['@/lib/a', 'factory'],
+ ['@/lib/b', 'factory'],
+ ['@/lib/c', 'factory'],
+ ['@/lib/d', 'factory'],
+ ['@/lib/auto', 'none'],
+ [null, 'none'],
+ [null, 'none'],
+ [null, 'none'],
+ ]);
+ });
+
test('factoryHasActualSpread: accepts ...actual* spreads (must-fire-true)', () => {
expect(factoryHasActualSpread('() => ({ ...actualSonner, toast })')).toBe(true);
expect(factoryHasActualSpread('() => ({ ...actual, useDocumentContext: () => ({}) })')).toBe(
@@ -91,7 +311,9 @@ describe('guard self-test (bidirectional + planted-positive)', () => {
test('end-to-end: a partial factory is flagged, an ...actual factory is not', () => {
const flagged = (src: string) =>
- extractDoMockCalls(src).filter((c) => !factoryHasActualSpread(c.factory)).length;
+ extractDoMockCalls(src).filter(
+ (c) => c.factory !== null && !factoryHasActualSpread(c.factory),
+ ).length;
expect(flagged("vi.doMock('sonner', () => ({ toast: { error() {} } }));")).toBe(1);
expect(
flagged("vi.doMock('sonner', () => ({ ...actualSonner, toast: { error() {} } }));"),
diff --git a/packages/app/vitest.config.ts b/packages/app/vitest.config.ts
index cff8bde57..5bbe4c1c9 100644
--- a/packages/app/vitest.config.ts
+++ b/packages/app/vitest.config.ts
@@ -25,6 +25,7 @@ const idbPreloadPath = fileURLToPath(
export const appVitestConfig = {
...okVitestBase,
+ publicDir: false,
plugins: [...okVitestBase.plugins, react()],
resolve: {
...okVitestBase.resolve,
@@ -40,6 +41,11 @@ export const appVitestConfig = {
},
test: {
...okVitestBase.test,
+ exclude: [
+ ...okVitestBase.test.exclude,
+ 'public/excalidraw-assets/**',
+ '.excalidraw-assets-staging-*',
+ ],
setupFiles: [...okVitestBase.test.setupFiles, idbPreloadPath],
},
} satisfies ViteUserConfig;
diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts
index 594445be5..efdce7ee4 100644
--- a/packages/cli/vitest.config.ts
+++ b/packages/cli/vitest.config.ts
@@ -10,4 +10,10 @@ import { okVitestBase } from '../../test-support/vitest.base';
* black-box `tests/e2e/cli-linux-e2e.ts` deliberately omits the `.test` infix so
* it stays out of this tier; it runs under `vitest.e2e.config.ts`.
*/
-export default defineConfig({ ...okVitestBase });
+export default defineConfig({
+ ...okVitestBase,
+ test: {
+ ...okVitestBase.test,
+ exclude: [...okVitestBase.test.exclude, '.ok-skill-publish-*'],
+ },
+});
diff --git a/packages/desktop/vitest.config.ts b/packages/desktop/vitest.config.ts
index 01798037d..46de942ed 100644
--- a/packages/desktop/vitest.config.ts
+++ b/packages/desktop/vitest.config.ts
@@ -29,6 +29,10 @@ export default defineConfig({
exclude: [
...okVitestBase.test.exclude,
...lumeQaExcludeFor(process.platform, process.env.OK_LUME_QA_SUITES),
+ 'out/**',
+ 'dist-desktop/**',
+ '.variant-build/**',
+ 'build/parcel-watcher-staging/**',
],
},
});
diff --git a/packages/server/src/test-helper-caller-typecheck-coverage.test.ts b/packages/server/src/test-helper-caller-typecheck-coverage.test.ts
index 1efdd1f84..379b9c82e 100644
--- a/packages/server/src/test-helper-caller-typecheck-coverage.test.ts
+++ b/packages/server/src/test-helper-caller-typecheck-coverage.test.ts
@@ -17,6 +17,10 @@ const HELPERS = [
];
const UNSCANNED_DIRS = new Set(['node_modules', 'dist', '.turbo']);
+const UNSCANNED_DIR_PREFIXES = ['.ok-skill-publish-'];
+
+const isUnscanned = (name: string): boolean =>
+ UNSCANNED_DIRS.has(name) || UNSCANNED_DIR_PREFIXES.some((prefix) => name.startsWith(prefix));
const RELATIVE_SPECIFIER = /(?:\bfrom|\bimport\s*\()\s*'(\.{1,2}\/[^']*)'/g;
@@ -35,7 +39,7 @@ function filesUnder(dir: string, suffix: string): string[] {
return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
const path = join(dir, entry.name);
if (entry.isDirectory()) {
- return UNSCANNED_DIRS.has(entry.name) ? [] : filesUnder(path, suffix);
+ return isUnscanned(entry.name) ? [] : filesUnder(path, suffix);
}
return entry.isFile() && entry.name.endsWith(suffix) ? [path] : [];
});
@@ -151,6 +155,24 @@ describe('every caller of a shared test-only helper sits inside a typecheck prog
).toEqual(['buried.test-helper.ts', 'flat.test-helper.ts']);
});
+ test('skips the skill-bundle staging roots a server build leaves beside its sources', () => {
+ const root = mkdtempSync(join(tmpdir(), 'test-helper-scan-'));
+ onTestFinished(() => rmSync(root, { recursive: true, force: true }));
+ const plant = (path: string) => {
+ mkdirSync(join(root, dirname(path)), { recursive: true });
+ writeFileSync(join(root, path), '');
+ };
+ plant('src/source.test.ts');
+ plant('.ok-skill-publish-a1b2c3/assets/staged.test.ts');
+ plant('.ok-skill-publish-kept-a1b2c3/staged.test.ts');
+
+ expect(
+ filesUnder(root, '.test.ts').map((path) => relative(root, path).replaceAll('\\', '/')),
+ 'a staging root is gitignored and written by the server build, so it sits outside the ' +
+ "cache key of the test task this walk runs in, and entering it reads the build's output",
+ ).toEqual(['src/source.test.ts']);
+ });
+
test('names in HELPERS only helpers that are on disk', () => {
const present = helpersOnDisk();
expect(
diff --git a/packages/server/vitest.config.ts b/packages/server/vitest.config.ts
index fa44c2e72..e9009fcdc 100644
--- a/packages/server/vitest.config.ts
+++ b/packages/server/vitest.config.ts
@@ -5,6 +5,6 @@ export default defineConfig({
...okVitestBase,
test: {
...okVitestBase.test,
- exclude: [...okVitestBase.test.exclude, '**/*.network.test.ts', '**/dist/**'],
+ exclude: [...okVitestBase.test.exclude, '**/*.network.test.ts', '.ok-skill-publish-*'],
},
});
diff --git a/test-support/vitest.base.ts b/test-support/vitest.base.ts
index 461e4afcc..7bf6520d5 100644
--- a/test-support/vitest.base.ts
+++ b/test-support/vitest.base.ts
@@ -61,6 +61,8 @@ export const okVitestBase = {
'**/*.e2e.*',
'**/*.dom.test.ts?(x)',
...UNCACHED_TEST_GLOBS,
+ '**/dist/**',
+ '**/.next/**',
],
...(boundedMaxForks === undefined ? {} : { minWorkers: 1, maxWorkers: boundedMaxForks }),
},