The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(audit): recognize Python consumer test suites
wellkilod216de920791
diff --git a/scripts/harness-audit.js b/scripts/harness-audit.jsindex a523eca9e..2ce394c38 100644--- a/scripts/harness-audit.js+++ b/scripts/harness-audit.js@@ -3,6 +3,11 @@const fs = require('fs');const os = require('os');const path = require('path');+const {+ countPythonTestFiles,+ gitignoreIgnoresEnvFiles,+ hasPythonTestSuite,+} = require('./lib/harness-audit-consumer');const CATEGORIES = ['Tool Coverage',@@ -823,6 +828,7 @@ function getConsumerChecks(rootDir) {const gitignore = safeRead(rootDir, '.gitignore');const projectHooks = safeRead(rootDir, '.claude/settings.json');const pluginInstall = findPluginInstall(rootDir);+ const pythonTestCount = countPythonTestFiles(rootDir);return [{@@ -876,7 +882,10 @@ function getConsumerChecks(rootDir) {scopes: ['repo'],path: 'tests/',description: 'The project has an automated test entrypoint',- pass: typeof packageJson?.scripts?.test === 'string' || countFiles(rootDir, 'tests', '.test.js') > 0 || hasFileWithExtension(rootDir, '.', ['.spec.js', '.spec.ts', '.test.ts']),+ pass: typeof packageJson?.scripts?.test === 'string' ||+ countFiles(rootDir, 'tests', '.test.js') > 0 ||+ hasFileWithExtension(rootDir, '.', ['.spec.js', '.spec.ts', '.test.ts']) ||+ hasPythonTestSuite(rootDir, pythonTestCount),fix: 'Add a test script or checked-in tests so harness recommendations can be verified automatically.',},{@@ -906,7 +915,8 @@ function getConsumerChecks(rootDir) {scopes: ['repo'],path: 'evals/',description: 'The project has evals or multiple automated tests',- pass: countFiles(rootDir, 'evals', null) > 0 || countFiles(rootDir, 'tests', '.test.js') >= 3,+ pass: countFiles(rootDir, 'evals', null) > 0 ||+ countFiles(rootDir, 'tests', '.test.js') + pythonTestCount >= 3,fix: 'Add eval fixtures or at least a few focused automated tests for critical flows.',},{@@ -926,7 +936,7 @@ function getConsumerChecks(rootDir) {scopes: ['repo'],path: '.gitignore',description: 'The project ignores common secret env files',- pass: gitignore.includes('.env'),+ pass: gitignoreIgnoresEnvFiles(gitignore),fix: 'Ignore .env-style files in .gitignore so secrets do not land in the repo.',},{diff --git a/scripts/lib/harness-audit-consumer.js b/scripts/lib/harness-audit-consumer.jsnew file mode 100644index 000000000..8d212a59c--- /dev/null+++ b/scripts/lib/harness-audit-consumer.js@@ -0,0 +1,237 @@+'use strict';++const fs = require('fs');+const path = require('path');+const TOML = require('@iarna/toml');++const IGNORED_SCAN_DIRS = new Set([+ '.git',+ '.nox',+ '.pytest_cache',+ '.tox',+ '.venv',+ '__pycache__',+ 'build',+ 'dist',+ 'env',+ 'node_modules',+ 'venv',+ 'vendor',+]);++function safeRead(filePath) {+ try {+ return fs.readFileSync(filePath, 'utf8');+ } catch (_error) {+ return '';+ }+}++function walkFiles(rootDir, relativeRoots, predicate) {+ const projectRoot = fs.realpathSync(path.resolve(rootDir));+ const files = new Set();++ for (const relativeRoot of relativeRoots) {+ const scanRoot = path.resolve(projectRoot, relativeRoot);+ const relative = path.relative(projectRoot, scanRoot);+ const segments = relative.split(path.sep).filter(Boolean);+ if (+ relative === '..' ||+ relative.startsWith(`..${path.sep}`) ||+ path.isAbsolute(relative) ||+ segments.some(segment => IGNORED_SCAN_DIRS.has(segment)) ||+ !fs.existsSync(scanRoot)+ ) {+ continue;+ }+ try {+ const realRoot = fs.realpathSync(scanRoot);+ const realRelative = path.relative(projectRoot, realRoot);+ if (+ realRelative === '..' ||+ realRelative.startsWith(`..${path.sep}`) ||+ path.isAbsolute(realRelative)+ ) {+ continue;+ }+ } catch (_error) {+ continue;+ }++ const stack = [scanRoot];+ while (stack.length > 0) {+ const current = stack.pop();+ let entries;+ try {+ entries = fs.readdirSync(current, { withFileTypes: true });+ } catch (_error) {+ continue;+ }++ for (const entry of entries) {+ const nextPath = path.join(current, entry.name);+ if (entry.isDirectory()) {+ if (!IGNORED_SCAN_DIRS.has(entry.name)) {+ stack.push(nextPath);+ }+ } else if (entry.isFile() && predicate(entry.name, nextPath)) {+ files.add(nextPath);+ }+ }+ }+ }++ return files;+}++function parseIniTestPaths(text, sectionName) {+ if (!text) return [];+ const sectionPattern = new RegExp(+ `^\\s*\\[${sectionName.replace(/[.*+?^${}()|[\\]\\]/g, '\\$&')}\\]\\s*$`,+ 'i'+ );+ const lines = text.split(/\r?\n/);+ let inSection = false;+ let collecting = false;+ const values = [];++ for (const line of lines) {+ if (/^\s*\[.*]\s*$/.test(line)) {+ inSection = sectionPattern.test(line);+ collecting = false;+ continue;+ }+ if (!inSection || /^\s*[#;]/.test(line)) continue;++ const match = line.match(/^\s*testpaths\s*=\s*(.*)$/i);+ if (match) {+ collecting = true;+ values.push(match[1]);+ continue;+ }+ if (collecting && /^\s+\S/.test(line)) {+ values.push(line.trim());+ } else if (line.trim()) {+ collecting = false;+ }+ }++ return values+ .flatMap(value => value.split(/\s+/))+ .map(value => value.replace(/^['"]|['"]$/g, '').trim())+ .filter(Boolean);+}++function getPytestTestPaths(rootDir) {+ const paths = [];+ const pyproject = safeRead(path.join(rootDir, 'pyproject.toml'));+ if (pyproject) {+ try {+ const configured = TOML.parse(pyproject)?.tool?.pytest?.ini_options?.testpaths;+ if (Array.isArray(configured)) paths.push(...configured);+ else if (typeof configured === 'string') paths.push(...configured.split(/\s+/));+ } catch (_error) {+ // Malformed project metadata is not evidence of a configured test path.+ }+ }++ paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'pytest.ini')), 'pytest'));+ paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'tox.ini')), 'pytest'));+ paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'setup.cfg')), 'tool:pytest'));++ return [...new Set(paths.filter(value => typeof value === 'string' && value.trim()))];+}++function isPythonTestFile(fileName) {+ return /^(test_.*|.*_test)\.py$/i.test(fileName);+}++function countPythonTestFiles(rootDir) {+ const roots = ['tests', ...getPytestTestPaths(rootDir)];+ return walkFiles(rootDir, roots, isPythonTestFile).size;+}++function hasPythonTestSuite(rootDir, pythonTestCount = countPythonTestFiles(rootDir)) {+ const pyproject = safeRead(path.join(rootDir, 'pyproject.toml'));+ const setupCfg = safeRead(path.join(rootDir, 'setup.cfg'));+ const hasConfig = (+ /^\s*\[tool\.pytest\.ini_options]\s*$/m.test(pyproject)+ || /^\s*\[tool:pytest]\s*$/m.test(setupCfg)+ || fs.existsSync(path.join(rootDir, 'pytest.ini'))+ || fs.existsSync(path.join(rootDir, 'tox.ini'))+ );+ return hasConfig ||+ pythonTestCount > 0 ||+ walkFiles(rootDir, ['.'], fileName => fileName === 'conftest.py').size > 0;+}++function globToRegex(pattern) {+ let output = '';+ for (let index = 0; index < pattern.length; index += 1) {+ const char = pattern[index];+ if (char === '*' && pattern[index + 1] === '*') {+ output += '.*';+ index += 1;+ } else if (char === '*') {+ output += '[^/]*';+ } else if (char === '?') {+ output += '[^/]';+ } else {+ output += char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+ }+ }+ return output;+}++function matchesGitignorePattern(candidate, rawPattern) {+ const directoryOnly = rawPattern.endsWith('/');+ const anchored = rawPattern.startsWith('/');+ const pattern = rawPattern.replace(/^\//, '').replace(/\/$/, '');+ if (!pattern) return false;++ const body = globToRegex(pattern);+ const prefix = anchored || pattern.includes('/') ? '^' : '(^|.*/)';+ const suffix = directoryOnly ? '(/.*)?$' : '$';+ return new RegExp(`${prefix}${body}${suffix}`).test(candidate);+}++function gitignoreIgnoresEnvFiles(text) {+ if (typeof text !== 'string' || !text.trim()) return false;+ const candidates = [+ '.env',+ '.env.local',+ '.env.production',+ '.env/example',+ 'config/.env',+ 'secrets.env',+ ];+ const states = new Map(candidates.map(candidate => [candidate, false]));++ for (const rawLine of text.split(/\r?\n/)) {+ let line = rawLine.trim();+ if (!line || line.startsWith('#')) continue;++ let negated = false;+ if (line.startsWith('!')) {+ negated = true;+ line = line.slice(1);+ } else if (line.startsWith('\\#') || line.startsWith('\\!')) {+ line = line.slice(1);+ }+ if (!line) continue;++ for (const candidate of candidates) {+ if (matchesGitignorePattern(candidate, line)) {+ states.set(candidate, !negated);+ }+ }+ }++ return [...states.values()].some(Boolean);+}++module.exports = {+ countPythonTestFiles,+ gitignoreIgnoresEnvFiles,+ hasPythonTestSuite,+};diff --git a/tests/scripts/harness-audit.test.js b/tests/scripts/harness-audit.test.jsindex c0087475d..a0ece0aee 100644--- a/tests/scripts/harness-audit.test.js+++ b/tests/scripts/harness-audit.test.js@@ -10,6 +10,7 @@ const { execFileSync, spawnSync } = require('child_process');const SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'harness-audit.js');const { parseArgs, findPluginInstall, compareVersionDesc } = require(SCRIPT);+const { countPythonTestFiles } = require('../../scripts/lib/harness-audit-consumer');function createTempDir(prefix) {return fs.mkdtempSync(path.join(os.tmpdir(), prefix));@@ -425,6 +426,125 @@ function runTests() {}})) passed++; else failed++;+ if (test('combines Python and JavaScript tests for eval coverage', () => {+ const homeDir = createTempDir('harness-audit-python-home-');+ const projectRoot = createTempDir('harness-audit-python-project-');++ try {+ fs.mkdirSync(path.join(projectRoot, 'qa'), { recursive: true });+ fs.mkdirSync(path.join(projectRoot, 'integration'), { recursive: true });+ fs.mkdirSync(path.join(projectRoot, 'tests'), { recursive: true });+ fs.mkdirSync(path.join(projectRoot, '.venv', 'tests'), { recursive: true });+ fs.mkdirSync(path.join(projectRoot, 'node_modules', 'pkg', 'tests'), { recursive: true });+ fs.writeFileSync(+ path.join(projectRoot, 'pyproject.toml'),+ '[tool.pytest.ini_options]\ntestpaths = ["qa", "integration"]\n'+ );+ fs.writeFileSync(path.join(projectRoot, 'qa', 'test_alpha.py'), 'def test_alpha(): pass\n');+ fs.writeFileSync(path.join(projectRoot, 'qa', 'beta_test.py'), 'def test_beta(): pass\n');+ fs.writeFileSync(path.join(projectRoot, 'tests', 'gamma.test.js'), 'test placeholder\n');+ fs.writeFileSync(path.join(projectRoot, '.venv', 'tests', 'test_vendor.py'), '');+ fs.writeFileSync(path.join(projectRoot, 'node_modules', 'pkg', 'tests', 'test_vendor.py'), '');++ assert.strictEqual(countPythonTestFiles(projectRoot), 2);+ const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir }));+ assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass));+ assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass));+ } finally {+ cleanup(homeDir);+ cleanup(projectRoot);+ }+ })) passed++; else failed++;++ if (test('recognizes standard pytest suite markers without JavaScript tests', () => {+ const homeDir = createTempDir('harness-audit-python-markers-home-');+ const fixtures = [+ ['pytest.ini', '[pytest]\n'],+ ['tox.ini', '[tox]\nenvlist = py\n'],+ ['setup.cfg', '[tool:pytest]\n'],+ ['quality/conftest.py', ''],+ ['tests/test_smoke.py', 'def test_smoke(): pass\n'],+ ];++ try {+ for (const [relativePath, contents] of fixtures) {+ const projectRoot = createTempDir('harness-audit-python-marker-');+ try {+ fs.mkdirSync(path.dirname(path.join(projectRoot, relativePath)), { recursive: true });+ fs.writeFileSync(path.join(projectRoot, relativePath), contents);+ const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir }));+ assert.ok(+ parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass),+ `${relativePath} should identify a Python test suite`+ );+ } finally {+ cleanup(projectRoot);+ }+ }+ } finally {+ cleanup(homeDir);+ }+ })) passed++; else failed++;++ if (test('counts tests from INI testpaths without scanning ignored dependency roots', () => {+ const homeDir = createTempDir('harness-audit-python-ini-home-');+ const projectRoot = createTempDir('harness-audit-python-ini-project-');++ try {+ fs.mkdirSync(path.join(projectRoot, 'quality'), { recursive: true });+ fs.mkdirSync(path.join(projectRoot, '.venv', 'tests'), { recursive: true });+ fs.writeFileSync(+ path.join(projectRoot, 'pytest.ini'),+ '[pytest]\ntestpaths =\n quality\n .venv/tests\n'+ );+ for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) {+ fs.writeFileSync(path.join(projectRoot, 'quality', name), '');+ }+ fs.writeFileSync(path.join(projectRoot, '.venv', 'tests', 'test_vendor.py'), '');++ assert.strictEqual(countPythonTestFiles(projectRoot), 3);+ const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir }));+ assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass));+ assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass));+ } finally {+ cleanup(homeDir);+ cleanup(projectRoot);+ }+ })) passed++; else failed++;++ if (test('secret hygiene requires an active gitignore rule for env files', () => {+ const homeDir = createTempDir('harness-audit-gitignore-home-');+ const cases = [+ ['# Do not commit .env files\nnode_modules/\n', false],+ ['environment.json\n.envexample\n', false],+ ['.env.example\n', false],+ ['.env\n', true],+ ['.env.local\n', true],+ ['.env.*\n', true],+ ['*.env\n', true],+ ['.env/\n', true],+ ['config/.env\n', true],+ ['.env\n!.env\n', false],+ ['.env*\n!.env.example\n', true],+ ];++ try {+ for (const [gitignore, expected] of cases) {+ const projectRoot = createTempDir('harness-audit-gitignore-project-');+ try {+ fs.writeFileSync(path.join(projectRoot, '.gitignore'), gitignore);+ const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir }));+ const check = parsed.checks.find(candidate => candidate.id === 'consumer-secret-hygiene');+ assert.strictEqual(check.pass, expected, JSON.stringify({ gitignore, expected }));+ } finally {+ cleanup(projectRoot);+ }+ }+ } finally {+ cleanup(homeDir);+ }+ })) passed++; else failed++;+if (test('prints no top actions when consumer checks all pass', () => {const homeDir = createTempDir('harness-audit-passing-home-');const projectRoot = createTempDir('harness-audit-passing-project-');