Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

fix(audit): recognize Python consumer test suites

wellkilod216de920791
diff --git a/scripts/harness-audit.js b/scripts/harness-audit.js
index a523eca9e..2ce394c38 100644
--- a/scripts/harness-audit.js
+++ b/scripts/harness-audit.js
@@ -3,6 +3,11 @@
const fs = require('fs');
const os = require('os');
const path = require('path');
+const {
+ countPythonTestFiles,
+ gitignoreIgnoresEnvFiles,
+ hasPythonTestSuite,
+} = require('./lib/harness-audit-consumer');
const CATEGORIES = [
'Tool Coverage',
@@ -823,6 +828,7 @@ function getConsumerChecks(rootDir) {
const gitignore = safeRead(rootDir, '.gitignore');
const projectHooks = safeRead(rootDir, '.claude/settings.json');
const pluginInstall = findPluginInstall(rootDir);
+ const pythonTestCount = countPythonTestFiles(rootDir);
return [
{
@@ -876,7 +882,10 @@ function getConsumerChecks(rootDir) {
scopes: ['repo'],
path: 'tests/',
description: 'The project has an automated test entrypoint',
- pass: typeof packageJson?.scripts?.test === 'string' || countFiles(rootDir, 'tests', '.test.js') > 0 || hasFileWithExtension(rootDir, '.', ['.spec.js', '.spec.ts', '.test.ts']),
+ pass: typeof packageJson?.scripts?.test === 'string' ||
+ countFiles(rootDir, 'tests', '.test.js') > 0 ||
+ hasFileWithExtension(rootDir, '.', ['.spec.js', '.spec.ts', '.test.ts']) ||
+ hasPythonTestSuite(rootDir, pythonTestCount),
fix: 'Add a test script or checked-in tests so harness recommendations can be verified automatically.',
},
{
@@ -906,7 +915,8 @@ function getConsumerChecks(rootDir) {
scopes: ['repo'],
path: 'evals/',
description: 'The project has evals or multiple automated tests',
- pass: countFiles(rootDir, 'evals', null) > 0 || countFiles(rootDir, 'tests', '.test.js') >= 3,
+ pass: countFiles(rootDir, 'evals', null) > 0 ||
+ countFiles(rootDir, 'tests', '.test.js') + pythonTestCount >= 3,
fix: 'Add eval fixtures or at least a few focused automated tests for critical flows.',
},
{
@@ -926,7 +936,7 @@ function getConsumerChecks(rootDir) {
scopes: ['repo'],
path: '.gitignore',
description: 'The project ignores common secret env files',
- pass: gitignore.includes('.env'),
+ pass: gitignoreIgnoresEnvFiles(gitignore),
fix: 'Ignore .env-style files in .gitignore so secrets do not land in the repo.',
},
{
diff --git a/scripts/lib/harness-audit-consumer.js b/scripts/lib/harness-audit-consumer.js
new file mode 100644
index 000000000..8d212a59c
--- /dev/null
+++ b/scripts/lib/harness-audit-consumer.js
@@ -0,0 +1,237 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const TOML = require('@iarna/toml');
+
+const IGNORED_SCAN_DIRS = new Set([
+ '.git',
+ '.nox',
+ '.pytest_cache',
+ '.tox',
+ '.venv',
+ '__pycache__',
+ 'build',
+ 'dist',
+ 'env',
+ 'node_modules',
+ 'venv',
+ 'vendor',
+]);
+
+function safeRead(filePath) {
+ try {
+ return fs.readFileSync(filePath, 'utf8');
+ } catch (_error) {
+ return '';
+ }
+}
+
+function walkFiles(rootDir, relativeRoots, predicate) {
+ const projectRoot = fs.realpathSync(path.resolve(rootDir));
+ const files = new Set();
+
+ for (const relativeRoot of relativeRoots) {
+ const scanRoot = path.resolve(projectRoot, relativeRoot);
+ const relative = path.relative(projectRoot, scanRoot);
+ const segments = relative.split(path.sep).filter(Boolean);
+ if (
+ relative === '..' ||
+ relative.startsWith(`..${path.sep}`) ||
+ path.isAbsolute(relative) ||
+ segments.some(segment => IGNORED_SCAN_DIRS.has(segment)) ||
+ !fs.existsSync(scanRoot)
+ ) {
+ continue;
+ }
+ try {
+ const realRoot = fs.realpathSync(scanRoot);
+ const realRelative = path.relative(projectRoot, realRoot);
+ if (
+ realRelative === '..' ||
+ realRelative.startsWith(`..${path.sep}`) ||
+ path.isAbsolute(realRelative)
+ ) {
+ continue;
+ }
+ } catch (_error) {
+ continue;
+ }
+
+ const stack = [scanRoot];
+ while (stack.length > 0) {
+ const current = stack.pop();
+ let entries;
+ try {
+ entries = fs.readdirSync(current, { withFileTypes: true });
+ } catch (_error) {
+ continue;
+ }
+
+ for (const entry of entries) {
+ const nextPath = path.join(current, entry.name);
+ if (entry.isDirectory()) {
+ if (!IGNORED_SCAN_DIRS.has(entry.name)) {
+ stack.push(nextPath);
+ }
+ } else if (entry.isFile() && predicate(entry.name, nextPath)) {
+ files.add(nextPath);
+ }
+ }
+ }
+ }
+
+ return files;
+}
+
+function parseIniTestPaths(text, sectionName) {
+ if (!text) return [];
+ const sectionPattern = new RegExp(
+ `^\\s*\\[${sectionName.replace(/[.*+?^${}()|[\\]\\]/g, '\\$&')}\\]\\s*$`,
+ 'i'
+ );
+ const lines = text.split(/\r?\n/);
+ let inSection = false;
+ let collecting = false;
+ const values = [];
+
+ for (const line of lines) {
+ if (/^\s*\[.*]\s*$/.test(line)) {
+ inSection = sectionPattern.test(line);
+ collecting = false;
+ continue;
+ }
+ if (!inSection || /^\s*[#;]/.test(line)) continue;
+
+ const match = line.match(/^\s*testpaths\s*=\s*(.*)$/i);
+ if (match) {
+ collecting = true;
+ values.push(match[1]);
+ continue;
+ }
+ if (collecting && /^\s+\S/.test(line)) {
+ values.push(line.trim());
+ } else if (line.trim()) {
+ collecting = false;
+ }
+ }
+
+ return values
+ .flatMap(value => value.split(/\s+/))
+ .map(value => value.replace(/^['"]|['"]$/g, '').trim())
+ .filter(Boolean);
+}
+
+function getPytestTestPaths(rootDir) {
+ const paths = [];
+ const pyproject = safeRead(path.join(rootDir, 'pyproject.toml'));
+ if (pyproject) {
+ try {
+ const configured = TOML.parse(pyproject)?.tool?.pytest?.ini_options?.testpaths;
+ if (Array.isArray(configured)) paths.push(...configured);
+ else if (typeof configured === 'string') paths.push(...configured.split(/\s+/));
+ } catch (_error) {
+ // Malformed project metadata is not evidence of a configured test path.
+ }
+ }
+
+ paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'pytest.ini')), 'pytest'));
+ paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'tox.ini')), 'pytest'));
+ paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'setup.cfg')), 'tool:pytest'));
+
+ return [...new Set(paths.filter(value => typeof value === 'string' && value.trim()))];
+}
+
+function isPythonTestFile(fileName) {
+ return /^(test_.*|.*_test)\.py$/i.test(fileName);
+}
+
+function countPythonTestFiles(rootDir) {
+ const roots = ['tests', ...getPytestTestPaths(rootDir)];
+ return walkFiles(rootDir, roots, isPythonTestFile).size;
+}
+
+function hasPythonTestSuite(rootDir, pythonTestCount = countPythonTestFiles(rootDir)) {
+ const pyproject = safeRead(path.join(rootDir, 'pyproject.toml'));
+ const setupCfg = safeRead(path.join(rootDir, 'setup.cfg'));
+ const hasConfig = (
+ /^\s*\[tool\.pytest\.ini_options]\s*$/m.test(pyproject)
+ || /^\s*\[tool:pytest]\s*$/m.test(setupCfg)
+ || fs.existsSync(path.join(rootDir, 'pytest.ini'))
+ || fs.existsSync(path.join(rootDir, 'tox.ini'))
+ );
+ return hasConfig ||
+ pythonTestCount > 0 ||
+ walkFiles(rootDir, ['.'], fileName => fileName === 'conftest.py').size > 0;
+}
+
+function globToRegex(pattern) {
+ let output = '';
+ for (let index = 0; index < pattern.length; index += 1) {
+ const char = pattern[index];
+ if (char === '*' && pattern[index + 1] === '*') {
+ output += '.*';
+ index += 1;
+ } else if (char === '*') {
+ output += '[^/]*';
+ } else if (char === '?') {
+ output += '[^/]';
+ } else {
+ output += char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+ }
+ }
+ return output;
+}
+
+function matchesGitignorePattern(candidate, rawPattern) {
+ const directoryOnly = rawPattern.endsWith('/');
+ const anchored = rawPattern.startsWith('/');
+ const pattern = rawPattern.replace(/^\//, '').replace(/\/$/, '');
+ if (!pattern) return false;
+
+ const body = globToRegex(pattern);
+ const prefix = anchored || pattern.includes('/') ? '^' : '(^|.*/)';
+ const suffix = directoryOnly ? '(/.*)?$' : '$';
+ return new RegExp(`${prefix}${body}${suffix}`).test(candidate);
+}
+
+function gitignoreIgnoresEnvFiles(text) {
+ if (typeof text !== 'string' || !text.trim()) return false;
+ const candidates = [
+ '.env',
+ '.env.local',
+ '.env.production',
+ '.env/example',
+ 'config/.env',
+ 'secrets.env',
+ ];
+ const states = new Map(candidates.map(candidate => [candidate, false]));
+
+ for (const rawLine of text.split(/\r?\n/)) {
+ let line = rawLine.trim();
+ if (!line || line.startsWith('#')) continue;
+
+ let negated = false;
+ if (line.startsWith('!')) {
+ negated = true;
+ line = line.slice(1);
+ } else if (line.startsWith('\\#') || line.startsWith('\\!')) {
+ line = line.slice(1);
+ }
+ if (!line) continue;
+
+ for (const candidate of candidates) {
+ if (matchesGitignorePattern(candidate, line)) {
+ states.set(candidate, !negated);
+ }
+ }
+ }
+
+ return [...states.values()].some(Boolean);
+}
+
+module.exports = {
+ countPythonTestFiles,
+ gitignoreIgnoresEnvFiles,
+ hasPythonTestSuite,
+};
diff --git a/tests/scripts/harness-audit.test.js b/tests/scripts/harness-audit.test.js
index c0087475d..a0ece0aee 100644
--- a/tests/scripts/harness-audit.test.js
+++ b/tests/scripts/harness-audit.test.js
@@ -10,6 +10,7 @@ const { execFileSync, spawnSync } = require('child_process');
const SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'harness-audit.js');
const { parseArgs, findPluginInstall, compareVersionDesc } = require(SCRIPT);
+const { countPythonTestFiles } = require('../../scripts/lib/harness-audit-consumer');
function createTempDir(prefix) {
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
@@ -425,6 +426,125 @@ function runTests() {
}
})) passed++; else failed++;
+ if (test('combines Python and JavaScript tests for eval coverage', () => {
+ const homeDir = createTempDir('harness-audit-python-home-');
+ const projectRoot = createTempDir('harness-audit-python-project-');
+
+ try {
+ fs.mkdirSync(path.join(projectRoot, 'qa'), { recursive: true });
+ fs.mkdirSync(path.join(projectRoot, 'integration'), { recursive: true });
+ fs.mkdirSync(path.join(projectRoot, 'tests'), { recursive: true });
+ fs.mkdirSync(path.join(projectRoot, '.venv', 'tests'), { recursive: true });
+ fs.mkdirSync(path.join(projectRoot, 'node_modules', 'pkg', 'tests'), { recursive: true });
+ fs.writeFileSync(
+ path.join(projectRoot, 'pyproject.toml'),
+ '[tool.pytest.ini_options]\ntestpaths = ["qa", "integration"]\n'
+ );
+ fs.writeFileSync(path.join(projectRoot, 'qa', 'test_alpha.py'), 'def test_alpha(): pass\n');
+ fs.writeFileSync(path.join(projectRoot, 'qa', 'beta_test.py'), 'def test_beta(): pass\n');
+ fs.writeFileSync(path.join(projectRoot, 'tests', 'gamma.test.js'), 'test placeholder\n');
+ fs.writeFileSync(path.join(projectRoot, '.venv', 'tests', 'test_vendor.py'), '');
+ fs.writeFileSync(path.join(projectRoot, 'node_modules', 'pkg', 'tests', 'test_vendor.py'), '');
+
+ assert.strictEqual(countPythonTestFiles(projectRoot), 2);
+ const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir }));
+ assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass));
+ assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass));
+ } finally {
+ cleanup(homeDir);
+ cleanup(projectRoot);
+ }
+ })) passed++; else failed++;
+
+ if (test('recognizes standard pytest suite markers without JavaScript tests', () => {
+ const homeDir = createTempDir('harness-audit-python-markers-home-');
+ const fixtures = [
+ ['pytest.ini', '[pytest]\n'],
+ ['tox.ini', '[tox]\nenvlist = py\n'],
+ ['setup.cfg', '[tool:pytest]\n'],
+ ['quality/conftest.py', ''],
+ ['tests/test_smoke.py', 'def test_smoke(): pass\n'],
+ ];
+
+ try {
+ for (const [relativePath, contents] of fixtures) {
+ const projectRoot = createTempDir('harness-audit-python-marker-');
+ try {
+ fs.mkdirSync(path.dirname(path.join(projectRoot, relativePath)), { recursive: true });
+ fs.writeFileSync(path.join(projectRoot, relativePath), contents);
+ const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir }));
+ assert.ok(
+ parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass),
+ `${relativePath} should identify a Python test suite`
+ );
+ } finally {
+ cleanup(projectRoot);
+ }
+ }
+ } finally {
+ cleanup(homeDir);
+ }
+ })) passed++; else failed++;
+
+ if (test('counts tests from INI testpaths without scanning ignored dependency roots', () => {
+ const homeDir = createTempDir('harness-audit-python-ini-home-');
+ const projectRoot = createTempDir('harness-audit-python-ini-project-');
+
+ try {
+ fs.mkdirSync(path.join(projectRoot, 'quality'), { recursive: true });
+ fs.mkdirSync(path.join(projectRoot, '.venv', 'tests'), { recursive: true });
+ fs.writeFileSync(
+ path.join(projectRoot, 'pytest.ini'),
+ '[pytest]\ntestpaths =\n quality\n .venv/tests\n'
+ );
+ for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) {
+ fs.writeFileSync(path.join(projectRoot, 'quality', name), '');
+ }
+ fs.writeFileSync(path.join(projectRoot, '.venv', 'tests', 'test_vendor.py'), '');
+
+ assert.strictEqual(countPythonTestFiles(projectRoot), 3);
+ const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir }));
+ assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass));
+ assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass));
+ } finally {
+ cleanup(homeDir);
+ cleanup(projectRoot);
+ }
+ })) passed++; else failed++;
+
+ if (test('secret hygiene requires an active gitignore rule for env files', () => {
+ const homeDir = createTempDir('harness-audit-gitignore-home-');
+ const cases = [
+ ['# Do not commit .env files\nnode_modules/\n', false],
+ ['environment.json\n.envexample\n', false],
+ ['.env.example\n', false],
+ ['.env\n', true],
+ ['.env.local\n', true],
+ ['.env.*\n', true],
+ ['*.env\n', true],
+ ['.env/\n', true],
+ ['config/.env\n', true],
+ ['.env\n!.env\n', false],
+ ['.env*\n!.env.example\n', true],
+ ];
+
+ try {
+ for (const [gitignore, expected] of cases) {
+ const projectRoot = createTempDir('harness-audit-gitignore-project-');
+ try {
+ fs.writeFileSync(path.join(projectRoot, '.gitignore'), gitignore);
+ const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir }));
+ const check = parsed.checks.find(candidate => candidate.id === 'consumer-secret-hygiene');
+ assert.strictEqual(check.pass, expected, JSON.stringify({ gitignore, expected }));
+ } finally {
+ cleanup(projectRoot);
+ }
+ }
+ } finally {
+ cleanup(homeDir);
+ }
+ })) passed++; else failed++;
+
if (test('prints no top actions when consumer checks all pass', () => {
const homeDir = createTempDir('harness-audit-passing-home-');
const projectRoot = createTempDir('harness-audit-passing-project-');