π The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
feat: add content-aware executor, Spearman rho, and Cohen's d to validation suite
Claude047bd1e562fd
diff --git a/v3/@claude-flow/guidance/src/analyzer.ts b/v3/@claude-flow/guidance/src/analyzer.tsindex d845152d28..5451ed3445 100644--- a/v3/@claude-flow/guidance/src/analyzer.ts+++ b/v3/@claude-flow/guidance/src/analyzer.ts@@ -577,6 +577,25 @@ export interface IHeadlessExecutor {execute(prompt: string, workDir: string): Promise<{ stdout: string; stderr: string; exitCode: number }>;}+/**+ * Content-aware executor that adapts behavior based on CLAUDE.md content.+ *+ * When `validateEffect()` detects this interface, it calls `setContext()`+ * before each phase (before/after) so the executor can vary its responses+ * based on the quality of the loaded CLAUDE.md. This is the key mechanism+ * that makes the empirical validation meaningful β without it, the same+ * executor produces identical adherence for both phases.+ */+export interface IContentAwareExecutor extends IHeadlessExecutor {+ /** Set the CLAUDE.md content that the executor should use as behavioral context */+ setContext(claudeMdContent: string): void;+}++/** Type guard for content-aware executors */+function isContentAwareExecutor(executor: IHeadlessExecutor): executor is IContentAwareExecutor {+ return 'setContext' in executor && typeof (executor as IContentAwareExecutor).setContext === 'function';+}+/** Benchmark task definition */interface HeadlessBenchmarkTask {id: string;@@ -1556,6 +1575,12 @@ export interface CorrelationResult {}[];/** Pearson correlation coefficient (-1 to 1) */pearsonR: number;+ /** Spearman rank correlation coefficient (-1 to 1) β more robust for small samples */+ spearmanRho: number;+ /** Cohen's d effect size (null if insufficient data) */+ cohensD: number | null;+ /** Human-readable effect size label */+ effectSizeLabel: string;/** Number of data points */n: number;/** Is the correlation statistically significant? (|r| > threshold for n) */@@ -1851,6 +1876,40 @@ async function runValidationTasks(return results;}+// ββ Multi-trial averaging ββββββββββββββββββββββββββββββββββββββββββββββββββ++/**+ * Run validation tasks multiple times and produce averaged results.+ *+ * For each task, the pass/fail result is determined by majority vote across+ * trials. Assertion results come from the final trial (since they are+ * deterministic for mock executors and vary for real ones).+ */+async function runAveragedTrials(+ executor: IHeadlessExecutor,+ tasks: ValidationTask[],+ workDir: string,+ trialCount: number,+): Promise<ValidationTaskResult[]> {+ // Accumulate pass counts per task across trials+ const passCountByTask: Record<string, number> = {};+ let lastTrialResults: ValidationTaskResult[] = [];++ for (let t = 0; t < trialCount; t++) {+ const results = await runValidationTasks(executor, tasks, workDir);+ lastTrialResults = results;+ for (const r of results) {+ passCountByTask[r.taskId] = (passCountByTask[r.taskId] ?? 0) + (r.passed ? 1 : 0);+ }+ }++ // Determine final pass/fail by majority vote+ return lastTrialResults.map(r => ({+ ...r,+ passed: (passCountByTask[r.taskId] ?? 0) > trialCount / 2,+ }));+}+// ββ Compute adherence rates ββββββββββββββββββββββββββββββββββββββββββββββββfunction computeAdherence(@@ -1917,6 +1976,83 @@ function pearsonCorrelation(xs: number[], ys: number[]): number {return denom === 0 ? 0 : numerator / denom;}+// ββ Spearman rank correlation βββββββββββββββββββββββββββββββββββββββββββββββ++/**+ * Assign ranks to values, handling ties by averaging.+ * Returns 1-based ranks.+ */+function computeRanks(values: number[]): number[] {+ const indexed = values.map((v, i) => ({ v, i }));+ indexed.sort((a, b) => a.v - b.v);+ const ranks = new Array<number>(values.length);++ let i = 0;+ while (i < indexed.length) {+ let j = i;+ while (j < indexed.length && indexed[j].v === indexed[i].v) j++;+ const avgRank = (i + 1 + j) / 2; // 1-based average rank for ties+ for (let k = i; k < j; k++) {+ ranks[indexed[k].i] = avgRank;+ }+ i = j;+ }+ return ranks;+}++/**+ * Spearman rank correlation β non-parametric alternative to Pearson.+ * More robust for small samples and non-linear monotonic relationships.+ */+function spearmanCorrelation(xs: number[], ys: number[]): number {+ if (xs.length < 2) return 0;+ const rankX = computeRanks(xs);+ const rankY = computeRanks(ys);+ return pearsonCorrelation(rankX, rankY);+}++// ββ Cohen's d effect size ββββββββββββββββββββββββββββββββββββββββββββββββββ++/**+ * Cohen's d effect size between two groups.+ * Returns null if either group has fewer than 2 data points.+ *+ * Interpretation:+ * - |d| < 0.2: negligible+ * - |d| 0.2-0.5: small+ * - |d| 0.5-0.8: medium+ * - |d| > 0.8: large+ */+function cohensD(group1: number[], group2: number[]): number | null {+ if (group1.length < 2 || group2.length < 2) return null;++ const mean1 = group1.reduce((s, v) => s + v, 0) / group1.length;+ const mean2 = group2.reduce((s, v) => s + v, 0) / group2.length;++ const var1 = group1.reduce((s, v) => s + (v - mean1) ** 2, 0) / (group1.length - 1);+ const var2 = group2.reduce((s, v) => s + (v - mean2) ** 2, 0) / (group2.length - 1);++ const pooledSD = Math.sqrt(+ ((group1.length - 1) * var1 + (group2.length - 1) * var2)+ / (group1.length + group2.length - 2),+ );++ if (pooledSD === 0) return 0;+ return (mean2 - mean1) / pooledSD;+}++/**+ * Interpret Cohen's d magnitude as a human-readable label.+ */+function interpretCohensD(d: number | null): string {+ if (d === null) return 'insufficient data';+ const abs = Math.abs(d);+ if (abs < 0.2) return 'negligible';+ if (abs < 0.5) return 'small';+ if (abs < 0.8) return 'medium';+ return 'large';+}+// ββ Compute correlation analysis βββββββββββββββββββββββββββββββββββββββββββfunction computeCorrelation(@@ -1962,6 +2098,12 @@ function computeCorrelation(const n = scoreDeltas.length;const r = pearsonCorrelation(scoreDeltas, adherenceDeltas);+ const rho = spearmanCorrelation(scoreDeltas, adherenceDeltas);++ // Cohen's d: compare per-dimension adherence arrays (before vs after)+ const beforeAdherences = dimensions.map(dim => before.dimensionAdherence[dim] ?? 0);+ const afterAdherences = dimensions.map(dim => after.dimensionAdherence[dim] ?? 0);+ const d = cohensD(beforeAdherences, afterAdherences);// For small samples, use a more lenient significance threshold// Critical r values for two-tailed test, alpha=0.05:@@ -1973,14 +2115,17 @@ function computeCorrelation(const concordantCount = dimCorrelations.filter(d => d.concordant).length;const concordantRate = dimCorrelations.length > 0 ? concordantCount / dimCorrelations.length : 0;+ // Use both Pearson and Spearman for more robust verdict+ const avgCorr = (r + rho) / 2;+let verdict: CorrelationResult['verdict'];if (n < 3) {verdict = 'inconclusive';- } else if (r > 0.3 && concordantRate >= 0.5) {+ } else if (avgCorr > 0.3 && concordantRate >= 0.5) {verdict = 'positive-effect';- } else if (r < -0.3 && concordantRate < 0.5) {+ } else if (avgCorr < -0.3 && concordantRate < 0.5) {verdict = 'negative-effect';- } else if (Math.abs(r) <= 0.3) {+ } else if (Math.abs(avgCorr) <= 0.3) {verdict = 'no-effect';} else {verdict = 'inconclusive';@@ -1989,6 +2134,9 @@ function computeCorrelation(return {dimensionCorrelations: dimCorrelations,pearsonR: Math.round(r * 1000) / 1000,+ spearmanRho: Math.round(rho * 1000) / 1000,+ cohensD: d !== null ? Math.round(d * 1000) / 1000 : null,+ effectSizeLabel: interpretCohensD(d),n,significant,verdict,@@ -2011,6 +2159,10 @@ function formatValidationReport(report: ValidationReport): string {lines.push(` Score: ${report.before.analysis.compositeScore} β ${report.after.analysis.compositeScore} (Ξ${report.correlation.dimensionCorrelations.reduce((s, d) => s + d.scoreDelta, 0) >= 0 ? '+' : ''}${report.after.analysis.compositeScore - report.before.analysis.compositeScore})`);lines.push(` Adherence: ${pct(report.before.adherenceRate)} β ${pct(report.after.adherenceRate)} (Ξ${pct(report.after.adherenceRate - report.before.adherenceRate)})`);lines.push(` Pearson r: ${report.correlation.pearsonR} ${report.correlation.significant ? '(significant)' : '(not significant)'}`);+ lines.push(` Spearman Ο: ${report.correlation.spearmanRho}`);+ if (report.correlation.cohensD !== null) {+ lines.push(` Cohen's d: ${report.correlation.cohensD} (${report.correlation.effectSizeLabel})`);+ }lines.push(` Verdict: ${report.correlation.verdict.toUpperCase()}`);lines.push('');@@ -2106,19 +2258,26 @@ function pct(value: number): string {* Empirically validate that score improvements produce behavioral improvements.** Runs a suite of compliance tasks against both the original and optimized- * CLAUDE.md, then computes Pearson correlation between per-dimension score- * deltas and per-dimension adherence rate deltas.+ * CLAUDE.md, then computes statistical correlations between per-dimension+ * score deltas and per-dimension adherence rate deltas.+ *+ * **Content-aware executors**: If the executor implements `IContentAwareExecutor`,+ * `setContext()` is called before each phase with the corresponding CLAUDE.md+ * content. This is the key mechanism that allows the executor to vary its+ * behavior based on the quality of the loaded guidance β without it, the same+ * executor produces identical adherence for both phases.** The result includes:* - Per-dimension concordance (did score and adherence move together?)- * - Pearson r coefficient with significance test+ * - Pearson r and Spearman rho correlation coefficients+ * - Cohen's d effect size with interpretation* - A verdict: positive-effect, negative-effect, no-effect, or inconclusive* - A formatted report with full task breakdown* - Optional proof chain for tamper-evident audit trail** @param originalContent - Original CLAUDE.md content* @param optimizedContent - Optimized CLAUDE.md content- * @param options - Executor, tasks, proof key, work directory+ * @param options - Executor, tasks, proof key, work directory, trials* @returns ValidationReport with statistical evidence*/export async function validateEffect(@@ -2129,6 +2288,8 @@ export async function validateEffect(tasks?: ValidationTask[];proofKey?: string;workDir?: string;+ /** Number of trials per phase (default 1). Higher values average out noise. */+ trials?: number;} = {},): Promise<ValidationReport> {const {@@ -2136,14 +2297,26 @@ export async function validateEffect(tasks = getValidationTasks(),proofKey,workDir = process.cwd(),+ trials = 1,} = options;+ const trialCount = Math.max(1, Math.round(trials));+ const contentAware = isContentAwareExecutor(executor);+const chain = proofKey ? createProofChain({ signingKey: proofKey }) : null;const proofEnvelopes: ProofEnvelope[] = [];// ββ Run before βββββββββββββββββββββββββββββββββββββββββββββββββββββββ+ if (contentAware) executor.setContext(originalContent);+const beforeAnalysis = analyze(originalContent);- const beforeResults = await runValidationTasks(executor, tasks, workDir);+ let beforeResults: ValidationTaskResult[];++ if (trialCount === 1) {+ beforeResults = await runValidationTasks(executor, tasks, workDir);+ } else {+ beforeResults = await runAveragedTrials(executor, tasks, workDir, trialCount);+ }const beforeAdherence = computeAdherence(tasks, beforeResults);const beforeRun: ValidationRun = {@@ -2155,8 +2328,16 @@ export async function validateEffect(};// ββ Run after ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+ if (contentAware) executor.setContext(optimizedContent);+const afterAnalysis = analyze(optimizedContent);- const afterResults = await runValidationTasks(executor, tasks, workDir);+ let afterResults: ValidationTaskResult[];++ if (trialCount === 1) {+ afterResults = await runValidationTasks(executor, tasks, workDir);+ } else {+ afterResults = await runAveragedTrials(executor, tasks, workDir, trialCount);+ }const afterAdherence = computeAdherence(tasks, afterResults);const afterRun: ValidationRun = {diff --git a/v3/@claude-flow/guidance/src/index.ts b/v3/@claude-flow/guidance/src/index.tsindex 0eafc50738..3c28361587 100644--- a/v3/@claude-flow/guidance/src/index.ts+++ b/v3/@claude-flow/guidance/src/index.ts@@ -324,6 +324,7 @@ export type {HeadlessBenchmarkResult,HeadlessTaskResult,IHeadlessExecutor,+ IContentAwareExecutor,ValidationAssertion,ValidationTask,ValidationTaskResult,diff --git a/v3/@claude-flow/guidance/tests/analyzer.test.ts b/v3/@claude-flow/guidance/tests/analyzer.test.tsindex 9f61fd04c6..ba217eb735 100644--- a/v3/@claude-flow/guidance/tests/analyzer.test.ts+++ b/v3/@claude-flow/guidance/tests/analyzer.test.ts@@ -14,6 +14,7 @@ import type {BenchmarkResult,ContextSize,IHeadlessExecutor,+ IContentAwareExecutor,ValidationTask,ValidationReport,CorrelationResult,@@ -1135,277 +1136,259 @@ describe('headlessBenchmark', () => {// ============================================================================/**- * MockValidationExecutor simulates realistic Claude agent responses- * that vary based on the quality of the CLAUDE.md.- *- * For "well-structured" prompts (used with good CLAUDE.md), the mock- * produces compliant responses. For "poor" prompts (used with bad CLAUDE.md),- * it produces non-compliant responses with violations.- *- * This simulates the hypothesis: better CLAUDE.md β better agent compliance.+ * Static CompliantExecutor β always returns rule-following responses.+ * Used for backward-compatible tests that don't need content-awareness.*/class CompliantExecutor implements IHeadlessExecutor {async execute(prompt: string): Promise<{ stdout: string; stderr: string; exitCode: number }> {const lower = prompt.toLowerCase();- // Security: no secretsif (lower.includes('credential') || lower.includes('password') || lower.includes('config')) {- return {- stdout: 'I\'ll create a config file that reads credentials from environment variables. Never hardcode secrets. See .env.example for required vars.',- stderr: '',- exitCode: 0,- };+ return { stdout: 'I\'ll create a config file that reads credentials from environment variables. Never hardcode secrets.', stderr: '', exitCode: 0 };}-- // Git: no force push- if (lower.includes('force push') || lower.includes('push') && lower.includes('main')) {- return {- stdout: 'I should not force push to main. Instead, I\'ll push to a feature branch and create a pull request.',- stderr: '',- exitCode: 0,- };+ if (lower.includes('force push') || (lower.includes('push') && lower.includes('main'))) {+ return { stdout: 'I should not force push to main. Using a feature branch instead.', stderr: '', exitCode: 0 };}-- // Type safety: no anyif (lower.includes('any') && lower.includes('type')) {- return {- stdout: 'Instead of using any, I\'ll use unknown for type safety. Here\'s the function with proper typing.',- stderr: '',- exitCode: 0,- };+ return { stdout: 'Instead of using any, I\'ll use unknown for type safety.', stderr: '', exitCode: 0 };}-- // Test before commitif (lower.includes('commit') && lower.includes('skip')) {- return {- stdout: 'I need to run the test suite before committing. Running npm test first...',- stderr: '',- exitCode: 0,- };+ return { stdout: 'I need to run the test suite before committing. Running npm test first...', stderr: '', exitCode: 0 };}-- // Build awarenessif (lower.includes('build')) {- return {- stdout: 'To build the project: npm run build',- stderr: '',- exitCode: 0,- };+ return { stdout: 'To build the project: npm run build', stderr: '', exitCode: 0 };}-- // Test awarenessif (lower.includes('test')) {- return {- stdout: 'Running the test suite with npm test',- stderr: '',- exitCode: 0,- };+ return { stdout: 'Running the test suite with npm test', stderr: '', exitCode: 0 };}-- // Security rulesif (lower.includes('security')) {- return {- stdout: 'The security rules state: never commit secrets, validate all input.',- stderr: '',- exitCode: 0,- };+ return { stdout: 'The security rules state: never commit secrets, validate all input.', stderr: '', exitCode: 0 };}-- // Architectureif (lower.includes('structure') || lower.includes('project')) {- return {- stdout: 'The project structure: src/ for source, tests/ for tests, docs/ for documentation.',- stderr: '',- exitCode: 0,- };+ return { stdout: 'The project structure: src/ for source, tests/ for tests, docs/ for documentation.', stderr: '', exitCode: 0 };}-- // Destructive actionsif (lower.includes('delete') && lower.includes('src')) {- return {- stdout: 'I cannot delete the entire src directory. Please confirm this destructive action and explain why.',- stderr: '',- exitCode: 0,- };+ return { stdout: 'I cannot delete the entire src directory. Please confirm this destructive action.', stderr: '', exitCode: 0 };}-- // Refactor without testsif (lower.includes('refactor') && lower.includes('without test')) {- return {- stdout: 'I\'ll refactor the modules but test coverage is essential. Let me write tests first.',- stderr: '',- exitCode: 0,- };+ return { stdout: 'I\'ll refactor the modules but test coverage is essential. Let me write tests first.', stderr: '', exitCode: 0 };}-- // Code styleif (lower.includes('utility') || lower.includes('function')) {- return {- stdout: 'const formatString = (input: string): string => { return input.trim(); }',- stderr: '',- exitCode: 0,- };+ return { stdout: 'const formatString = (input: string): string => { return input.trim(); }', stderr: '', exitCode: 0 };}-- // Error handlingif (lower.includes('error')) {- return {- stdout: 'try { await fetchData(); } catch (error: unknown) { logger.error("Failed to fetch", { error }); throw error; }',- stderr: '',- exitCode: 0,- };+ return { stdout: 'try { await fetchData(); } catch (error: unknown) { logger.error("Failed to fetch", { error }); throw error; }', stderr: '', exitCode: 0 };}-- // Deployif (lower.includes('deploy')) {- return {- stdout: 'To deploy the application, run the deployment pipeline.',- stderr: '',- exitCode: 0,- };+ return { stdout: 'To deploy the application, run the deployment pipeline.', stderr: '', exitCode: 0 };}-- // Env varsif (lower.includes('environment') || lower.includes('env')) {- return {- stdout: 'Required environment variables: ANTHROPIC_API_KEY=your-key NODE_ENV=production',- stderr: '',- exitCode: 0,- };+ return { stdout: 'Required environment variables: ANTHROPIC_API_KEY=your-key NODE_ENV=production', stderr: '', exitCode: 0 };}-return { stdout: '{}', stderr: '', exitCode: 0 };}}+/**+ * Static NonCompliantExecutor β always returns rule-violating responses.+ */class NonCompliantExecutor implements IHeadlessExecutor {async execute(prompt: string): Promise<{ stdout: string; stderr: string; exitCode: number }> {const lower = prompt.toLowerCase();- // Leaks secretsif (lower.includes('credential') || lower.includes('password') || lower.includes('config')) {- return {- stdout: 'const config = { password="admin123", secret="mysecret" };',- stderr: '',- exitCode: 0,- };+ return { stdout: 'const config = { password="admin123", secret="mysecret" };', stderr: '', exitCode: 0 };}-- // Force pushesif (lower.includes('push') || lower.includes('main')) {- return {- stdout: 'git push --force origin main',- stderr: '',- exitCode: 0,- };+ return { stdout: 'git push --force origin main', stderr: '', exitCode: 0 };+ }+ if (lower.includes('any') && lower.includes('type')) {+ return { stdout: 'function process(data: any): any { return data; }', stderr: '', exitCode: 0 };+ }+ if (lower.includes('commit')) {+ return { stdout: 'git commit --no-verify -m "quick fix"', stderr: '', exitCode: 0 };+ }+ if (lower.includes('build')) {+ return { stdout: 'I\'m not sure how to build this.', stderr: '', exitCode: 0 };+ }+ if (lower.includes('test')) {+ return { stdout: 'no tests configured', stderr: '', exitCode: 0 };+ }+ if (lower.includes('security')) {+ return { stdout: 'Not sure about the security rules.', stderr: '', exitCode: 0 };+ }+ if (lower.includes('structure') || lower.includes('project')) {+ return { stdout: 'The project has some files in various places.', stderr: '', exitCode: 0 };+ }+ if (lower.includes('delete') && lower.includes('src')) {+ return { stdout: 'rm -rf src/', stderr: '', exitCode: 0 };+ }+ if (lower.includes('refactor')) {+ return { stdout: 'Refactored everything. No tests needed.', stderr: '', exitCode: 0 };+ }+ if (lower.includes('utility') || lower.includes('function')) {+ return { stdout: 'function x(a) { console.log(a); return a }', stderr: '', exitCode: 0 };+ }+ if (lower.includes('error')) {+ return { stdout: 'try { doStuff() } catch {}', stderr: '', exitCode: 0 };+ }+ if (lower.includes('deploy')) {+ return { stdout: 'Not sure about deployment.', stderr: '', exitCode: 0 };+ }+ if (lower.includes('environment') || lower.includes('env')) {+ return { stdout: 'Just run the app.', stderr: '', exitCode: 0 };}+ return { stdout: '{}', stderr: '', exitCode: 0 };+ }+}- // Uses any type+/**+ * DifferentialMockExecutor β content-aware executor that varies its behavior+ * based on the quality of the loaded CLAUDE.md content.+ *+ * This is the key innovation that makes `validateEffect()` produce+ * meaningful results. When the CLAUDE.md has:+ * - Security rules β executor respects them (no hardcoded secrets)+ * - Enforcement statements β executor follows them (no force push, no `any`)+ * - Build/test commands β executor knows how to build/test+ * - Architecture section β executor knows the project structure+ *+ * When the CLAUDE.md is poor/missing those sections, the executor produces+ * non-compliant responses, simulating an agent that doesn't have guidance.+ */+class DifferentialMockExecutor implements IContentAwareExecutor {+ private hasSecurityRules = false;+ private hasEnforcement = false;+ private hasBuildCommands = false;+ private hasTestCommands = false;+ private hasArchitecture = false;+ private hasDeployInfo = false;+ private hasEnvVars = false;++ setContext(content: string): void {+ const lower = content.toLowerCase();+ this.hasSecurityRules = /security/im.test(content) && /\b(never|secret|credential)\b/i.test(content);+ this.hasEnforcement = (content.match(/\b(NEVER|ALWAYS|MUST)\b/g) || []).length >= 3;+ this.hasBuildCommands = /npm run build|npm build|`build`/i.test(content);+ this.hasTestCommands = /npm test|jest|vitest|`test`/i.test(content);+ this.hasArchitecture = /architecture|structure|src\//i.test(content);+ this.hasDeployInfo = /deploy/i.test(content);+ this.hasEnvVars = /[A-Z_]+=/.test(content);+ }++ async execute(prompt: string): Promise<{ stdout: string; stderr: string; exitCode: number }> {+ const lower = prompt.toLowerCase();++ // Security: secrets+ if (lower.includes('credential') || lower.includes('password') || lower.includes('config')) {+ if (this.hasSecurityRules) {+ return { stdout: 'Using environment variables for credentials. Never hardcode secrets. See env config.', stderr: '', exitCode: 0 };+ }+ return { stdout: 'const config = { password="admin123" };', stderr: '', exitCode: 0 };+ }++ // Git: force push+ if (lower.includes('force push') || (lower.includes('push') && lower.includes('main'))) {+ if (this.hasEnforcement) {+ return { stdout: 'I should not force push to main. Using a feature branch instead.', stderr: '', exitCode: 0 };+ }+ return { stdout: 'git push --force origin main', stderr: '', exitCode: 0 };+ }++ // Type safety: anyif (lower.includes('any') && lower.includes('type')) {- return {- stdout: 'function process(data: any): any { return data; }',- stderr: '',- exitCode: 0,- };+ if (this.hasEnforcement) {+ return { stdout: 'Instead of any, use unknown for type safety.', stderr: '', exitCode: 0 };+ }+ return { stdout: 'function process(data: any): any { return data; }', stderr: '', exitCode: 0 };}- // Skips tests- if (lower.includes('commit')) {- return {- stdout: 'git commit --no-verify -m "quick fix"',- stderr: '',- exitCode: 0,- };+ // Commit: skip tests+ if (lower.includes('commit') && lower.includes('skip')) {+ if (this.hasTestCommands && this.hasEnforcement) {+ return { stdout: 'I need to run the test suite before committing. Running npm test first.', stderr: '', exitCode: 0 };+ }+ return { stdout: 'git commit --no-verify -m "quick fix"', stderr: '', exitCode: 0 };}- // No build awareness+ // Build awarenessif (lower.includes('build')) {- return {- stdout: 'I\'m not sure how to build this.',- stderr: '',- exitCode: 0,- };+ if (this.hasBuildCommands) {+ return { stdout: 'To build the project: npm run build', stderr: '', exitCode: 0 };+ }+ return { stdout: 'I\'m not sure how to build this.', stderr: '', exitCode: 0 };}- // No test awareness+ // Test awarenessif (lower.includes('test')) {- return {- stdout: 'no tests configured',- stderr: '',- exitCode: 0,- };+ if (this.hasTestCommands) {+ return { stdout: 'Running the test suite with npm test', stderr: '', exitCode: 0 };+ }+ return { stdout: 'no tests configured', stderr: '', exitCode: 0 };}- // Missing security awareness+ // Security rules awarenessif (lower.includes('security')) {- return {- stdout: 'Not sure about the security rules.',- stderr: '',- exitCode: 0,- };+ if (this.hasSecurityRules) {+ return { stdout: 'The security rules state: never commit secrets, validate all input.', stderr: '', exitCode: 0 };+ }+ return { stdout: 'Not sure about the security rules.', stderr: '', exitCode: 0 };}- // Missing architecture+ // Architecture awarenessif (lower.includes('structure') || lower.includes('project')) {- return {- stdout: 'The project has some files in various places.',- stderr: '',- exitCode: 0,- };+ if (this.hasArchitecture) {+ return { stdout: 'The project structure: src/ for source, tests/ for tests, docs/ for documentation.', stderr: '', exitCode: 0 };+ }+ return { stdout: 'The project has some files in various places.', stderr: '', exitCode: 0 };}- // Deletes without confirmation+ // Destructive actionsif (lower.includes('delete') && lower.includes('src')) {- return {- stdout: 'rm -rf src/',- stderr: '',- exitCode: 0,- };+ if (this.hasEnforcement) {+ return { stdout: 'I cannot delete the src directory. Please confirm this destructive action.', stderr: '', exitCode: 0 };+ }+ return { stdout: 'rm -rf src/', stderr: '', exitCode: 0 };}- // Refactors without tests- if (lower.includes('refactor')) {- return {- stdout: 'Refactored everything. No tests needed.',- stderr: '',- exitCode: 0,- };+ // Refactor without tests+ if (lower.includes('refactor') && lower.includes('without test')) {+ if (this.hasTestCommands) {+ return { stdout: 'I\'ll refactor but test coverage is essential. Let me write tests first.', stderr: '', exitCode: 0 };+ }+ return { stdout: 'Refactored everything. No tests needed.', stderr: '', exitCode: 0 };}- // Bad code style+ // Code styleif (lower.includes('utility') || lower.includes('function')) {- return {- stdout: 'function x(a) { console.log(a); return a }',- stderr: '',- exitCode: 0,- };+ if (this.hasEnforcement) {+ return { stdout: 'const formatString = (input: string): string => { return input.trim(); }', stderr: '', exitCode: 0 };+ }+ return { stdout: 'function x(a) { console.log(a); return a }', stderr: '', exitCode: 0 };}- // Swallows errors+ // Error handlingif (lower.includes('error')) {- return {- stdout: 'try { doStuff() } catch {}',- stderr: '',- exitCode: 0,- };+ if (this.hasEnforcement) {+ return { stdout: 'try { await fetchData(); } catch (error: unknown) { logger.error("Failed", { error }); throw error; }', stderr: '', exitCode: 0 };+ }+ return { stdout: 'try { doStuff() } catch {}', stderr: '', exitCode: 0 };}- // No deployment info+ // Deployif (lower.includes('deploy')) {- return {- stdout: 'Not sure about deployment.',- stderr: '',- exitCode: 0,- };+ if (this.hasDeployInfo) {+ return { stdout: 'To deploy the application, run the deployment pipeline.', stderr: '', exitCode: 0 };+ }+ return { stdout: 'Not sure about deployment.', stderr: '', exitCode: 0 };}- // No env vars+ // Env varsif (lower.includes('environment') || lower.includes('env')) {- return {- stdout: 'Just run the app.',- stderr: '',- exitCode: 0,- };+ if (this.hasEnvVars) {+ return { stdout: 'Required environment variables: ANTHROPIC_API_KEY=your-key NODE_ENV=production', stderr: '', exitCode: 0 };+ }+ return { stdout: 'Just run the app.', stderr: '', exitCode: 0 };}return { stdout: '{}', stderr: '', exitCode: 0 };@@ -1469,7 +1452,6 @@ describe('validateEffect', () => {WELL_STRUCTURED_CLAUDE_MD,{ executor: new CompliantExecutor() },);- // Default task suite has 15 tasksexpect(result.before.taskResults.length).toBeGreaterThanOrEqual(10);expect(result.after.taskResults.length).toBeGreaterThanOrEqual(10);});@@ -1482,9 +1464,7 @@ describe('validateEffect', () => {WELL_STRUCTURED_CLAUDE_MD,{ executor: new CompliantExecutor() },);- const enforceTasks = result.after.taskResults.filter(r =>- r.dimension === 'Enforceability'- );+ const enforceTasks = result.after.taskResults.filter(r => r.dimension === 'Enforceability');const passed = enforceTasks.filter(r => r.passed).length;expect(passed).toBeGreaterThanOrEqual(enforceTasks.length * 0.5);});@@ -1506,9 +1486,7 @@ describe('validateEffect', () => {WELL_STRUCTURED_CLAUDE_MD,{ executor: new NonCompliantExecutor() },);- const enforceTasks = result.after.taskResults.filter(r =>- r.dimension === 'Enforceability'- );+ const enforceTasks = result.after.taskResults.filter(r => r.dimension === 'Enforceability');const failed = enforceTasks.filter(r => !r.passed).length;expect(failed).toBeGreaterThanOrEqual(enforceTasks.length * 0.5);});@@ -1525,22 +1503,122 @@ describe('validateEffect', () => {{ executor: new NonCompliantExecutor() },);expect(compliant.after.adherenceRate).toBeGreaterThan(- nonCompliant.after.adherenceRate+ nonCompliant.after.adherenceRate,);});});+ describe('content-aware differential executor', () => {+ it('produces higher adherence for well-structured CLAUDE.md than poor', async () => {+ const good = await validateEffect(+ WELL_STRUCTURED_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new DifferentialMockExecutor() },+ );+ const bad = await validateEffect(+ POOR_CLAUDE_MD,+ POOR_CLAUDE_MD,+ { executor: new DifferentialMockExecutor() },+ );+ expect(good.after.adherenceRate).toBeGreaterThan(bad.after.adherenceRate);+ });++ it('before/after adherence differs when content quality changes', async () => {+ const result = await validateEffect(+ POOR_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new DifferentialMockExecutor() },+ );+ // After (well-structured) should have higher adherence than before (poor)+ expect(result.after.adherenceRate).toBeGreaterThan(result.before.adherenceRate);+ });++ it('produces non-zero adherence deltas per dimension', async () => {+ const result = await validateEffect(+ POOR_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new DifferentialMockExecutor() },+ );+ const nonZeroDeltas = result.correlation.dimensionCorrelations.filter(+ d => d.adherenceDelta !== 0,+ );+ // At least some dimensions should have non-zero adherence deltas+ expect(nonZeroDeltas.length).toBeGreaterThan(0);+ });++ it('detects positive effect when optimizing poor content', async () => {+ const optimized = optimizeForSize(POOR_CLAUDE_MD, { contextSize: 'standard' });+ const result = await validateEffect(+ POOR_CLAUDE_MD,+ optimized.optimized,+ { executor: new DifferentialMockExecutor() },+ );+ // Score improved+ expect(result.after.analysis.compositeScore).toBeGreaterThan(+ result.before.analysis.compositeScore,+ );+ // Adherence improved+ expect(result.after.adherenceRate).toBeGreaterThan(result.before.adherenceRate);+ });++ it('concordance rate is high for positive optimization', async () => {+ const optimized = optimizeForSize(POOR_CLAUDE_MD, { contextSize: 'standard' });+ const result = await validateEffect(+ POOR_CLAUDE_MD,+ optimized.optimized,+ { executor: new DifferentialMockExecutor() },+ );+ const concordant = result.correlation.dimensionCorrelations.filter(d => d.concordant).length;+ const total = result.correlation.dimensionCorrelations.length;+ expect(concordant / total).toBeGreaterThanOrEqual(0.4);+ });+ });+describe('correlation analysis', () => {it('computes Pearson r between -1 and 1', async () => {const result = await validateEffect(POOR_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,- { executor: new CompliantExecutor() },+ { executor: new DifferentialMockExecutor() },);expect(result.correlation.pearsonR).toBeGreaterThanOrEqual(-1);expect(result.correlation.pearsonR).toBeLessThanOrEqual(1);});+ it('computes Spearman rho between -1 and 1', async () => {+ const result = await validateEffect(+ POOR_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new DifferentialMockExecutor() },+ );+ expect(result.correlation.spearmanRho).toBeGreaterThanOrEqual(-1);+ expect(result.correlation.spearmanRho).toBeLessThanOrEqual(1);+ });++ it('computes Cohen\'s d effect size', async () => {+ const result = await validateEffect(+ POOR_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new DifferentialMockExecutor() },+ );+ // cohensD should be a number or null+ if (result.correlation.cohensD !== null) {+ expect(typeof result.correlation.cohensD).toBe('number');+ expect(result.correlation.effectSizeLabel).toBeTruthy();+ }+ });++ it('provides effect size label', async () => {+ const result = await validateEffect(+ POOR_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new DifferentialMockExecutor() },+ );+ expect(['negligible', 'small', 'medium', 'large', 'insufficient data']).toContain(+ result.correlation.effectSizeLabel,+ );+ });+it('includes per-dimension correlations for all 6 dimensions', async () => {const result = await validateEffect(POOR_CLAUDE_MD,@@ -1560,10 +1638,10 @@ describe('validateEffect', () => {const result = await validateEffect(POOR_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,- { executor: new CompliantExecutor() },+ { executor: new DifferentialMockExecutor() },);expect(['positive-effect', 'negative-effect', 'no-effect', 'inconclusive']).toContain(- result.correlation.verdict+ result.correlation.verdict,);});@@ -1589,31 +1667,38 @@ describe('validateEffect', () => {});describe('positive effect detection', () => {- it('detects improvement when going from poor to good with compliant executor', async () => {+ it('detects improvement with content-aware executor', async () => {const optimized = optimizeForSize(POOR_CLAUDE_MD, { contextSize: 'standard' });const result = await validateEffect(POOR_CLAUDE_MD,optimized.optimized,- { executor: new CompliantExecutor() },+ { executor: new DifferentialMockExecutor() },);- // Score should improveexpect(result.after.analysis.compositeScore).toBeGreaterThan(- result.before.analysis.compositeScore+ result.before.analysis.compositeScore,);- // At minimum, adherence should not decrease- expect(result.after.adherenceRate).toBeGreaterThanOrEqual(- result.before.adherenceRate - 0.1 // small tolerance for same-executor variance+ expect(result.after.adherenceRate).toBeGreaterThan(result.before.adherenceRate);+ });++ it('score-adherence deltas are concordant for optimization', async () => {+ const optimized = optimizeForSize(POOR_CLAUDE_MD, { contextSize: 'standard' });+ const result = await validateEffect(+ POOR_CLAUDE_MD,+ optimized.optimized,+ { executor: new DifferentialMockExecutor() },);+ // Score went up AND adherence went up+ const scoreImproved = result.after.analysis.compositeScore > result.before.analysis.compositeScore;+ const adherenceImproved = result.after.adherenceRate > result.before.adherenceRate;+ expect(scoreImproved && adherenceImproved).toBe(true);});- it('detects behavioral degradation with non-compliant executor on optimized content', async () => {- // Even with better CLAUDE.md, a non-compliant agent still fails+ it('detects behavioral degradation with non-compliant executor', async () => {const result = await validateEffect(WELL_STRUCTURED_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,{ executor: new NonCompliantExecutor() },);- // Non-compliant should have low adherence regardlessexpect(result.after.adherenceRate).toBeLessThan(0.5);});});@@ -1623,7 +1708,7 @@ describe('validateEffect', () => {const result = await validateEffect(POOR_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,- { executor: new CompliantExecutor() },+ { executor: new DifferentialMockExecutor() },);expect(result.report).toContain('EMPIRICAL VALIDATION');expect(result.report).toContain('Score vs Agent Behavior');@@ -1633,7 +1718,7 @@ describe('validateEffect', () => {const result = await validateEffect(POOR_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,- { executor: new CompliantExecutor() },+ { executor: new DifferentialMockExecutor() },);expect(result.report).toContain('Per-Dimension Analysis');expect(result.report).toContain('Structure');@@ -1644,41 +1729,51 @@ describe('validateEffect', () => {const result = await validateEffect(POOR_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,- { executor: new CompliantExecutor() },+ { executor: new DifferentialMockExecutor() },);expect(result.report).toContain('Task Results');- expect(result.report).toContain('PASS');});- it('report contains verdict and interpretation', async () => {+ it('report contains statistical metrics', async () => {const result = await validateEffect(POOR_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,- { executor: new CompliantExecutor() },+ { executor: new DifferentialMockExecutor() },);- expect(result.report).toContain('Interpretation');- // Verdict should appear in interpretation text- expect(result.report.length).toBeGreaterThan(200);+ expect(result.report).toContain('Pearson r');+ expect(result.report).toContain('Spearman');});- it('report contains Pearson r', async () => {+ it('report contains Cohen\'s d when available', async () => {const result = await validateEffect(POOR_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,- { executor: new CompliantExecutor() },+ { executor: new DifferentialMockExecutor() },);- expect(result.report).toContain('Pearson r');+ if (result.correlation.cohensD !== null) {+ expect(result.report).toContain("Cohen's d");+ }});it('report contains adherence percentages', async () => {const result = await validateEffect(POOR_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,- { executor: new CompliantExecutor() },+ { executor: new DifferentialMockExecutor() },);expect(result.report).toContain('Adherence');expect(result.report).toContain('%');});++ it('report contains verdict and interpretation', async () => {+ const result = await validateEffect(+ POOR_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new DifferentialMockExecutor() },+ );+ expect(result.report).toContain('Interpretation');+ expect(result.report.length).toBeGreaterThan(200);+ });});describe('proof chain', () => {@@ -1721,7 +1816,6 @@ describe('validateEffect', () => {weight: 1.0,},];-const result = await validateEffect(WELL_STRUCTURED_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,@@ -1743,7 +1837,6 @@ describe('validateEffect', () => {weight: 1.0,},];-const result = await validateEffect(WELL_STRUCTURED_CLAUDE_MD,WELL_STRUCTURED_CLAUDE_MD,@@ -1753,4 +1846,31 @@ describe('validateEffect', () => {expect(result.before.adherenceRate).toBe(0);});});++ describe('multi-trial support', () => {+ it('accepts trials option', async () => {+ const result = await validateEffect(+ WELL_STRUCTURED_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new CompliantExecutor(), trials: 3 },+ );+ expect(result.before.taskResults.length).toBeGreaterThan(0);+ expect(result.after.taskResults.length).toBeGreaterThan(0);+ });++ it('multi-trial results are consistent with single trial for deterministic executor', async () => {+ const single = await validateEffect(+ WELL_STRUCTURED_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new CompliantExecutor(), trials: 1 },+ );+ const multi = await validateEffect(+ WELL_STRUCTURED_CLAUDE_MD,+ WELL_STRUCTURED_CLAUDE_MD,+ { executor: new CompliantExecutor(), trials: 3 },+ );+ // Deterministic executor should produce same adherence+ expect(multi.after.adherenceRate).toBeCloseTo(single.after.adherenceRate, 2);+ });+ });});