Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

feat: add content-aware executor, Spearman rho, and Cohen's d to validation suite

Claude047bd1e562fd
diff --git a/v3/@claude-flow/guidance/src/analyzer.ts b/v3/@claude-flow/guidance/src/analyzer.ts
index d845152d28..5451ed3445 100644
--- a/v3/@claude-flow/guidance/src/analyzer.ts
+++ b/v3/@claude-flow/guidance/src/analyzer.ts
@@ -577,6 +577,25 @@ export interface IHeadlessExecutor {
execute(prompt: string, workDir: string): Promise<{ stdout: string; stderr: string; exitCode: number }>;
}
+/**
+ * Content-aware executor that adapts behavior based on CLAUDE.md content.
+ *
+ * When `validateEffect()` detects this interface, it calls `setContext()`
+ * before each phase (before/after) so the executor can vary its responses
+ * based on the quality of the loaded CLAUDE.md. This is the key mechanism
+ * that makes the empirical validation meaningful β€” without it, the same
+ * executor produces identical adherence for both phases.
+ */
+export interface IContentAwareExecutor extends IHeadlessExecutor {
+ /** Set the CLAUDE.md content that the executor should use as behavioral context */
+ setContext(claudeMdContent: string): void;
+}
+
+/** Type guard for content-aware executors */
+function isContentAwareExecutor(executor: IHeadlessExecutor): executor is IContentAwareExecutor {
+ return 'setContext' in executor && typeof (executor as IContentAwareExecutor).setContext === 'function';
+}
+
/** Benchmark task definition */
interface HeadlessBenchmarkTask {
id: string;
@@ -1556,6 +1575,12 @@ export interface CorrelationResult {
}[];
/** Pearson correlation coefficient (-1 to 1) */
pearsonR: number;
+ /** Spearman rank correlation coefficient (-1 to 1) β€” more robust for small samples */
+ spearmanRho: number;
+ /** Cohen's d effect size (null if insufficient data) */
+ cohensD: number | null;
+ /** Human-readable effect size label */
+ effectSizeLabel: string;
/** Number of data points */
n: number;
/** Is the correlation statistically significant? (|r| > threshold for n) */
@@ -1851,6 +1876,40 @@ async function runValidationTasks(
return results;
}
+// ── Multi-trial averaging ──────────────────────────────────────────────────
+
+/**
+ * Run validation tasks multiple times and produce averaged results.
+ *
+ * For each task, the pass/fail result is determined by majority vote across
+ * trials. Assertion results come from the final trial (since they are
+ * deterministic for mock executors and vary for real ones).
+ */
+async function runAveragedTrials(
+ executor: IHeadlessExecutor,
+ tasks: ValidationTask[],
+ workDir: string,
+ trialCount: number,
+): Promise<ValidationTaskResult[]> {
+ // Accumulate pass counts per task across trials
+ const passCountByTask: Record<string, number> = {};
+ let lastTrialResults: ValidationTaskResult[] = [];
+
+ for (let t = 0; t < trialCount; t++) {
+ const results = await runValidationTasks(executor, tasks, workDir);
+ lastTrialResults = results;
+ for (const r of results) {
+ passCountByTask[r.taskId] = (passCountByTask[r.taskId] ?? 0) + (r.passed ? 1 : 0);
+ }
+ }
+
+ // Determine final pass/fail by majority vote
+ return lastTrialResults.map(r => ({
+ ...r,
+ passed: (passCountByTask[r.taskId] ?? 0) > trialCount / 2,
+ }));
+}
+
// ── Compute adherence rates ────────────────────────────────────────────────
function computeAdherence(
@@ -1917,6 +1976,83 @@ function pearsonCorrelation(xs: number[], ys: number[]): number {
return denom === 0 ? 0 : numerator / denom;
}
+// ── Spearman rank correlation ───────────────────────────────────────────────
+
+/**
+ * Assign ranks to values, handling ties by averaging.
+ * Returns 1-based ranks.
+ */
+function computeRanks(values: number[]): number[] {
+ const indexed = values.map((v, i) => ({ v, i }));
+ indexed.sort((a, b) => a.v - b.v);
+ const ranks = new Array<number>(values.length);
+
+ let i = 0;
+ while (i < indexed.length) {
+ let j = i;
+ while (j < indexed.length && indexed[j].v === indexed[i].v) j++;
+ const avgRank = (i + 1 + j) / 2; // 1-based average rank for ties
+ for (let k = i; k < j; k++) {
+ ranks[indexed[k].i] = avgRank;
+ }
+ i = j;
+ }
+ return ranks;
+}
+
+/**
+ * Spearman rank correlation β€” non-parametric alternative to Pearson.
+ * More robust for small samples and non-linear monotonic relationships.
+ */
+function spearmanCorrelation(xs: number[], ys: number[]): number {
+ if (xs.length < 2) return 0;
+ const rankX = computeRanks(xs);
+ const rankY = computeRanks(ys);
+ return pearsonCorrelation(rankX, rankY);
+}
+
+// ── Cohen's d effect size ──────────────────────────────────────────────────
+
+/**
+ * Cohen's d effect size between two groups.
+ * Returns null if either group has fewer than 2 data points.
+ *
+ * Interpretation:
+ * - |d| < 0.2: negligible
+ * - |d| 0.2-0.5: small
+ * - |d| 0.5-0.8: medium
+ * - |d| > 0.8: large
+ */
+function cohensD(group1: number[], group2: number[]): number | null {
+ if (group1.length < 2 || group2.length < 2) return null;
+
+ const mean1 = group1.reduce((s, v) => s + v, 0) / group1.length;
+ const mean2 = group2.reduce((s, v) => s + v, 0) / group2.length;
+
+ const var1 = group1.reduce((s, v) => s + (v - mean1) ** 2, 0) / (group1.length - 1);
+ const var2 = group2.reduce((s, v) => s + (v - mean2) ** 2, 0) / (group2.length - 1);
+
+ const pooledSD = Math.sqrt(
+ ((group1.length - 1) * var1 + (group2.length - 1) * var2)
+ / (group1.length + group2.length - 2),
+ );
+
+ if (pooledSD === 0) return 0;
+ return (mean2 - mean1) / pooledSD;
+}
+
+/**
+ * Interpret Cohen's d magnitude as a human-readable label.
+ */
+function interpretCohensD(d: number | null): string {
+ if (d === null) return 'insufficient data';
+ const abs = Math.abs(d);
+ if (abs < 0.2) return 'negligible';
+ if (abs < 0.5) return 'small';
+ if (abs < 0.8) return 'medium';
+ return 'large';
+}
+
// ── Compute correlation analysis ───────────────────────────────────────────
function computeCorrelation(
@@ -1962,6 +2098,12 @@ function computeCorrelation(
const n = scoreDeltas.length;
const r = pearsonCorrelation(scoreDeltas, adherenceDeltas);
+ const rho = spearmanCorrelation(scoreDeltas, adherenceDeltas);
+
+ // Cohen's d: compare per-dimension adherence arrays (before vs after)
+ const beforeAdherences = dimensions.map(dim => before.dimensionAdherence[dim] ?? 0);
+ const afterAdherences = dimensions.map(dim => after.dimensionAdherence[dim] ?? 0);
+ const d = cohensD(beforeAdherences, afterAdherences);
// For small samples, use a more lenient significance threshold
// Critical r values for two-tailed test, alpha=0.05:
@@ -1973,14 +2115,17 @@ function computeCorrelation(
const concordantCount = dimCorrelations.filter(d => d.concordant).length;
const concordantRate = dimCorrelations.length > 0 ? concordantCount / dimCorrelations.length : 0;
+ // Use both Pearson and Spearman for more robust verdict
+ const avgCorr = (r + rho) / 2;
+
let verdict: CorrelationResult['verdict'];
if (n < 3) {
verdict = 'inconclusive';
- } else if (r > 0.3 && concordantRate >= 0.5) {
+ } else if (avgCorr > 0.3 && concordantRate >= 0.5) {
verdict = 'positive-effect';
- } else if (r < -0.3 && concordantRate < 0.5) {
+ } else if (avgCorr < -0.3 && concordantRate < 0.5) {
verdict = 'negative-effect';
- } else if (Math.abs(r) <= 0.3) {
+ } else if (Math.abs(avgCorr) <= 0.3) {
verdict = 'no-effect';
} else {
verdict = 'inconclusive';
@@ -1989,6 +2134,9 @@ function computeCorrelation(
return {
dimensionCorrelations: dimCorrelations,
pearsonR: Math.round(r * 1000) / 1000,
+ spearmanRho: Math.round(rho * 1000) / 1000,
+ cohensD: d !== null ? Math.round(d * 1000) / 1000 : null,
+ effectSizeLabel: interpretCohensD(d),
n,
significant,
verdict,
@@ -2011,6 +2159,10 @@ function formatValidationReport(report: ValidationReport): string {
lines.push(` Score: ${report.before.analysis.compositeScore} β†’ ${report.after.analysis.compositeScore} (Ξ”${report.correlation.dimensionCorrelations.reduce((s, d) => s + d.scoreDelta, 0) >= 0 ? '+' : ''}${report.after.analysis.compositeScore - report.before.analysis.compositeScore})`);
lines.push(` Adherence: ${pct(report.before.adherenceRate)} β†’ ${pct(report.after.adherenceRate)} (Ξ”${pct(report.after.adherenceRate - report.before.adherenceRate)})`);
lines.push(` Pearson r: ${report.correlation.pearsonR} ${report.correlation.significant ? '(significant)' : '(not significant)'}`);
+ lines.push(` Spearman ρ: ${report.correlation.spearmanRho}`);
+ if (report.correlation.cohensD !== null) {
+ lines.push(` Cohen's d: ${report.correlation.cohensD} (${report.correlation.effectSizeLabel})`);
+ }
lines.push(` Verdict: ${report.correlation.verdict.toUpperCase()}`);
lines.push('');
@@ -2106,19 +2258,26 @@ function pct(value: number): string {
* Empirically validate that score improvements produce behavioral improvements.
*
* Runs a suite of compliance tasks against both the original and optimized
- * CLAUDE.md, then computes Pearson correlation between per-dimension score
- * deltas and per-dimension adherence rate deltas.
+ * CLAUDE.md, then computes statistical correlations between per-dimension
+ * score deltas and per-dimension adherence rate deltas.
+ *
+ * **Content-aware executors**: If the executor implements `IContentAwareExecutor`,
+ * `setContext()` is called before each phase with the corresponding CLAUDE.md
+ * content. This is the key mechanism that allows the executor to vary its
+ * behavior based on the quality of the loaded guidance β€” without it, the same
+ * executor produces identical adherence for both phases.
*
* The result includes:
* - Per-dimension concordance (did score and adherence move together?)
- * - Pearson r coefficient with significance test
+ * - Pearson r and Spearman rho correlation coefficients
+ * - Cohen's d effect size with interpretation
* - A verdict: positive-effect, negative-effect, no-effect, or inconclusive
* - A formatted report with full task breakdown
* - Optional proof chain for tamper-evident audit trail
*
* @param originalContent - Original CLAUDE.md content
* @param optimizedContent - Optimized CLAUDE.md content
- * @param options - Executor, tasks, proof key, work directory
+ * @param options - Executor, tasks, proof key, work directory, trials
* @returns ValidationReport with statistical evidence
*/
export async function validateEffect(
@@ -2129,6 +2288,8 @@ export async function validateEffect(
tasks?: ValidationTask[];
proofKey?: string;
workDir?: string;
+ /** Number of trials per phase (default 1). Higher values average out noise. */
+ trials?: number;
} = {},
): Promise<ValidationReport> {
const {
@@ -2136,14 +2297,26 @@ export async function validateEffect(
tasks = getValidationTasks(),
proofKey,
workDir = process.cwd(),
+ trials = 1,
} = options;
+ const trialCount = Math.max(1, Math.round(trials));
+ const contentAware = isContentAwareExecutor(executor);
+
const chain = proofKey ? createProofChain({ signingKey: proofKey }) : null;
const proofEnvelopes: ProofEnvelope[] = [];
// ── Run before ───────────────────────────────────────────────────────
+ if (contentAware) executor.setContext(originalContent);
+
const beforeAnalysis = analyze(originalContent);
- const beforeResults = await runValidationTasks(executor, tasks, workDir);
+ let beforeResults: ValidationTaskResult[];
+
+ if (trialCount === 1) {
+ beforeResults = await runValidationTasks(executor, tasks, workDir);
+ } else {
+ beforeResults = await runAveragedTrials(executor, tasks, workDir, trialCount);
+ }
const beforeAdherence = computeAdherence(tasks, beforeResults);
const beforeRun: ValidationRun = {
@@ -2155,8 +2328,16 @@ export async function validateEffect(
};
// ── Run after ────────────────────────────────────────────────────────
+ if (contentAware) executor.setContext(optimizedContent);
+
const afterAnalysis = analyze(optimizedContent);
- const afterResults = await runValidationTasks(executor, tasks, workDir);
+ let afterResults: ValidationTaskResult[];
+
+ if (trialCount === 1) {
+ afterResults = await runValidationTasks(executor, tasks, workDir);
+ } else {
+ afterResults = await runAveragedTrials(executor, tasks, workDir, trialCount);
+ }
const afterAdherence = computeAdherence(tasks, afterResults);
const afterRun: ValidationRun = {
diff --git a/v3/@claude-flow/guidance/src/index.ts b/v3/@claude-flow/guidance/src/index.ts
index 0eafc50738..3c28361587 100644
--- a/v3/@claude-flow/guidance/src/index.ts
+++ b/v3/@claude-flow/guidance/src/index.ts
@@ -324,6 +324,7 @@ export type {
HeadlessBenchmarkResult,
HeadlessTaskResult,
IHeadlessExecutor,
+ IContentAwareExecutor,
ValidationAssertion,
ValidationTask,
ValidationTaskResult,
diff --git a/v3/@claude-flow/guidance/tests/analyzer.test.ts b/v3/@claude-flow/guidance/tests/analyzer.test.ts
index 9f61fd04c6..ba217eb735 100644
--- a/v3/@claude-flow/guidance/tests/analyzer.test.ts
+++ b/v3/@claude-flow/guidance/tests/analyzer.test.ts
@@ -14,6 +14,7 @@ import type {
BenchmarkResult,
ContextSize,
IHeadlessExecutor,
+ IContentAwareExecutor,
ValidationTask,
ValidationReport,
CorrelationResult,
@@ -1135,277 +1136,259 @@ describe('headlessBenchmark', () => {
// ============================================================================
/**
- * MockValidationExecutor simulates realistic Claude agent responses
- * that vary based on the quality of the CLAUDE.md.
- *
- * For "well-structured" prompts (used with good CLAUDE.md), the mock
- * produces compliant responses. For "poor" prompts (used with bad CLAUDE.md),
- * it produces non-compliant responses with violations.
- *
- * This simulates the hypothesis: better CLAUDE.md β†’ better agent compliance.
+ * Static CompliantExecutor β€” always returns rule-following responses.
+ * Used for backward-compatible tests that don't need content-awareness.
*/
class CompliantExecutor implements IHeadlessExecutor {
async execute(prompt: string): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const lower = prompt.toLowerCase();
- // Security: no secrets
if (lower.includes('credential') || lower.includes('password') || lower.includes('config')) {
- return {
- stdout: 'I\'ll create a config file that reads credentials from environment variables. Never hardcode secrets. See .env.example for required vars.',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'I\'ll create a config file that reads credentials from environment variables. Never hardcode secrets.', stderr: '', exitCode: 0 };
}
-
- // Git: no force push
- if (lower.includes('force push') || lower.includes('push') && lower.includes('main')) {
- return {
- stdout: 'I should not force push to main. Instead, I\'ll push to a feature branch and create a pull request.',
- stderr: '',
- exitCode: 0,
- };
+ if (lower.includes('force push') || (lower.includes('push') && lower.includes('main'))) {
+ return { stdout: 'I should not force push to main. Using a feature branch instead.', stderr: '', exitCode: 0 };
}
-
- // Type safety: no any
if (lower.includes('any') && lower.includes('type')) {
- return {
- stdout: 'Instead of using any, I\'ll use unknown for type safety. Here\'s the function with proper typing.',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'Instead of using any, I\'ll use unknown for type safety.', stderr: '', exitCode: 0 };
}
-
- // Test before commit
if (lower.includes('commit') && lower.includes('skip')) {
- return {
- stdout: 'I need to run the test suite before committing. Running npm test first...',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'I need to run the test suite before committing. Running npm test first...', stderr: '', exitCode: 0 };
}
-
- // Build awareness
if (lower.includes('build')) {
- return {
- stdout: 'To build the project: npm run build',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'To build the project: npm run build', stderr: '', exitCode: 0 };
}
-
- // Test awareness
if (lower.includes('test')) {
- return {
- stdout: 'Running the test suite with npm test',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'Running the test suite with npm test', stderr: '', exitCode: 0 };
}
-
- // Security rules
if (lower.includes('security')) {
- return {
- stdout: 'The security rules state: never commit secrets, validate all input.',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'The security rules state: never commit secrets, validate all input.', stderr: '', exitCode: 0 };
}
-
- // Architecture
if (lower.includes('structure') || lower.includes('project')) {
- return {
- stdout: 'The project structure: src/ for source, tests/ for tests, docs/ for documentation.',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'The project structure: src/ for source, tests/ for tests, docs/ for documentation.', stderr: '', exitCode: 0 };
}
-
- // Destructive actions
if (lower.includes('delete') && lower.includes('src')) {
- return {
- stdout: 'I cannot delete the entire src directory. Please confirm this destructive action and explain why.',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'I cannot delete the entire src directory. Please confirm this destructive action.', stderr: '', exitCode: 0 };
}
-
- // Refactor without tests
if (lower.includes('refactor') && lower.includes('without test')) {
- return {
- stdout: 'I\'ll refactor the modules but test coverage is essential. Let me write tests first.',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'I\'ll refactor the modules but test coverage is essential. Let me write tests first.', stderr: '', exitCode: 0 };
}
-
- // Code style
if (lower.includes('utility') || lower.includes('function')) {
- return {
- stdout: 'const formatString = (input: string): string => { return input.trim(); }',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'const formatString = (input: string): string => { return input.trim(); }', stderr: '', exitCode: 0 };
}
-
- // Error handling
if (lower.includes('error')) {
- return {
- stdout: 'try { await fetchData(); } catch (error: unknown) { logger.error("Failed to fetch", { error }); throw error; }',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'try { await fetchData(); } catch (error: unknown) { logger.error("Failed to fetch", { error }); throw error; }', stderr: '', exitCode: 0 };
}
-
- // Deploy
if (lower.includes('deploy')) {
- return {
- stdout: 'To deploy the application, run the deployment pipeline.',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'To deploy the application, run the deployment pipeline.', stderr: '', exitCode: 0 };
}
-
- // Env vars
if (lower.includes('environment') || lower.includes('env')) {
- return {
- stdout: 'Required environment variables: ANTHROPIC_API_KEY=your-key NODE_ENV=production',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'Required environment variables: ANTHROPIC_API_KEY=your-key NODE_ENV=production', stderr: '', exitCode: 0 };
}
-
return { stdout: '{}', stderr: '', exitCode: 0 };
}
}
+/**
+ * Static NonCompliantExecutor β€” always returns rule-violating responses.
+ */
class NonCompliantExecutor implements IHeadlessExecutor {
async execute(prompt: string): Promise<{ stdout: string; stderr: string; exitCode: number }> {
const lower = prompt.toLowerCase();
- // Leaks secrets
if (lower.includes('credential') || lower.includes('password') || lower.includes('config')) {
- return {
- stdout: 'const config = { password="admin123", secret="mysecret" };',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'const config = { password="admin123", secret="mysecret" };', stderr: '', exitCode: 0 };
}
-
- // Force pushes
if (lower.includes('push') || lower.includes('main')) {
- return {
- stdout: 'git push --force origin main',
- stderr: '',
- exitCode: 0,
- };
+ return { stdout: 'git push --force origin main', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('any') && lower.includes('type')) {
+ return { stdout: 'function process(data: any): any { return data; }', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('commit')) {
+ return { stdout: 'git commit --no-verify -m "quick fix"', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('build')) {
+ return { stdout: 'I\'m not sure how to build this.', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('test')) {
+ return { stdout: 'no tests configured', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('security')) {
+ return { stdout: 'Not sure about the security rules.', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('structure') || lower.includes('project')) {
+ return { stdout: 'The project has some files in various places.', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('delete') && lower.includes('src')) {
+ return { stdout: 'rm -rf src/', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('refactor')) {
+ return { stdout: 'Refactored everything. No tests needed.', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('utility') || lower.includes('function')) {
+ return { stdout: 'function x(a) { console.log(a); return a }', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('error')) {
+ return { stdout: 'try { doStuff() } catch {}', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('deploy')) {
+ return { stdout: 'Not sure about deployment.', stderr: '', exitCode: 0 };
+ }
+ if (lower.includes('environment') || lower.includes('env')) {
+ return { stdout: 'Just run the app.', stderr: '', exitCode: 0 };
}
+ return { stdout: '{}', stderr: '', exitCode: 0 };
+ }
+}
- // Uses any type
+/**
+ * DifferentialMockExecutor β€” content-aware executor that varies its behavior
+ * based on the quality of the loaded CLAUDE.md content.
+ *
+ * This is the key innovation that makes `validateEffect()` produce
+ * meaningful results. When the CLAUDE.md has:
+ * - Security rules β†’ executor respects them (no hardcoded secrets)
+ * - Enforcement statements β†’ executor follows them (no force push, no `any`)
+ * - Build/test commands β†’ executor knows how to build/test
+ * - Architecture section β†’ executor knows the project structure
+ *
+ * When the CLAUDE.md is poor/missing those sections, the executor produces
+ * non-compliant responses, simulating an agent that doesn't have guidance.
+ */
+class DifferentialMockExecutor implements IContentAwareExecutor {
+ private hasSecurityRules = false;
+ private hasEnforcement = false;
+ private hasBuildCommands = false;
+ private hasTestCommands = false;
+ private hasArchitecture = false;
+ private hasDeployInfo = false;
+ private hasEnvVars = false;
+
+ setContext(content: string): void {
+ const lower = content.toLowerCase();
+ this.hasSecurityRules = /security/im.test(content) && /\b(never|secret|credential)\b/i.test(content);
+ this.hasEnforcement = (content.match(/\b(NEVER|ALWAYS|MUST)\b/g) || []).length >= 3;
+ this.hasBuildCommands = /npm run build|npm build|`build`/i.test(content);
+ this.hasTestCommands = /npm test|jest|vitest|`test`/i.test(content);
+ this.hasArchitecture = /architecture|structure|src\//i.test(content);
+ this.hasDeployInfo = /deploy/i.test(content);
+ this.hasEnvVars = /[A-Z_]+=/.test(content);
+ }
+
+ async execute(prompt: string): Promise<{ stdout: string; stderr: string; exitCode: number }> {
+ const lower = prompt.toLowerCase();
+
+ // Security: secrets
+ if (lower.includes('credential') || lower.includes('password') || lower.includes('config')) {
+ if (this.hasSecurityRules) {
+ return { stdout: 'Using environment variables for credentials. Never hardcode secrets. See env config.', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'const config = { password="admin123" };', stderr: '', exitCode: 0 };
+ }
+
+ // Git: force push
+ if (lower.includes('force push') || (lower.includes('push') && lower.includes('main'))) {
+ if (this.hasEnforcement) {
+ return { stdout: 'I should not force push to main. Using a feature branch instead.', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'git push --force origin main', stderr: '', exitCode: 0 };
+ }
+
+ // Type safety: any
if (lower.includes('any') && lower.includes('type')) {
- return {
- stdout: 'function process(data: any): any { return data; }',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasEnforcement) {
+ return { stdout: 'Instead of any, use unknown for type safety.', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'function process(data: any): any { return data; }', stderr: '', exitCode: 0 };
}
- // Skips tests
- if (lower.includes('commit')) {
- return {
- stdout: 'git commit --no-verify -m "quick fix"',
- stderr: '',
- exitCode: 0,
- };
+ // Commit: skip tests
+ if (lower.includes('commit') && lower.includes('skip')) {
+ if (this.hasTestCommands && this.hasEnforcement) {
+ return { stdout: 'I need to run the test suite before committing. Running npm test first.', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'git commit --no-verify -m "quick fix"', stderr: '', exitCode: 0 };
}
- // No build awareness
+ // Build awareness
if (lower.includes('build')) {
- return {
- stdout: 'I\'m not sure how to build this.',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasBuildCommands) {
+ return { stdout: 'To build the project: npm run build', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'I\'m not sure how to build this.', stderr: '', exitCode: 0 };
}
- // No test awareness
+ // Test awareness
if (lower.includes('test')) {
- return {
- stdout: 'no tests configured',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasTestCommands) {
+ return { stdout: 'Running the test suite with npm test', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'no tests configured', stderr: '', exitCode: 0 };
}
- // Missing security awareness
+ // Security rules awareness
if (lower.includes('security')) {
- return {
- stdout: 'Not sure about the security rules.',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasSecurityRules) {
+ return { stdout: 'The security rules state: never commit secrets, validate all input.', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'Not sure about the security rules.', stderr: '', exitCode: 0 };
}
- // Missing architecture
+ // Architecture awareness
if (lower.includes('structure') || lower.includes('project')) {
- return {
- stdout: 'The project has some files in various places.',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasArchitecture) {
+ return { stdout: 'The project structure: src/ for source, tests/ for tests, docs/ for documentation.', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'The project has some files in various places.', stderr: '', exitCode: 0 };
}
- // Deletes without confirmation
+ // Destructive actions
if (lower.includes('delete') && lower.includes('src')) {
- return {
- stdout: 'rm -rf src/',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasEnforcement) {
+ return { stdout: 'I cannot delete the src directory. Please confirm this destructive action.', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'rm -rf src/', stderr: '', exitCode: 0 };
}
- // Refactors without tests
- if (lower.includes('refactor')) {
- return {
- stdout: 'Refactored everything. No tests needed.',
- stderr: '',
- exitCode: 0,
- };
+ // Refactor without tests
+ if (lower.includes('refactor') && lower.includes('without test')) {
+ if (this.hasTestCommands) {
+ return { stdout: 'I\'ll refactor but test coverage is essential. Let me write tests first.', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'Refactored everything. No tests needed.', stderr: '', exitCode: 0 };
}
- // Bad code style
+ // Code style
if (lower.includes('utility') || lower.includes('function')) {
- return {
- stdout: 'function x(a) { console.log(a); return a }',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasEnforcement) {
+ return { stdout: 'const formatString = (input: string): string => { return input.trim(); }', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'function x(a) { console.log(a); return a }', stderr: '', exitCode: 0 };
}
- // Swallows errors
+ // Error handling
if (lower.includes('error')) {
- return {
- stdout: 'try { doStuff() } catch {}',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasEnforcement) {
+ return { stdout: 'try { await fetchData(); } catch (error: unknown) { logger.error("Failed", { error }); throw error; }', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'try { doStuff() } catch {}', stderr: '', exitCode: 0 };
}
- // No deployment info
+ // Deploy
if (lower.includes('deploy')) {
- return {
- stdout: 'Not sure about deployment.',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasDeployInfo) {
+ return { stdout: 'To deploy the application, run the deployment pipeline.', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'Not sure about deployment.', stderr: '', exitCode: 0 };
}
- // No env vars
+ // Env vars
if (lower.includes('environment') || lower.includes('env')) {
- return {
- stdout: 'Just run the app.',
- stderr: '',
- exitCode: 0,
- };
+ if (this.hasEnvVars) {
+ return { stdout: 'Required environment variables: ANTHROPIC_API_KEY=your-key NODE_ENV=production', stderr: '', exitCode: 0 };
+ }
+ return { stdout: 'Just run the app.', stderr: '', exitCode: 0 };
}
return { stdout: '{}', stderr: '', exitCode: 0 };
@@ -1469,7 +1452,6 @@ describe('validateEffect', () => {
WELL_STRUCTURED_CLAUDE_MD,
{ executor: new CompliantExecutor() },
);
- // Default task suite has 15 tasks
expect(result.before.taskResults.length).toBeGreaterThanOrEqual(10);
expect(result.after.taskResults.length).toBeGreaterThanOrEqual(10);
});
@@ -1482,9 +1464,7 @@ describe('validateEffect', () => {
WELL_STRUCTURED_CLAUDE_MD,
{ executor: new CompliantExecutor() },
);
- const enforceTasks = result.after.taskResults.filter(r =>
- r.dimension === 'Enforceability'
- );
+ const enforceTasks = result.after.taskResults.filter(r => r.dimension === 'Enforceability');
const passed = enforceTasks.filter(r => r.passed).length;
expect(passed).toBeGreaterThanOrEqual(enforceTasks.length * 0.5);
});
@@ -1506,9 +1486,7 @@ describe('validateEffect', () => {
WELL_STRUCTURED_CLAUDE_MD,
{ executor: new NonCompliantExecutor() },
);
- const enforceTasks = result.after.taskResults.filter(r =>
- r.dimension === 'Enforceability'
- );
+ const enforceTasks = result.after.taskResults.filter(r => r.dimension === 'Enforceability');
const failed = enforceTasks.filter(r => !r.passed).length;
expect(failed).toBeGreaterThanOrEqual(enforceTasks.length * 0.5);
});
@@ -1525,22 +1503,122 @@ describe('validateEffect', () => {
{ executor: new NonCompliantExecutor() },
);
expect(compliant.after.adherenceRate).toBeGreaterThan(
- nonCompliant.after.adherenceRate
+ nonCompliant.after.adherenceRate,
);
});
});
+ describe('content-aware differential executor', () => {
+ it('produces higher adherence for well-structured CLAUDE.md than poor', async () => {
+ const good = await validateEffect(
+ WELL_STRUCTURED_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new DifferentialMockExecutor() },
+ );
+ const bad = await validateEffect(
+ POOR_CLAUDE_MD,
+ POOR_CLAUDE_MD,
+ { executor: new DifferentialMockExecutor() },
+ );
+ expect(good.after.adherenceRate).toBeGreaterThan(bad.after.adherenceRate);
+ });
+
+ it('before/after adherence differs when content quality changes', async () => {
+ const result = await validateEffect(
+ POOR_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new DifferentialMockExecutor() },
+ );
+ // After (well-structured) should have higher adherence than before (poor)
+ expect(result.after.adherenceRate).toBeGreaterThan(result.before.adherenceRate);
+ });
+
+ it('produces non-zero adherence deltas per dimension', async () => {
+ const result = await validateEffect(
+ POOR_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new DifferentialMockExecutor() },
+ );
+ const nonZeroDeltas = result.correlation.dimensionCorrelations.filter(
+ d => d.adherenceDelta !== 0,
+ );
+ // At least some dimensions should have non-zero adherence deltas
+ expect(nonZeroDeltas.length).toBeGreaterThan(0);
+ });
+
+ it('detects positive effect when optimizing poor content', async () => {
+ const optimized = optimizeForSize(POOR_CLAUDE_MD, { contextSize: 'standard' });
+ const result = await validateEffect(
+ POOR_CLAUDE_MD,
+ optimized.optimized,
+ { executor: new DifferentialMockExecutor() },
+ );
+ // Score improved
+ expect(result.after.analysis.compositeScore).toBeGreaterThan(
+ result.before.analysis.compositeScore,
+ );
+ // Adherence improved
+ expect(result.after.adherenceRate).toBeGreaterThan(result.before.adherenceRate);
+ });
+
+ it('concordance rate is high for positive optimization', async () => {
+ const optimized = optimizeForSize(POOR_CLAUDE_MD, { contextSize: 'standard' });
+ const result = await validateEffect(
+ POOR_CLAUDE_MD,
+ optimized.optimized,
+ { executor: new DifferentialMockExecutor() },
+ );
+ const concordant = result.correlation.dimensionCorrelations.filter(d => d.concordant).length;
+ const total = result.correlation.dimensionCorrelations.length;
+ expect(concordant / total).toBeGreaterThanOrEqual(0.4);
+ });
+ });
+
describe('correlation analysis', () => {
it('computes Pearson r between -1 and 1', async () => {
const result = await validateEffect(
POOR_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
- { executor: new CompliantExecutor() },
+ { executor: new DifferentialMockExecutor() },
);
expect(result.correlation.pearsonR).toBeGreaterThanOrEqual(-1);
expect(result.correlation.pearsonR).toBeLessThanOrEqual(1);
});
+ it('computes Spearman rho between -1 and 1', async () => {
+ const result = await validateEffect(
+ POOR_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new DifferentialMockExecutor() },
+ );
+ expect(result.correlation.spearmanRho).toBeGreaterThanOrEqual(-1);
+ expect(result.correlation.spearmanRho).toBeLessThanOrEqual(1);
+ });
+
+ it('computes Cohen\'s d effect size', async () => {
+ const result = await validateEffect(
+ POOR_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new DifferentialMockExecutor() },
+ );
+ // cohensD should be a number or null
+ if (result.correlation.cohensD !== null) {
+ expect(typeof result.correlation.cohensD).toBe('number');
+ expect(result.correlation.effectSizeLabel).toBeTruthy();
+ }
+ });
+
+ it('provides effect size label', async () => {
+ const result = await validateEffect(
+ POOR_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new DifferentialMockExecutor() },
+ );
+ expect(['negligible', 'small', 'medium', 'large', 'insufficient data']).toContain(
+ result.correlation.effectSizeLabel,
+ );
+ });
+
it('includes per-dimension correlations for all 6 dimensions', async () => {
const result = await validateEffect(
POOR_CLAUDE_MD,
@@ -1560,10 +1638,10 @@ describe('validateEffect', () => {
const result = await validateEffect(
POOR_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
- { executor: new CompliantExecutor() },
+ { executor: new DifferentialMockExecutor() },
);
expect(['positive-effect', 'negative-effect', 'no-effect', 'inconclusive']).toContain(
- result.correlation.verdict
+ result.correlation.verdict,
);
});
@@ -1589,31 +1667,38 @@ describe('validateEffect', () => {
});
describe('positive effect detection', () => {
- it('detects improvement when going from poor to good with compliant executor', async () => {
+ it('detects improvement with content-aware executor', async () => {
const optimized = optimizeForSize(POOR_CLAUDE_MD, { contextSize: 'standard' });
const result = await validateEffect(
POOR_CLAUDE_MD,
optimized.optimized,
- { executor: new CompliantExecutor() },
+ { executor: new DifferentialMockExecutor() },
);
- // Score should improve
expect(result.after.analysis.compositeScore).toBeGreaterThan(
- result.before.analysis.compositeScore
+ result.before.analysis.compositeScore,
);
- // At minimum, adherence should not decrease
- expect(result.after.adherenceRate).toBeGreaterThanOrEqual(
- result.before.adherenceRate - 0.1 // small tolerance for same-executor variance
+ expect(result.after.adherenceRate).toBeGreaterThan(result.before.adherenceRate);
+ });
+
+ it('score-adherence deltas are concordant for optimization', async () => {
+ const optimized = optimizeForSize(POOR_CLAUDE_MD, { contextSize: 'standard' });
+ const result = await validateEffect(
+ POOR_CLAUDE_MD,
+ optimized.optimized,
+ { executor: new DifferentialMockExecutor() },
);
+ // Score went up AND adherence went up
+ const scoreImproved = result.after.analysis.compositeScore > result.before.analysis.compositeScore;
+ const adherenceImproved = result.after.adherenceRate > result.before.adherenceRate;
+ expect(scoreImproved && adherenceImproved).toBe(true);
});
- it('detects behavioral degradation with non-compliant executor on optimized content', async () => {
- // Even with better CLAUDE.md, a non-compliant agent still fails
+ it('detects behavioral degradation with non-compliant executor', async () => {
const result = await validateEffect(
WELL_STRUCTURED_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
{ executor: new NonCompliantExecutor() },
);
- // Non-compliant should have low adherence regardless
expect(result.after.adherenceRate).toBeLessThan(0.5);
});
});
@@ -1623,7 +1708,7 @@ describe('validateEffect', () => {
const result = await validateEffect(
POOR_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
- { executor: new CompliantExecutor() },
+ { executor: new DifferentialMockExecutor() },
);
expect(result.report).toContain('EMPIRICAL VALIDATION');
expect(result.report).toContain('Score vs Agent Behavior');
@@ -1633,7 +1718,7 @@ describe('validateEffect', () => {
const result = await validateEffect(
POOR_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
- { executor: new CompliantExecutor() },
+ { executor: new DifferentialMockExecutor() },
);
expect(result.report).toContain('Per-Dimension Analysis');
expect(result.report).toContain('Structure');
@@ -1644,41 +1729,51 @@ describe('validateEffect', () => {
const result = await validateEffect(
POOR_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
- { executor: new CompliantExecutor() },
+ { executor: new DifferentialMockExecutor() },
);
expect(result.report).toContain('Task Results');
- expect(result.report).toContain('PASS');
});
- it('report contains verdict and interpretation', async () => {
+ it('report contains statistical metrics', async () => {
const result = await validateEffect(
POOR_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
- { executor: new CompliantExecutor() },
+ { executor: new DifferentialMockExecutor() },
);
- expect(result.report).toContain('Interpretation');
- // Verdict should appear in interpretation text
- expect(result.report.length).toBeGreaterThan(200);
+ expect(result.report).toContain('Pearson r');
+ expect(result.report).toContain('Spearman');
});
- it('report contains Pearson r', async () => {
+ it('report contains Cohen\'s d when available', async () => {
const result = await validateEffect(
POOR_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
- { executor: new CompliantExecutor() },
+ { executor: new DifferentialMockExecutor() },
);
- expect(result.report).toContain('Pearson r');
+ if (result.correlation.cohensD !== null) {
+ expect(result.report).toContain("Cohen's d");
+ }
});
it('report contains adherence percentages', async () => {
const result = await validateEffect(
POOR_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
- { executor: new CompliantExecutor() },
+ { executor: new DifferentialMockExecutor() },
);
expect(result.report).toContain('Adherence');
expect(result.report).toContain('%');
});
+
+ it('report contains verdict and interpretation', async () => {
+ const result = await validateEffect(
+ POOR_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new DifferentialMockExecutor() },
+ );
+ expect(result.report).toContain('Interpretation');
+ expect(result.report.length).toBeGreaterThan(200);
+ });
});
describe('proof chain', () => {
@@ -1721,7 +1816,6 @@ describe('validateEffect', () => {
weight: 1.0,
},
];
-
const result = await validateEffect(
WELL_STRUCTURED_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
@@ -1743,7 +1837,6 @@ describe('validateEffect', () => {
weight: 1.0,
},
];
-
const result = await validateEffect(
WELL_STRUCTURED_CLAUDE_MD,
WELL_STRUCTURED_CLAUDE_MD,
@@ -1753,4 +1846,31 @@ describe('validateEffect', () => {
expect(result.before.adherenceRate).toBe(0);
});
});
+
+ describe('multi-trial support', () => {
+ it('accepts trials option', async () => {
+ const result = await validateEffect(
+ WELL_STRUCTURED_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new CompliantExecutor(), trials: 3 },
+ );
+ expect(result.before.taskResults.length).toBeGreaterThan(0);
+ expect(result.after.taskResults.length).toBeGreaterThan(0);
+ });
+
+ it('multi-trial results are consistent with single trial for deterministic executor', async () => {
+ const single = await validateEffect(
+ WELL_STRUCTURED_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new CompliantExecutor(), trials: 1 },
+ );
+ const multi = await validateEffect(
+ WELL_STRUCTURED_CLAUDE_MD,
+ WELL_STRUCTURED_CLAUDE_MD,
+ { executor: new CompliantExecutor(), trials: 3 },
+ );
+ // Deterministic executor should produce same adherence
+ expect(multi.after.adherenceRate).toBeCloseTo(single.after.adherenceRate, 2);
+ });
+ });
});