Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

feat: add manifest validator and evolution pipeline tests

Claudedb9cedc5eaa1
diff --git a/v3/@claude-flow/guidance/src/manifest-validator.ts b/v3/@claude-flow/guidance/src/manifest-validator.ts
new file mode 100644
index 0000000000..0b0995e209
--- /dev/null
+++ b/v3/@claude-flow/guidance/src/manifest-validator.ts
@@ -0,0 +1,1139 @@
+/**
+ * Manifest Validator & Conformance Suite
+ *
+ * Validates AgentCellManifest documents against the Agentic Container spec,
+ * computes risk scores, selects execution lanes, and fails closed on any
+ * validation error. The ConformanceSuite runs golden traces through an
+ * evaluator to prove the platform behaves as specified.
+ *
+ * @module @claude-flow/guidance/manifest-validator
+ */
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+const SUPPORTED_API_VERSION = 'agentic_cells.v0_1';
+const SHA256_DIGEST_RE = /^sha256:[a-f0-9]{64}$/;
+
+/** Maximum budget limits (sanity caps) */
+const MAX_BUDGET_LIMITS = {
+ maxWallClockSeconds: 86_400, // 24 hours
+ maxToolCalls: 100_000,
+ maxBytesEgress: 10_737_418_240, // 10 GiB
+ maxTokensInMtok: 100, // 100M tokens
+ maxTokensOutMtok: 100, // 100M tokens
+ maxMemoryWrites: 1_000_000,
+} as const;
+
+/** Data sensitivity levels ordered by severity */
+const DATA_SENSITIVITY_LEVELS = ['public', 'internal', 'confidential', 'restricted'] as const;
+type DataSensitivity = typeof DATA_SENSITIVITY_LEVELS[number];
+
+/** Write modes for memory policy */
+const WRITE_MODES = ['append', 'overwrite', 'merge'] as const;
+type WriteMode = typeof WRITE_MODES[number];
+
+/** Authority scopes for memory policy */
+const AUTHORITY_SCOPES = ['self', 'team', 'tenant', 'global'] as const;
+type AuthorityScope = typeof AUTHORITY_SCOPES[number];
+
+/** Known tool names the system recognizes */
+const KNOWN_TOOLS = new Set([
+ 'Read', 'Write', 'Edit', 'MultiEdit', 'Glob', 'Grep',
+ 'Bash', 'Task', 'TodoWrite', 'NotebookEdit', 'WebFetch', 'WebSearch',
+ 'mcp_memory', 'mcp_swarm', 'mcp_hooks', 'mcp_agent',
+]);
+
+/** Trace levels for observability */
+const TRACE_LEVELS = ['none', 'errors', 'decisions', 'full'] as const;
+
+/** Execution lanes ordered by privilege (lowest to highest) */
+const LANES = ['wasm', 'sandboxed', 'native'] as const;
+type Lane = typeof LANES[number];
+
+// ============================================================================
+// AgentCellManifest Interface
+// ============================================================================
+
+/**
+ * The manifest describing an agent cell per the Agentic Container spec.
+ */
+export interface AgentCellManifest {
+ /** API version string (must be 'agentic_cells.v0_1') */
+ apiVersion: string;
+
+ /** Cell identity */
+ cell: {
+ name: string;
+ purpose: string;
+ ownerTenant: string;
+ codeRef: {
+ kind: string;
+ digest: string;
+ entry: string;
+ };
+ };
+
+ /** Lane execution policy */
+ lanePolicy: {
+ portabilityRequired: boolean;
+ needsNativeThreads: boolean;
+ preferredLane: Lane;
+ maxRiskScore: number;
+ };
+
+ /** Resource budgets */
+ budgets: {
+ maxWallClockSeconds: number;
+ maxToolCalls: number;
+ maxBytesEgress: number;
+ maxTokensInMtok: number;
+ maxTokensOutMtok: number;
+ maxMemoryWrites: number;
+ };
+
+ /** Data handling policy */
+ dataPolicy: {
+ dataSensitivity: DataSensitivity;
+ piiAllowed: boolean;
+ retentionDays: number;
+ exportControls: {
+ allowedRegions: string[];
+ blockedRegions: string[];
+ };
+ };
+
+ /** Tool usage policy */
+ toolPolicy: {
+ toolsAllowed: string[];
+ networkAllowlist: string[];
+ writeActionsRequireConfirmation: boolean;
+ };
+
+ /** Memory system policy */
+ memoryPolicy: {
+ namespace: string;
+ authorityScope: AuthorityScope;
+ writeMode: WriteMode;
+ requiresCoherenceGate: boolean;
+ requiresAntiHallucinationGate: boolean;
+ };
+
+ /** Observability configuration */
+ observability: {
+ traceLevel: typeof TRACE_LEVELS[number];
+ emitArtifacts: boolean;
+ artifactBucket: string;
+ };
+}
+
+// ============================================================================
+// Validation Result Types
+// ============================================================================
+
+/**
+ * A single validation error or warning.
+ */
+export interface ValidationError {
+ /** Error code (e.g., 'MISSING_FIELD', 'INVALID_DIGEST', 'BUDGET_EXCEED') */
+ code: string;
+ /** JSON path to the problematic field */
+ field: string;
+ /** Human-readable description */
+ message: string;
+ /** Severity level */
+ severity: 'error';
+}
+
+/**
+ * A single validation warning.
+ */
+export interface ValidationWarning {
+ /** Warning code */
+ code: string;
+ /** JSON path to the problematic field */
+ field: string;
+ /** Human-readable description */
+ message: string;
+ /** Severity level */
+ severity: 'warning';
+}
+
+/**
+ * Complete validation result for a manifest.
+ */
+export interface ValidationResult {
+ /** Whether the manifest passed all validation checks */
+ valid: boolean;
+ /** Validation errors (each causes rejection) */
+ errors: ValidationError[];
+ /** Validation warnings (informational, do not block admission) */
+ warnings: ValidationWarning[];
+ /** Admission decision: admit, reject, or review */
+ admissionDecision: 'admit' | 'reject' | 'review';
+ /** Selected execution lane (null if rejected) */
+ laneSelection: Lane | null;
+ /** Computed risk score (0-100) */
+ riskScore: number;
+}
+
+// ============================================================================
+// ManifestValidator
+// ============================================================================
+
+/**
+ * Validates AgentCellManifest documents against the Agentic Container spec.
+ *
+ * Fails closed: any validation error results in a 'reject' decision.
+ * Warnings alone do not block admission but may trigger a 'review' decision
+ * when the risk score is between thresholds.
+ */
+export class ManifestValidator {
+ /** Risk score threshold: below this, admit. Above reject threshold, reject. Between, review. */
+ private readonly admitThreshold: number;
+ private readonly rejectThreshold: number;
+
+ constructor(options?: { admitThreshold?: number; rejectThreshold?: number }) {
+ this.admitThreshold = options?.admitThreshold ?? 30;
+ this.rejectThreshold = options?.rejectThreshold ?? 70;
+ }
+
+ /**
+ * Validate a manifest, compute its risk score, select a lane, and decide admission.
+ *
+ * FAILS CLOSED: any validation error leads to reject.
+ */
+ validate(manifest: AgentCellManifest): ValidationResult {
+ const errors: ValidationError[] = [];
+ const warnings: ValidationWarning[] = [];
+
+ // Structural validation
+ errors.push(...this.validateRequiredFields(manifest));
+ errors.push(...this.validateApiVersion(manifest));
+ errors.push(...this.validateDigest(manifest));
+ errors.push(...this.validateBudgets(manifest.budgets));
+ errors.push(...this.validateToolPolicy(manifest.toolPolicy));
+ errors.push(...this.validateDataPolicy(manifest.dataPolicy));
+ warnings.push(...this.validateWarnings(manifest));
+
+ // Compute risk score (even if there are errors, for diagnostics)
+ const riskScore = this.computeRiskScore(manifest);
+
+ // FAIL CLOSED: any error means reject
+ if (errors.length > 0) {
+ return {
+ valid: false,
+ errors,
+ warnings,
+ admissionDecision: 'reject',
+ laneSelection: null,
+ riskScore,
+ };
+ }
+
+ // Lane selection
+ const laneSelection = this.selectLane(manifest, riskScore);
+
+ // Admission decision based on risk score
+ let admissionDecision: 'admit' | 'reject' | 'review';
+ if (riskScore > this.rejectThreshold) {
+ admissionDecision = 'reject';
+ } else if (riskScore > this.admitThreshold) {
+ admissionDecision = 'review';
+ } else {
+ admissionDecision = 'admit';
+ }
+
+ return {
+ valid: true,
+ errors,
+ warnings,
+ admissionDecision,
+ laneSelection,
+ riskScore,
+ };
+ }
+
+ /**
+ * Compute a risk score (0-100) from tool risk, data sensitivity, and privilege surface.
+ *
+ * Components:
+ * - tool_risk (0-40): based on tool types and network access
+ * - data_sensitivity (0-30): based on sensitivity level and PII
+ * - privilege_surface (0-30): based on memory scope, write mode, native threads
+ */
+ computeRiskScore(manifest: AgentCellManifest): number {
+ let toolRisk = 0;
+ let dataSensitivity = 0;
+ let privilegeSurface = 0;
+
+ // --- Tool risk (0-40) ---
+ const tools = manifest.toolPolicy?.toolsAllowed ?? [];
+ const networkList = manifest.toolPolicy?.networkAllowlist ?? [];
+
+ // Bash/command execution is high risk
+ if (tools.includes('Bash') || tools.includes('bash')) {
+ toolRisk += 15;
+ }
+ // Task spawning
+ if (tools.includes('Task') || tools.includes('task')) {
+ toolRisk += 8;
+ }
+ // Write operations
+ if (tools.some(t => ['Write', 'Edit', 'MultiEdit', 'NotebookEdit'].includes(t))) {
+ toolRisk += 5;
+ }
+ // MCP tools
+ if (tools.some(t => t.startsWith('mcp_'))) {
+ toolRisk += 5;
+ }
+ // Network access
+ if (networkList.length > 0) {
+ toolRisk += 5;
+ }
+ // Wildcard in network (already caught as error if not privileged, but score anyway)
+ if (networkList.some(h => h === '*' || h.startsWith('*.'))) {
+ toolRisk += 10;
+ }
+ // No confirmation on writes
+ if (manifest.toolPolicy && !manifest.toolPolicy.writeActionsRequireConfirmation) {
+ toolRisk += 3;
+ }
+
+ toolRisk = Math.min(toolRisk, 40);
+
+ // --- Data sensitivity (0-30) ---
+ const sensitivityIndex = DATA_SENSITIVITY_LEVELS.indexOf(
+ manifest.dataPolicy?.dataSensitivity as DataSensitivity
+ );
+ if (sensitivityIndex >= 0) {
+ dataSensitivity += sensitivityIndex * 8; // 0, 8, 16, 24
+ }
+ if (manifest.dataPolicy?.piiAllowed) {
+ dataSensitivity += 6;
+ }
+ dataSensitivity = Math.min(dataSensitivity, 30);
+
+ // --- Privilege surface (0-30) ---
+ const scopeIndex = AUTHORITY_SCOPES.indexOf(
+ manifest.memoryPolicy?.authorityScope as AuthorityScope
+ );
+ if (scopeIndex >= 0) {
+ privilegeSurface += scopeIndex * 5; // 0, 5, 10, 15
+ }
+ if (manifest.memoryPolicy?.writeMode === 'overwrite') {
+ privilegeSurface += 5;
+ }
+ if (manifest.lanePolicy?.needsNativeThreads) {
+ privilegeSurface += 8;
+ }
+ if (manifest.memoryPolicy && !manifest.memoryPolicy.requiresCoherenceGate) {
+ privilegeSurface += 3;
+ }
+ if (manifest.memoryPolicy && !manifest.memoryPolicy.requiresAntiHallucinationGate) {
+ privilegeSurface += 3;
+ }
+
+ privilegeSurface = Math.min(privilegeSurface, 30);
+
+ return Math.min(toolRisk + dataSensitivity + privilegeSurface, 100);
+ }
+
+ /**
+ * Select the execution lane based on risk score and manifest policy.
+ *
+ * Lane selection rules:
+ * - If portabilityRequired or risk <= 30: wasm
+ * - If needsNativeThreads and risk > 50: native
+ * - Otherwise: sandboxed
+ * - Always respect preferredLane if risk score allows it
+ * - Risk exceeding maxRiskScore forces the most restrictive lane
+ */
+ selectLane(manifest: AgentCellManifest, riskScore: number): Lane {
+ const policy = manifest.lanePolicy;
+
+ // If risk exceeds the manifest's own maxRiskScore, force wasm
+ if (riskScore > policy.maxRiskScore) {
+ return 'wasm';
+ }
+
+ // Portability requirement forces wasm
+ if (policy.portabilityRequired) {
+ return 'wasm';
+ }
+
+ // Native threads require native lane
+ if (policy.needsNativeThreads) {
+ // Only grant native if risk is acceptable
+ if (riskScore <= 50) {
+ return 'native';
+ }
+ return 'sandboxed';
+ }
+
+ // Low risk can go to wasm
+ if (riskScore <= 20) {
+ return policy.preferredLane;
+ }
+
+ // Medium risk gets sandboxed
+ if (riskScore <= 50) {
+ // Respect preference if it's not native
+ if (policy.preferredLane !== 'native') {
+ return policy.preferredLane;
+ }
+ return 'sandboxed';
+ }
+
+ // High risk gets wasm
+ return 'wasm';
+ }
+
+ /**
+ * Validate budget values: no negatives, within sanity limits.
+ */
+ validateBudgets(budgets: AgentCellManifest['budgets']): ValidationError[] {
+ const errors: ValidationError[] = [];
+
+ if (!budgets) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: 'budgets',
+ message: 'Budget configuration is required',
+ severity: 'error',
+ });
+ return errors;
+ }
+
+ const budgetFields: Array<{
+ key: keyof typeof budgets;
+ max: number;
+ }> = [
+ { key: 'maxWallClockSeconds', max: MAX_BUDGET_LIMITS.maxWallClockSeconds },
+ { key: 'maxToolCalls', max: MAX_BUDGET_LIMITS.maxToolCalls },
+ { key: 'maxBytesEgress', max: MAX_BUDGET_LIMITS.maxBytesEgress },
+ { key: 'maxTokensInMtok', max: MAX_BUDGET_LIMITS.maxTokensInMtok },
+ { key: 'maxTokensOutMtok', max: MAX_BUDGET_LIMITS.maxTokensOutMtok },
+ { key: 'maxMemoryWrites', max: MAX_BUDGET_LIMITS.maxMemoryWrites },
+ ];
+
+ for (const { key, max } of budgetFields) {
+ const value = budgets[key];
+ if (value === undefined || value === null) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: `budgets.${key}`,
+ message: `Budget field "${key}" is required`,
+ severity: 'error',
+ });
+ continue;
+ }
+
+ if (typeof value !== 'number' || Number.isNaN(value)) {
+ errors.push({
+ code: 'INVALID_TYPE',
+ field: `budgets.${key}`,
+ message: `Budget field "${key}" must be a number`,
+ severity: 'error',
+ });
+ continue;
+ }
+
+ if (value < 0) {
+ errors.push({
+ code: 'BUDGET_NEGATIVE',
+ field: `budgets.${key}`,
+ message: `Budget field "${key}" must not be negative (got ${value})`,
+ severity: 'error',
+ });
+ }
+
+ if (value > max) {
+ errors.push({
+ code: 'BUDGET_EXCEED',
+ field: `budgets.${key}`,
+ message: `Budget field "${key}" exceeds maximum (${value} > ${max})`,
+ severity: 'error',
+ });
+ }
+ }
+
+ return errors;
+ }
+
+ /**
+ * Validate tool policy: network allowlist must not contain wildcards
+ * unless the cell explicitly has Bash (privileged).
+ */
+ validateToolPolicy(toolPolicy: AgentCellManifest['toolPolicy']): ValidationError[] {
+ const errors: ValidationError[] = [];
+
+ if (!toolPolicy) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: 'toolPolicy',
+ message: 'Tool policy is required',
+ severity: 'error',
+ });
+ return errors;
+ }
+
+ if (!Array.isArray(toolPolicy.toolsAllowed)) {
+ errors.push({
+ code: 'INVALID_TYPE',
+ field: 'toolPolicy.toolsAllowed',
+ message: 'toolsAllowed must be an array',
+ severity: 'error',
+ });
+ }
+
+ if (!Array.isArray(toolPolicy.networkAllowlist)) {
+ errors.push({
+ code: 'INVALID_TYPE',
+ field: 'toolPolicy.networkAllowlist',
+ message: 'networkAllowlist must be an array',
+ severity: 'error',
+ });
+ }
+
+ // Check for wildcards in network allowlist
+ const isPrivileged = Array.isArray(toolPolicy.toolsAllowed) &&
+ toolPolicy.toolsAllowed.includes('Bash');
+
+ if (Array.isArray(toolPolicy.networkAllowlist)) {
+ for (let i = 0; i < toolPolicy.networkAllowlist.length; i++) {
+ const entry = toolPolicy.networkAllowlist[i];
+ if (entry === '*' || entry.startsWith('*.')) {
+ if (!isPrivileged) {
+ errors.push({
+ code: 'WILDCARD_NETWORK',
+ field: `toolPolicy.networkAllowlist[${i}]`,
+ message: `Wildcard "${entry}" in network allowlist requires privileged access (Bash tool)`,
+ severity: 'error',
+ });
+ }
+ }
+ }
+ }
+
+ return errors;
+ }
+
+ /**
+ * Validate data policy fields.
+ */
+ validateDataPolicy(dataPolicy: AgentCellManifest['dataPolicy']): ValidationError[] {
+ const errors: ValidationError[] = [];
+
+ if (!dataPolicy) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: 'dataPolicy',
+ message: 'Data policy is required',
+ severity: 'error',
+ });
+ return errors;
+ }
+
+ if (!DATA_SENSITIVITY_LEVELS.includes(dataPolicy.dataSensitivity as DataSensitivity)) {
+ errors.push({
+ code: 'INVALID_ENUM',
+ field: 'dataPolicy.dataSensitivity',
+ message: `dataSensitivity must be one of: ${DATA_SENSITIVITY_LEVELS.join(', ')} (got "${dataPolicy.dataSensitivity}")`,
+ severity: 'error',
+ });
+ }
+
+ if (typeof dataPolicy.retentionDays !== 'number' || dataPolicy.retentionDays < 0) {
+ errors.push({
+ code: 'INVALID_VALUE',
+ field: 'dataPolicy.retentionDays',
+ message: 'retentionDays must be a non-negative number',
+ severity: 'error',
+ });
+ }
+
+ if (!dataPolicy.exportControls) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: 'dataPolicy.exportControls',
+ message: 'exportControls is required in data policy',
+ severity: 'error',
+ });
+ } else {
+ if (!Array.isArray(dataPolicy.exportControls.allowedRegions)) {
+ errors.push({
+ code: 'INVALID_TYPE',
+ field: 'dataPolicy.exportControls.allowedRegions',
+ message: 'allowedRegions must be an array',
+ severity: 'error',
+ });
+ }
+ if (!Array.isArray(dataPolicy.exportControls.blockedRegions)) {
+ errors.push({
+ code: 'INVALID_TYPE',
+ field: 'dataPolicy.exportControls.blockedRegions',
+ message: 'blockedRegions must be an array',
+ severity: 'error',
+ });
+ }
+
+ // Check for overlap between allowed and blocked regions
+ if (
+ Array.isArray(dataPolicy.exportControls.allowedRegions) &&
+ Array.isArray(dataPolicy.exportControls.blockedRegions)
+ ) {
+ const overlap = dataPolicy.exportControls.allowedRegions.filter(
+ r => dataPolicy.exportControls.blockedRegions.includes(r)
+ );
+ if (overlap.length > 0) {
+ errors.push({
+ code: 'REGION_CONFLICT',
+ field: 'dataPolicy.exportControls',
+ message: `Regions appear in both allowed and blocked lists: ${overlap.join(', ')}`,
+ severity: 'error',
+ });
+ }
+ }
+ }
+
+ return errors;
+ }
+
+ // ===== Private validation helpers =====
+
+ private validateRequiredFields(manifest: AgentCellManifest): ValidationError[] {
+ const errors: ValidationError[] = [];
+
+ if (!manifest) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: '',
+ message: 'Manifest is required',
+ severity: 'error',
+ });
+ return errors;
+ }
+
+ // Top-level required sections
+ const requiredSections = [
+ 'apiVersion', 'cell', 'lanePolicy', 'budgets',
+ 'dataPolicy', 'toolPolicy', 'memoryPolicy', 'observability',
+ ] as const;
+
+ for (const section of requiredSections) {
+ if (manifest[section] === undefined || manifest[section] === null) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: section,
+ message: `Required field "${section}" is missing`,
+ severity: 'error',
+ });
+ }
+ }
+
+ // Cell sub-fields
+ if (manifest.cell) {
+ for (const field of ['name', 'purpose', 'ownerTenant'] as const) {
+ if (!manifest.cell[field]) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: `cell.${field}`,
+ message: `Required field "cell.${field}" is missing`,
+ severity: 'error',
+ });
+ }
+ }
+
+ if (!manifest.cell.codeRef) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: 'cell.codeRef',
+ message: 'Required field "cell.codeRef" is missing',
+ severity: 'error',
+ });
+ } else {
+ for (const field of ['kind', 'digest', 'entry'] as const) {
+ if (!manifest.cell.codeRef[field]) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: `cell.codeRef.${field}`,
+ message: `Required field "cell.codeRef.${field}" is missing`,
+ severity: 'error',
+ });
+ }
+ }
+ }
+ }
+
+ // Memory policy sub-fields
+ if (manifest.memoryPolicy) {
+ if (!manifest.memoryPolicy.namespace) {
+ errors.push({
+ code: 'MISSING_FIELD',
+ field: 'memoryPolicy.namespace',
+ message: 'Required field "memoryPolicy.namespace" is missing',
+ severity: 'error',
+ });
+ }
+
+ if (!AUTHORITY_SCOPES.includes(manifest.memoryPolicy.authorityScope as AuthorityScope)) {
+ errors.push({
+ code: 'INVALID_ENUM',
+ field: 'memoryPolicy.authorityScope',
+ message: `authorityScope must be one of: ${AUTHORITY_SCOPES.join(', ')}`,
+ severity: 'error',
+ });
+ }
+
+ if (!WRITE_MODES.includes(manifest.memoryPolicy.writeMode as WriteMode)) {
+ errors.push({
+ code: 'INVALID_ENUM',
+ field: 'memoryPolicy.writeMode',
+ message: `writeMode must be one of: ${WRITE_MODES.join(', ')}`,
+ severity: 'error',
+ });
+ }
+ }
+
+ // Observability sub-fields
+ if (manifest.observability) {
+ if (!TRACE_LEVELS.includes(manifest.observability.traceLevel as typeof TRACE_LEVELS[number])) {
+ errors.push({
+ code: 'INVALID_ENUM',
+ field: 'observability.traceLevel',
+ message: `traceLevel must be one of: ${TRACE_LEVELS.join(', ')}`,
+ severity: 'error',
+ });
+ }
+ }
+
+ return errors;
+ }
+
+ private validateApiVersion(manifest: AgentCellManifest): ValidationError[] {
+ if (!manifest.apiVersion) return []; // caught by requiredFields
+
+ if (manifest.apiVersion !== SUPPORTED_API_VERSION) {
+ return [{
+ code: 'UNSUPPORTED_API_VERSION',
+ field: 'apiVersion',
+ message: `API version "${manifest.apiVersion}" is not supported (expected "${SUPPORTED_API_VERSION}")`,
+ severity: 'error',
+ }];
+ }
+ return [];
+ }
+
+ private validateDigest(manifest: AgentCellManifest): ValidationError[] {
+ if (!manifest.cell?.codeRef?.digest) return []; // caught by requiredFields
+
+ if (!SHA256_DIGEST_RE.test(manifest.cell.codeRef.digest)) {
+ return [{
+ code: 'INVALID_DIGEST',
+ field: 'cell.codeRef.digest',
+ message: `Digest must match "sha256:<64 hex chars>" format (got "${manifest.cell.codeRef.digest}")`,
+ severity: 'error',
+ }];
+ }
+ return [];
+ }
+
+ private validateWarnings(manifest: AgentCellManifest): ValidationWarning[] {
+ const warnings: ValidationWarning[] = [];
+
+ // Warn about unknown tools
+ if (manifest.toolPolicy?.toolsAllowed) {
+ for (const tool of manifest.toolPolicy.toolsAllowed) {
+ if (!KNOWN_TOOLS.has(tool)) {
+ warnings.push({
+ code: 'UNKNOWN_TOOL',
+ field: 'toolPolicy.toolsAllowed',
+ message: `Tool "${tool}" is not a recognized system tool`,
+ severity: 'warning',
+ });
+ }
+ }
+ }
+
+ // Warn if both coherence and anti-hallucination gates are disabled
+ if (
+ manifest.memoryPolicy &&
+ !manifest.memoryPolicy.requiresCoherenceGate &&
+ !manifest.memoryPolicy.requiresAntiHallucinationGate
+ ) {
+ warnings.push({
+ code: 'NO_MEMORY_GATES',
+ field: 'memoryPolicy',
+ message: 'Both coherence and anti-hallucination gates are disabled; memory writes are ungated',
+ severity: 'warning',
+ });
+ }
+
+ // Warn about high retention with sensitive data
+ if (
+ manifest.dataPolicy &&
+ manifest.dataPolicy.dataSensitivity === 'restricted' &&
+ manifest.dataPolicy.retentionDays > 30
+ ) {
+ warnings.push({
+ code: 'HIGH_RETENTION_SENSITIVE',
+ field: 'dataPolicy.retentionDays',
+ message: `Retention of ${manifest.dataPolicy.retentionDays} days is high for restricted data`,
+ severity: 'warning',
+ });
+ }
+
+ // Warn if no trace level is set to full but artifacts are emitted
+ if (
+ manifest.observability &&
+ manifest.observability.emitArtifacts &&
+ manifest.observability.traceLevel === 'none'
+ ) {
+ warnings.push({
+ code: 'ARTIFACTS_WITHOUT_TRACING',
+ field: 'observability',
+ message: 'Artifact emission is enabled but trace level is "none"',
+ severity: 'warning',
+ });
+ }
+
+ return warnings;
+ }
+}
+
+// ============================================================================
+// Golden Trace Types
+// ============================================================================
+
+/**
+ * A single event within a golden trace.
+ */
+export interface GoldenTraceEvent {
+ /** Sequence number within the trace */
+ seq: number;
+ /** Type of event (e.g., 'command', 'tool-use', 'memory-write', 'budget-check') */
+ eventType: string;
+ /** Event payload */
+ payload: Record<string, unknown>;
+ /** Expected outcome from the platform */
+ expectedOutcome: 'allow' | 'deny' | 'warn';
+}
+
+/**
+ * A complete golden trace including events and expected decisions.
+ */
+export interface GoldenTrace {
+ /** Unique trace identifier */
+ traceId: string;
+ /** Human-readable name */
+ name: string;
+ /** Description of what the trace verifies */
+ description: string;
+ /** Ordered sequence of events */
+ events: GoldenTraceEvent[];
+ /** Map from event seq (as string) to expected decision string */
+ expectedDecisions: Record<string, string>;
+ /** Map from memory key to expected parent chain for lineage verification */
+ expectedMemoryLineage: Record<string, string[]>;
+}
+
+// ============================================================================
+// ConformanceResult
+// ============================================================================
+
+/**
+ * Result of running the conformance suite.
+ */
+export interface ConformanceResult {
+ /** Whether all events matched their expected outcomes */
+ passed: boolean;
+ /** Total number of events evaluated */
+ totalEvents: number;
+ /** Number of events that matched expectations */
+ matchedEvents: number;
+ /** Details of any mismatches */
+ mismatches: Array<{
+ traceId: string;
+ seq: number;
+ expected: string;
+ actual: string;
+ details: unknown;
+ }>;
+}
+
+// ============================================================================
+// ConformanceSuite
+// ============================================================================
+
+/**
+ * Runs golden traces through an evaluator and reports conformance.
+ *
+ * Each trace contains events with expected outcomes. The suite feeds every
+ * event to the evaluator and compares the actual decision to the expectation.
+ */
+export class ConformanceSuite {
+ private traces: GoldenTrace[] = [];
+
+ /**
+ * Add a golden trace to the suite.
+ */
+ addTrace(trace: GoldenTrace): void {
+ this.traces.push(trace);
+ }
+
+ /**
+ * Run every event in every trace through the evaluator and compare
+ * actual decisions against expected outcomes.
+ */
+ run(
+ evaluator: (event: GoldenTraceEvent) => { decision: string; details: unknown },
+ ): ConformanceResult {
+ let totalEvents = 0;
+ let matchedEvents = 0;
+ const mismatches: ConformanceResult['mismatches'] = [];
+
+ for (const trace of this.traces) {
+ for (const event of trace.events) {
+ totalEvents++;
+
+ const { decision, details } = evaluator(event);
+ const expected = trace.expectedDecisions[String(event.seq)] ?? event.expectedOutcome;
+
+ if (decision === expected) {
+ matchedEvents++;
+ } else {
+ mismatches.push({
+ traceId: trace.traceId,
+ seq: event.seq,
+ expected,
+ actual: decision,
+ details,
+ });
+ }
+ }
+ }
+
+ return {
+ passed: mismatches.length === 0,
+ totalEvents,
+ matchedEvents,
+ mismatches,
+ };
+ }
+
+ /**
+ * Get all registered traces.
+ */
+ getTraces(): GoldenTrace[] {
+ return [...this.traces];
+ }
+
+ /**
+ * Create built-in default golden traces that verify core platform invariants:
+ *
+ * 1. Destructive command blocked
+ * 2. Secret detected and blocked
+ * 3. Budget exceeded and denied
+ * 4. Memory write without evidence blocked
+ * 5. Valid operation allowed
+ */
+ createDefaultTraces(): GoldenTrace[] {
+ const traces: GoldenTrace[] = [
+ // Trace 1: Destructive command must be blocked
+ {
+ traceId: 'default-destructive-blocked',
+ name: 'Destructive command blocked',
+ description: 'Verifies that destructive commands (rm -rf, DROP TABLE) are denied',
+ events: [
+ {
+ seq: 1,
+ eventType: 'command',
+ payload: { command: 'rm -rf /', tool: 'Bash' },
+ expectedOutcome: 'deny',
+ },
+ {
+ seq: 2,
+ eventType: 'command',
+ payload: { command: 'DROP TABLE users', tool: 'Bash' },
+ expectedOutcome: 'deny',
+ },
+ {
+ seq: 3,
+ eventType: 'command',
+ payload: { command: 'git push --force origin main', tool: 'Bash' },
+ expectedOutcome: 'deny',
+ },
+ ],
+ expectedDecisions: { '1': 'deny', '2': 'deny', '3': 'deny' },
+ expectedMemoryLineage: {},
+ },
+
+ // Trace 2: Secret detected and blocked
+ {
+ traceId: 'default-secret-blocked',
+ name: 'Secret detected and blocked',
+ description: 'Verifies that secrets in tool parameters are detected and blocked',
+ events: [
+ {
+ seq: 1,
+ eventType: 'tool-use',
+ payload: {
+ tool: 'Write',
+ params: { content: 'api_key = "sk-abc123456789012345678901234567890"' },
+ },
+ expectedOutcome: 'deny',
+ },
+ {
+ seq: 2,
+ eventType: 'tool-use',
+ payload: {
+ tool: 'Edit',
+ params: { content: '-----BEGIN RSA PRIVATE KEY-----' },
+ },
+ expectedOutcome: 'deny',
+ },
+ ],
+ expectedDecisions: { '1': 'deny', '2': 'deny' },
+ expectedMemoryLineage: {},
+ },
+
+ // Trace 3: Budget exceeded and denied
+ {
+ traceId: 'default-budget-exceeded',
+ name: 'Budget exceeded and denied',
+ description: 'Verifies that operations exceeding budget limits are denied',
+ events: [
+ {
+ seq: 1,
+ eventType: 'budget-check',
+ payload: {
+ resource: 'toolCalls',
+ current: 999,
+ limit: 1000,
+ requested: 5,
+ },
+ expectedOutcome: 'deny',
+ },
+ {
+ seq: 2,
+ eventType: 'budget-check',
+ payload: {
+ resource: 'wallClockSeconds',
+ current: 3500,
+ limit: 3600,
+ requested: 200,
+ },
+ expectedOutcome: 'deny',
+ },
+ ],
+ expectedDecisions: { '1': 'deny', '2': 'deny' },
+ expectedMemoryLineage: {},
+ },
+
+ // Trace 4: Memory write without evidence blocked
+ {
+ traceId: 'default-memory-no-evidence',
+ name: 'Memory write without evidence blocked',
+ description: 'Verifies that memory writes without proof/evidence trail are denied',
+ events: [
+ {
+ seq: 1,
+ eventType: 'memory-write',
+ payload: {
+ key: 'critical-decision',
+ namespace: 'coordination',
+ hasEvidence: false,
+ coherenceScore: 0.3,
+ },
+ expectedOutcome: 'deny',
+ },
+ {
+ seq: 2,
+ eventType: 'memory-write',
+ payload: {
+ key: 'hallucinated-data',
+ namespace: 'facts',
+ hasEvidence: false,
+ antiHallucinationPassed: false,
+ },
+ expectedOutcome: 'deny',
+ },
+ ],
+ expectedDecisions: { '1': 'deny', '2': 'deny' },
+ expectedMemoryLineage: {
+ 'critical-decision': ['initial-assessment', 'root-task'],
+ },
+ },
+
+ // Trace 5: Valid operation allowed
+ {
+ traceId: 'default-valid-allowed',
+ name: 'Valid operation allowed',
+ description: 'Verifies that well-formed, safe operations are allowed through',
+ events: [
+ {
+ seq: 1,
+ eventType: 'command',
+ payload: { command: 'git status', tool: 'Bash' },
+ expectedOutcome: 'allow',
+ },
+ {
+ seq: 2,
+ eventType: 'tool-use',
+ payload: {
+ tool: 'Read',
+ params: { file_path: '/home/user/project/src/index.ts' },
+ },
+ expectedOutcome: 'allow',
+ },
+ {
+ seq: 3,
+ eventType: 'memory-write',
+ payload: {
+ key: 'agent-status',
+ namespace: 'coordination',
+ hasEvidence: true,
+ coherenceScore: 0.95,
+ antiHallucinationPassed: true,
+ },
+ expectedOutcome: 'allow',
+ },
+ ],
+ expectedDecisions: { '1': 'allow', '2': 'allow', '3': 'allow' },
+ expectedMemoryLineage: {},
+ },
+ ];
+
+ return traces;
+ }
+}
+
+// ============================================================================
+// Factory Functions
+// ============================================================================
+
+/**
+ * Create a new ManifestValidator instance.
+ */
+export function createManifestValidator(
+ options?: { admitThreshold?: number; rejectThreshold?: number },
+): ManifestValidator {
+ return new ManifestValidator(options);
+}
+
+/**
+ * Create a new ConformanceSuite instance, optionally pre-loaded with default traces.
+ */
+export function createConformanceSuite(
+ options?: { includeDefaults?: boolean },
+): ConformanceSuite {
+ const suite = new ConformanceSuite();
+ if (options?.includeDefaults) {
+ for (const trace of suite.createDefaultTraces()) {
+ suite.addTrace(trace);
+ }
+ }
+ return suite;
+}
diff --git a/v3/@claude-flow/guidance/tests/evolution.test.ts b/v3/@claude-flow/guidance/tests/evolution.test.ts
new file mode 100644
index 0000000000..0d9b958ec4
--- /dev/null
+++ b/v3/@claude-flow/guidance/tests/evolution.test.ts
@@ -0,0 +1,756 @@
+/**
+ * Tests for the Evolution Pipeline
+ */
+import { describe, it, expect, beforeEach } from 'vitest';
+import {
+ EvolutionPipeline,
+ createEvolutionPipeline,
+} from '../src/evolution.js';
+import type {
+ ChangeProposal,
+ SimulationResult,
+ StagedRollout,
+ TraceEvaluator,
+ RolloutStage,
+} from '../src/evolution.js';
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+function makeProposalParams(overrides: Record<string, unknown> = {}) {
+ return {
+ kind: 'rule-modify' as const,
+ title: 'Improve rule R001',
+ description: 'Clarify wording for R001 to reduce violations',
+ author: 'optimizer-agent',
+ targetPath: 'rules.R001.text',
+ diff: { before: 'old text', after: 'new text' },
+ rationale: 'R001 is violated 12 times per cycle',
+ riskAssessment: { level: 'low' as const, factors: ['wording-only change'] },
+ ...overrides,
+ };
+}
+
+/**
+ * Evaluator that returns identical results for both configs (zero divergence).
+ */
+const identicalEvaluator: TraceEvaluator = (trace, config) => ({
+ traceHash: 'aaa',
+ metrics: { accuracy: 0.95, latency: 100 },
+ decisions: [{ action: 'allow' }, { action: 'allow' }],
+});
+
+/**
+ * Evaluator that returns completely different results per config (high divergence).
+ */
+const divergentEvaluator: TraceEvaluator = (trace, config) => {
+ if (config === 'baseline') {
+ return {
+ traceHash: 'baseline-hash',
+ metrics: { accuracy: 0.95, latency: 100 },
+ decisions: [
+ { action: 'allow' },
+ { action: 'allow' },
+ { action: 'warn' },
+ ],
+ };
+ }
+ return {
+ traceHash: 'candidate-hash',
+ metrics: { accuracy: 0.60, latency: 200 },
+ decisions: [
+ { action: 'block' },
+ { action: 'block' },
+ { action: 'block' },
+ { action: 'block' },
+ { action: 'block' },
+ ],
+ };
+};
+
+/**
+ * Evaluator with slightly different decisions but within threshold.
+ */
+const slightlyDivergentEvaluator: TraceEvaluator = (trace, config) => {
+ if (config === 'baseline') {
+ return {
+ traceHash: 'baseline-hash',
+ metrics: { accuracy: 0.90, latency: 100 },
+ decisions: [{ action: 'allow' }, { action: 'warn' }],
+ };
+ }
+ return {
+ traceHash: 'candidate-hash',
+ metrics: { accuracy: 0.92, latency: 95 },
+ decisions: [{ action: 'allow' }, { action: 'allow' }],
+ };
+};
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+describe('EvolutionPipeline', () => {
+ let pipeline: EvolutionPipeline;
+
+ beforeEach(() => {
+ pipeline = createEvolutionPipeline({
+ signingKey: 'test-evolution-key',
+ maxDivergence: 0.3,
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // propose
+ // --------------------------------------------------------------------------
+
+ describe('propose', () => {
+ it('should create a signed proposal with a UUID', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+
+ expect(proposal.proposalId).toBeTruthy();
+ expect(proposal.proposalId).toMatch(
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/,
+ );
+ });
+
+ it('should produce a valid HMAC-SHA256 signature', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+
+ expect(proposal.signature).toMatch(/^[a-f0-9]{64}$/);
+ });
+
+ it('should set status to signed after creation', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+
+ expect(proposal.status).toBe('signed');
+ });
+
+ it('should preserve all proposal fields', () => {
+ const params = makeProposalParams({
+ kind: 'rule-add',
+ title: 'Add rule R099',
+ author: 'human-1',
+ targetPath: 'rules.R099',
+ });
+ const proposal = pipeline.propose(params);
+
+ expect(proposal.kind).toBe('rule-add');
+ expect(proposal.title).toBe('Add rule R099');
+ expect(proposal.author).toBe('human-1');
+ expect(proposal.targetPath).toBe('rules.R099');
+ expect(proposal.diff).toEqual({ before: 'old text', after: 'new text' });
+ expect(proposal.riskAssessment.level).toBe('low');
+ expect(proposal.createdAt).toBeGreaterThan(0);
+ });
+
+ it('should produce different signatures for different proposals', () => {
+ const p1 = pipeline.propose(makeProposalParams({ title: 'Change A' }));
+ const p2 = pipeline.propose(makeProposalParams({ title: 'Change B' }));
+
+ expect(p1.signature).not.toBe(p2.signature);
+ });
+
+ it('should store the proposal for later retrieval', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+
+ expect(pipeline.getProposal(proposal.proposalId)).toBe(proposal);
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // simulate
+ // --------------------------------------------------------------------------
+
+ describe('simulate', () => {
+ it('should compute zero divergence for identical evaluators', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const goldenTraces = [{ input: 'test-1' }, { input: 'test-2' }];
+
+ const result = pipeline.simulate(proposal.proposalId, goldenTraces, identicalEvaluator);
+
+ expect(result.divergenceScore).toBe(0);
+ expect(result.passed).toBe(true);
+ expect(result.decisionDiffs).toHaveLength(0);
+ });
+
+ it('should compute non-zero divergence when decisions differ', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const goldenTraces = [{ input: 'test-1' }];
+
+ const result = pipeline.simulate(
+ proposal.proposalId,
+ goldenTraces,
+ slightlyDivergentEvaluator,
+ );
+
+ expect(result.divergenceScore).toBeGreaterThan(0);
+ expect(result.decisionDiffs.length).toBeGreaterThan(0);
+ });
+
+ it('should include baseline and candidate trace hashes', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const result = pipeline.simulate(
+ proposal.proposalId,
+ [{ input: 'trace' }],
+ divergentEvaluator,
+ );
+
+ expect(result.baselineTraceHash).toMatch(/^[a-f0-9]{64}$/);
+ expect(result.candidateTraceHash).toMatch(/^[a-f0-9]{64}$/);
+ expect(result.baselineTraceHash).not.toBe(result.candidateTraceHash);
+ });
+
+ it('should aggregate metrics from all traces', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const result = pipeline.simulate(
+ proposal.proposalId,
+ [{ input: '1' }, { input: '2' }],
+ identicalEvaluator,
+ );
+
+ expect(result.metricsComparison.baseline.accuracy).toBe(0.95);
+ expect(result.metricsComparison.candidate.accuracy).toBe(0.95);
+ });
+
+ it('should set proposal status to simulating during evaluation', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+
+ pipeline.simulate(proposal.proposalId, [{ input: '1' }], identicalEvaluator);
+
+ // After simulation completes, status may still be 'simulating' until compare
+ // is called, but the simulate method itself sets it.
+ const fetched = pipeline.getProposal(proposal.proposalId);
+ expect(fetched?.status).toBe('simulating');
+ });
+
+ it('should throw for unknown proposal', () => {
+ expect(() =>
+ pipeline.simulate('nonexistent-id', [], identicalEvaluator),
+ ).toThrow('Proposal not found');
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // compare
+ // --------------------------------------------------------------------------
+
+ describe('compare', () => {
+ it('should approve low-divergence changes', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const simResult = pipeline.simulate(
+ proposal.proposalId,
+ [{ input: '1' }],
+ identicalEvaluator,
+ );
+
+ const comparison = pipeline.compare(proposal.proposalId, simResult);
+
+ expect(comparison.approved).toBe(true);
+ expect(comparison.reason).toContain('within threshold');
+ });
+
+ it('should reject high-divergence changes', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const simResult = pipeline.simulate(
+ proposal.proposalId,
+ [{ input: '1' }, { input: '2' }, { input: '3' }],
+ divergentEvaluator,
+ );
+
+ // Force a high divergence score for the test
+ const highDivResult: SimulationResult = {
+ ...simResult,
+ divergenceScore: 0.8,
+ };
+
+ const comparison = pipeline.compare(proposal.proposalId, highDivResult);
+
+ expect(comparison.approved).toBe(false);
+ expect(comparison.reason).toContain('exceeds threshold');
+ });
+
+ it('should reject changes with significant metric regressions', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+
+ const simResult: SimulationResult = {
+ proposalId: proposal.proposalId,
+ baselineTraceHash: 'a'.repeat(64),
+ candidateTraceHash: 'b'.repeat(64),
+ divergenceScore: 0.1, // low divergence, should pass that check
+ decisionDiffs: [],
+ metricsComparison: {
+ baseline: { accuracy: 1.0, latency: 100 },
+ candidate: { accuracy: 0.5, latency: 100 }, // 50% regression in accuracy
+ },
+ passed: true,
+ reason: 'test',
+ };
+
+ const comparison = pipeline.compare(proposal.proposalId, simResult);
+
+ expect(comparison.approved).toBe(false);
+ expect(comparison.reason).toContain('regressed');
+ });
+
+ it('should set proposal status to compared on approval', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const simResult = pipeline.simulate(
+ proposal.proposalId,
+ [{ input: '1' }],
+ identicalEvaluator,
+ );
+
+ pipeline.compare(proposal.proposalId, simResult);
+
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('compared');
+ });
+
+ it('should set proposal status to rejected on rejection', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+
+ const simResult: SimulationResult = {
+ proposalId: proposal.proposalId,
+ baselineTraceHash: 'a'.repeat(64),
+ candidateTraceHash: 'b'.repeat(64),
+ divergenceScore: 0.9,
+ decisionDiffs: [],
+ metricsComparison: { baseline: {}, candidate: {} },
+ passed: false,
+ reason: 'too divergent',
+ };
+
+ pipeline.compare(proposal.proposalId, simResult);
+
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('rejected');
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // stage
+ // --------------------------------------------------------------------------
+
+ describe('stage', () => {
+ it('should create a staged rollout plan with default stages', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ expect(rollout.rolloutId).toBeTruthy();
+ expect(rollout.proposalId).toBe(proposal.proposalId);
+ expect(rollout.stages).toHaveLength(3);
+ expect(rollout.stages[0].name).toBe('canary');
+ expect(rollout.stages[1].name).toBe('partial');
+ expect(rollout.stages[2].name).toBe('full');
+ expect(rollout.currentStage).toBe(0);
+ expect(rollout.status).toBe('in-progress');
+ expect(rollout.startedAt).toBeGreaterThan(0);
+ expect(rollout.completedAt).toBeNull();
+ });
+
+ it('should start the first stage immediately', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ expect(rollout.stages[0].startedAt).toBeGreaterThan(0);
+ expect(rollout.stages[1].startedAt).toBeNull();
+ });
+
+ it('should set proposal status to staged', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ pipeline.stage(proposal.proposalId);
+
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('staged');
+ });
+
+ it('should use custom stages when provided', () => {
+ const customStages: RolloutStage[] = [
+ {
+ name: 'alpha',
+ percentage: 1,
+ durationMs: 10_000,
+ metrics: {},
+ divergenceThreshold: 0.1,
+ passed: null,
+ startedAt: null,
+ completedAt: null,
+ },
+ {
+ name: 'beta',
+ percentage: 100,
+ durationMs: 30_000,
+ metrics: {},
+ divergenceThreshold: 0.2,
+ passed: null,
+ startedAt: null,
+ completedAt: null,
+ },
+ ];
+
+ const customPipeline = createEvolutionPipeline({
+ signingKey: 'test-key',
+ stages: customStages,
+ });
+ const proposal = customPipeline.propose(makeProposalParams());
+ const rollout = customPipeline.stage(proposal.proposalId);
+
+ expect(rollout.stages).toHaveLength(2);
+ expect(rollout.stages[0].name).toBe('alpha');
+ expect(rollout.stages[1].name).toBe('beta');
+ });
+
+ it('should store rollout for later retrieval', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ expect(pipeline.getRollout(rollout.rolloutId)).toBe(rollout);
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // advanceStage
+ // --------------------------------------------------------------------------
+
+ describe('advanceStage', () => {
+ it('should progress to the next stage when metrics pass', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ const result = pipeline.advanceStage(rollout.rolloutId, {
+ divergence: 0.05,
+ accuracy: 0.95,
+ });
+
+ expect(result.advanced).toBe(true);
+ expect(result.rolledBack).toBe(false);
+ expect(result.reason).toContain('partial');
+
+ const updated = pipeline.getRollout(rollout.rolloutId)!;
+ expect(updated.currentStage).toBe(1);
+ expect(updated.stages[0].passed).toBe(true);
+ expect(updated.stages[0].completedAt).toBeGreaterThan(0);
+ expect(updated.stages[1].startedAt).toBeGreaterThan(0);
+ });
+
+ it('should auto-rollback when divergence exceeds threshold', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ const result = pipeline.advanceStage(rollout.rolloutId, {
+ divergence: 0.99, // way over the 0.2 canary threshold
+ });
+
+ expect(result.advanced).toBe(false);
+ expect(result.rolledBack).toBe(true);
+ expect(result.reason).toContain('Auto-rollback');
+
+ const updated = pipeline.getRollout(rollout.rolloutId)!;
+ expect(updated.status).toBe('rolled-back');
+ expect(updated.stages[0].passed).toBe(false);
+ });
+
+ it('should promote after all stages pass', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ // Advance through all 3 stages
+ pipeline.advanceStage(rollout.rolloutId, { divergence: 0.05 });
+ pipeline.advanceStage(rollout.rolloutId, { divergence: 0.10 });
+ const final = pipeline.advanceStage(rollout.rolloutId, { divergence: 0.15 });
+
+ expect(final.advanced).toBe(true);
+ expect(final.reason).toContain('promoted');
+
+ const updated = pipeline.getRollout(rollout.rolloutId)!;
+ expect(updated.status).toBe('completed');
+ expect(updated.completedAt).toBeGreaterThan(0);
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('promoted');
+ });
+
+ it('should record metrics on the current stage', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ pipeline.advanceStage(rollout.rolloutId, {
+ divergence: 0.01,
+ accuracy: 0.98,
+ latency: 55,
+ });
+
+ const updated = pipeline.getRollout(rollout.rolloutId)!;
+ expect(updated.stages[0].metrics).toEqual({
+ divergence: 0.01,
+ accuracy: 0.98,
+ latency: 55,
+ });
+ });
+
+ it('should reject advance on non-in-progress rollout', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ // Rollback first
+ pipeline.rollback(rollout.rolloutId, 'manual rollback');
+
+ const result = pipeline.advanceStage(rollout.rolloutId, { divergence: 0 });
+
+ expect(result.advanced).toBe(false);
+ expect(result.rolledBack).toBe(false);
+ expect(result.reason).toContain('rolled-back');
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // rollback
+ // --------------------------------------------------------------------------
+
+ describe('rollback', () => {
+ it('should mark rollout as rolled-back', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ pipeline.rollback(rollout.rolloutId, 'detected regression in production');
+
+ const updated = pipeline.getRollout(rollout.rolloutId)!;
+ expect(updated.status).toBe('rolled-back');
+ expect(updated.completedAt).toBeGreaterThan(0);
+ });
+
+ it('should mark the proposal as rolled-back', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ pipeline.rollback(rollout.rolloutId, 'manual');
+
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('rolled-back');
+ });
+
+ it('should throw for unknown rollout', () => {
+ expect(() => pipeline.rollback('nonexistent', 'reason')).toThrow(
+ 'Rollout not found',
+ );
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // promote
+ // --------------------------------------------------------------------------
+
+ describe('promote', () => {
+ it('should mark rollout as completed', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ pipeline.promote(rollout.rolloutId);
+
+ const updated = pipeline.getRollout(rollout.rolloutId)!;
+ expect(updated.status).toBe('completed');
+ expect(updated.completedAt).toBeGreaterThan(0);
+ });
+
+ it('should mark proposal as promoted', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const rollout = pipeline.stage(proposal.proposalId);
+
+ pipeline.promote(rollout.rolloutId);
+
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('promoted');
+ });
+
+ it('should throw for unknown rollout', () => {
+ expect(() => pipeline.promote('nonexistent')).toThrow('Rollout not found');
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // getProposals
+ // --------------------------------------------------------------------------
+
+ describe('getProposals', () => {
+ it('should return all proposals when no filter is given', () => {
+ pipeline.propose(makeProposalParams({ title: 'A' }));
+ pipeline.propose(makeProposalParams({ title: 'B' }));
+ pipeline.propose(makeProposalParams({ title: 'C' }));
+
+ expect(pipeline.getProposals()).toHaveLength(3);
+ });
+
+ it('should filter proposals by status', () => {
+ const p1 = pipeline.propose(makeProposalParams({ title: 'A' }));
+ const p2 = pipeline.propose(makeProposalParams({ title: 'B' }));
+
+ // Stage p1 (changes status to 'staged')
+ pipeline.stage(p1.proposalId);
+
+ const signed = pipeline.getProposals('signed');
+ const staged = pipeline.getProposals('staged');
+
+ expect(signed).toHaveLength(1);
+ expect(signed[0].title).toBe('B');
+ expect(staged).toHaveLength(1);
+ expect(staged[0].title).toBe('A');
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // getHistory
+ // --------------------------------------------------------------------------
+
+ describe('getHistory', () => {
+ it('should return empty history for fresh pipeline', () => {
+ expect(pipeline.getHistory()).toHaveLength(0);
+ });
+
+ it('should return full timeline for a proposal through all stages', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+ const simResult = pipeline.simulate(
+ proposal.proposalId,
+ [{ input: '1' }],
+ identicalEvaluator,
+ );
+ pipeline.compare(proposal.proposalId, simResult);
+ const rollout = pipeline.stage(proposal.proposalId);
+ pipeline.advanceStage(rollout.rolloutId, { divergence: 0.01 });
+ pipeline.advanceStage(rollout.rolloutId, { divergence: 0.01 });
+ pipeline.advanceStage(rollout.rolloutId, { divergence: 0.01 });
+
+ const history = pipeline.getHistory();
+
+ expect(history).toHaveLength(1);
+ expect(history[0].proposal.proposalId).toBe(proposal.proposalId);
+ expect(history[0].simulation).toBeDefined();
+ expect(history[0].simulation?.proposalId).toBe(proposal.proposalId);
+ expect(history[0].rollout).toBeDefined();
+ expect(history[0].rollout?.rolloutId).toBe(rollout.rolloutId);
+ expect(history[0].outcome).toBe('promoted');
+ });
+
+ it('should include proposals without simulations or rollouts', () => {
+ pipeline.propose(makeProposalParams());
+
+ const history = pipeline.getHistory();
+
+ expect(history).toHaveLength(1);
+ expect(history[0].simulation).toBeUndefined();
+ expect(history[0].rollout).toBeUndefined();
+ expect(history[0].outcome).toBe('signed');
+ });
+
+ it('should track multiple proposals independently', () => {
+ const p1 = pipeline.propose(makeProposalParams({ title: 'Change 1' }));
+ const p2 = pipeline.propose(makeProposalParams({ title: 'Change 2' }));
+
+ // Only simulate p1
+ pipeline.simulate(p1.proposalId, [{ input: '1' }], identicalEvaluator);
+
+ const history = pipeline.getHistory();
+
+ expect(history).toHaveLength(2);
+
+ const h1 = history.find(h => h.proposal.proposalId === p1.proposalId)!;
+ const h2 = history.find(h => h.proposal.proposalId === p2.proposalId)!;
+
+ expect(h1.simulation).toBeDefined();
+ expect(h2.simulation).toBeUndefined();
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // Factory function
+ // --------------------------------------------------------------------------
+
+ describe('createEvolutionPipeline', () => {
+ it('should create a pipeline with default config', () => {
+ const defaultPipeline = createEvolutionPipeline();
+ const proposal = defaultPipeline.propose(makeProposalParams());
+
+ expect(proposal.signature).toMatch(/^[a-f0-9]{64}$/);
+ expect(proposal.status).toBe('signed');
+ });
+
+ it('should create a pipeline with custom config', () => {
+ const customPipeline = createEvolutionPipeline({
+ signingKey: 'custom-key',
+ maxDivergence: 0.5,
+ });
+
+ const proposal = customPipeline.propose(makeProposalParams());
+ expect(proposal.signature).toBeTruthy();
+
+ // Higher threshold should accept more divergence
+ const simResult = customPipeline.simulate(
+ proposal.proposalId,
+ [{ input: '1' }],
+ slightlyDivergentEvaluator,
+ );
+
+ const comparison = customPipeline.compare(proposal.proposalId, simResult);
+ expect(comparison.approved).toBe(true);
+ });
+ });
+
+ // --------------------------------------------------------------------------
+ // End-to-end lifecycle
+ // --------------------------------------------------------------------------
+
+ describe('end-to-end lifecycle', () => {
+ it('should handle propose -> simulate -> compare -> stage -> advance -> promote', () => {
+ // 1. Propose
+ const proposal = pipeline.propose(makeProposalParams());
+ expect(proposal.status).toBe('signed');
+
+ // 2. Simulate
+ const simResult = pipeline.simulate(
+ proposal.proposalId,
+ [{ input: 'golden-1' }, { input: 'golden-2' }],
+ identicalEvaluator,
+ );
+ expect(simResult.passed).toBe(true);
+
+ // 3. Compare
+ const comparison = pipeline.compare(proposal.proposalId, simResult);
+ expect(comparison.approved).toBe(true);
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('compared');
+
+ // 4. Stage
+ const rollout = pipeline.stage(proposal.proposalId);
+ expect(rollout.status).toBe('in-progress');
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('staged');
+
+ // 5. Advance through all stages
+ const r1 = pipeline.advanceStage(rollout.rolloutId, { divergence: 0.01 });
+ expect(r1.advanced).toBe(true);
+
+ const r2 = pipeline.advanceStage(rollout.rolloutId, { divergence: 0.02 });
+ expect(r2.advanced).toBe(true);
+
+ const r3 = pipeline.advanceStage(rollout.rolloutId, { divergence: 0.03 });
+ expect(r3.advanced).toBe(true);
+ expect(r3.reason).toContain('promoted');
+
+ // 6. Verify final state
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('promoted');
+ expect(pipeline.getRollout(rollout.rolloutId)?.status).toBe('completed');
+ });
+
+ it('should handle propose -> simulate -> compare(reject) flow', () => {
+ const proposal = pipeline.propose(makeProposalParams());
+
+ const simResult: SimulationResult = {
+ proposalId: proposal.proposalId,
+ baselineTraceHash: 'a'.repeat(64),
+ candidateTraceHash: 'b'.repeat(64),
+ divergenceScore: 0.95,
+ decisionDiffs: [],
+ metricsComparison: { baseline: {}, candidate: {} },
+ passed: false,
+ reason: 'too divergent',
+ };
+
+ const comparison = pipeline.compare(proposal.proposalId, simResult);
+ expect(comparison.approved).toBe(false);
+ expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('rejected');
+ });
+ });
+});