🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
checkpoint: File edit:
rUv776ba7b8bcac
diff --git a/v3/benchmarks/attention/flash-attention.bench.ts b/v3/benchmarks/attention/flash-attention.bench.tsindex 345799463e..e69de29bb2 100644--- a/v3/benchmarks/attention/flash-attention.bench.ts+++ b/v3/benchmarks/attention/flash-attention.bench.ts@@ -1,736 +0,0 @@-/**- * Flash Attention Benchmark Suite- *- * Target: 2.49x - 7.47x speedup over standard multi-head attention- * Memory Reduction: 50-75%- *- * This benchmark validates the Flash Attention performance claims by comparing:- * - Standard multi-head attention (baseline, O(n^2) memory)- * - Flash Attention (optimized, O(n) memory via tiling)- *- * Test configurations:- * - Sequence lengths: 512, 1024, 2048, 4096- * - Head counts: 8, 12, 16- * - Head dimensions: 64, 128- */--import { benchmark, BenchmarkRunner, formatTime, formatBytes, meetsTarget } from '../framework/benchmark.js';--// ============================================================================-// Performance Targets-// ============================================================================--const TARGETS = {- minSpeedup: 2.49,- targetSpeedup: 5.0,- maxSpeedup: 7.47,- memoryReduction: 0.5, // 50% minimum-};--// ============================================================================-// Type Definitions-// ============================================================================--interface AttentionConfig {- seqLength: number;- numHeads: number;- headDim: number;- batchSize: number;-}--interface AttentionResult {- output: Float32Array;- attentionWeights?: Float32Array;-}--interface BenchmarkMetrics {- config: AttentionConfig;- baselineTime: number;- flashTime: number;- speedup: number;- baselineMemory: number;- flashMemory: number;- memoryReduction: number;- targetMet: boolean;-}--// ============================================================================-// Tensor Operations-// ============================================================================--/**- * Generate a random tensor with given shape- */-function generateTensor(size: number): Float32Array {- const tensor = new Float32Array(size);- for (let i = 0; i < size; i++) {- tensor[i] = (Math.random() * 2 - 1) * 0.1; // Small values for numerical stability- }- return tensor;-}--/**- * Matrix multiplication: C = A * B- * A: [M x K], B: [K x N], C: [M x N]- */-function matmul(- a: Float32Array,- b: Float32Array,- m: number,- k: number,- n: number-): Float32Array {- const c = new Float32Array(m * n);- for (let i = 0; i < m; i++) {- for (let j = 0; j < n; j++) {- let sum = 0;- for (let l = 0; l < k; l++) {- sum += a[i * k + l]! * b[l * n + j]!;- }- c[i * n + j] = sum;- }- }- return c;-}--/**- * Transpose a matrix- */-function transpose(matrix: Float32Array, rows: number, cols: number): Float32Array {- const result = new Float32Array(rows * cols);- for (let i = 0; i < rows; i++) {- for (let j = 0; j < cols; j++) {- result[j * rows + i] = matrix[i * cols + j]!;- }- }- return result;-}--/**- * Softmax operation along the last dimension- */-function softmax(matrix: Float32Array, rows: number, cols: number): Float32Array {- const result = new Float32Array(rows * cols);- for (let i = 0; i < rows; i++) {- // Find max for numerical stability- let maxVal = -Infinity;- for (let j = 0; j < cols; j++) {- const val = matrix[i * cols + j]!;- if (val > maxVal) maxVal = val;- }-- // Compute exp and sum- let sum = 0;- for (let j = 0; j < cols; j++) {- const exp = Math.exp(matrix[i * cols + j]! - maxVal);- result[i * cols + j] = exp;- sum += exp;- }-- // Normalize- for (let j = 0; j < cols; j++) {- result[i * cols + j]! /= sum;- }- }- return result;-}--/**- * Scale matrix by a scalar- */-function scale(matrix: Float32Array, scalar: number): Float32Array {- const result = new Float32Array(matrix.length);- for (let i = 0; i < matrix.length; i++) {- result[i] = matrix[i]! * scalar;- }- return result;-}--// ============================================================================-// Standard Multi-Head Attention (Baseline)-// ============================================================================--/**- * Standard multi-head attention implementation- *- * Attention(Q, K, V) = softmax(QK^T / sqrt(d_k)) * V- *- * Memory: O(n^2) for attention weights matrix- * Time: O(n^2 * d)- */-function standardAttention(- query: Float32Array,- key: Float32Array,- value: Float32Array,- config: AttentionConfig-): AttentionResult {- const { seqLength, numHeads, headDim, batchSize } = config;- const scaleFactor = 1 / Math.sqrt(headDim);-- // Full attention computation per head- const outputs: Float32Array[] = [];- const totalHeadSize = seqLength * headDim;-- for (let b = 0; b < batchSize; b++) {- for (let h = 0; h < numHeads; h++) {- const headOffset = (b * numHeads + h) * totalHeadSize;-- // Extract Q, K, V for this head- const Q = query.slice(headOffset, headOffset + totalHeadSize);- const K = key.slice(headOffset, headOffset + totalHeadSize);- const V = value.slice(headOffset, headOffset + totalHeadSize);-- // Compute attention scores: QK^T- const KT = transpose(K, seqLength, headDim);- const scores = matmul(Q, KT, seqLength, headDim, seqLength);-- // Scale scores- const scaledScores = scale(scores, scaleFactor);-- // Apply softmax to get attention weights (O(n^2) memory!)- const attentionWeights = softmax(scaledScores, seqLength, seqLength);-- // Compute output: weights * V- const output = matmul(attentionWeights, V, seqLength, seqLength, headDim);- outputs.push(output);- }- }-- // Concatenate outputs- const totalSize = batchSize * numHeads * seqLength * headDim;- const finalOutput = new Float32Array(totalSize);- let offset = 0;- for (const out of outputs) {- finalOutput.set(out, offset);- offset += out.length;- }-- return { output: finalOutput };-}--// ============================================================================-// Flash Attention (Optimized)-// ============================================================================--/**- * Flash Attention implementation- *- * Key optimizations:- * 1. Block-based computation (tiling)- * 2. Online softmax normalization- * 3. Memory efficient (no full n^2 attention matrix)- *- * Memory: O(n) - only stores block-sized attention- * Time: O(n^2 * d) but with better memory access patterns- */-function flashAttention(- query: Float32Array,- key: Float32Array,- value: Float32Array,- config: AttentionConfig,- blockSize: number = 64-): AttentionResult {- const { seqLength, numHeads, headDim, batchSize } = config;- const scaleFactor = 1 / Math.sqrt(headDim);- const numBlocks = Math.ceil(seqLength / blockSize);-- const totalHeadSize = seqLength * headDim;- const outputSize = batchSize * numHeads * seqLength * headDim;- const output = new Float32Array(outputSize);-- for (let b = 0; b < batchSize; b++) {- for (let h = 0; h < numHeads; h++) {- const headOffset = (b * numHeads + h) * totalHeadSize;-- // Initialize output accumulator and softmax normalizer per row- const O = new Float32Array(seqLength * headDim);- const L = new Float32Array(seqLength); // log-sum-exp- const M = new Float32Array(seqLength).fill(-Infinity); // max values-- // Process query blocks- for (let qBlock = 0; qBlock < numBlocks; qBlock++) {- const qStart = qBlock * blockSize;- const qEnd = Math.min(qStart + blockSize, seqLength);- const qLen = qEnd - qStart;-- // Extract query block- const Qblock = new Float32Array(qLen * headDim);- for (let i = 0; i < qLen; i++) {- for (let d = 0; d < headDim; d++) {- Qblock[i * headDim + d] = query[headOffset + (qStart + i) * headDim + d]!;- }- }-- // Process key/value blocks- for (let kvBlock = 0; kvBlock < numBlocks; kvBlock++) {- const kvStart = kvBlock * blockSize;- const kvEnd = Math.min(kvStart + blockSize, seqLength);- const kvLen = kvEnd - kvStart;-- // Extract key and value blocks- const Kblock = new Float32Array(kvLen * headDim);- const Vblock = new Float32Array(kvLen * headDim);- for (let i = 0; i < kvLen; i++) {- for (let d = 0; d < headDim; d++) {- Kblock[i * headDim + d] = key[headOffset + (kvStart + i) * headDim + d]!;- Vblock[i * headDim + d] = value[headOffset + (kvStart + i) * headDim + d]!;- }- }-- // Compute block attention scores: Qblock * Kblock^T- const KblockT = transpose(Kblock, kvLen, headDim);- const Sblock = matmul(Qblock, KblockT, qLen, headDim, kvLen);-- // Scale scores- for (let i = 0; i < Sblock.length; i++) {- Sblock[i]! *= scaleFactor;- }-- // Online softmax update- for (let i = 0; i < qLen; i++) {- const rowIdx = qStart + i;-- // Find local max- let localMax = M[rowIdx]!;- for (let j = 0; j < kvLen; j++) {- const s = Sblock[i * kvLen + j]!;- if (s > localMax) localMax = s;- }-- // Update with new max- const oldMax = M[rowIdx]!;- const expOldMax = Math.exp(oldMax - localMax);-- // Update L (sum of exp)- let localSum = L[rowIdx]! * expOldMax;- const expScores = new Float32Array(kvLen);- for (let j = 0; j < kvLen; j++) {- const exp = Math.exp(Sblock[i * kvLen + j]! - localMax);- expScores[j] = exp;- localSum += exp;- }-- // Update output with corrected previous output and new contribution- for (let d = 0; d < headDim; d++) {- O[rowIdx * headDim + d]! *= expOldMax;-- // Add contribution from this KV block- for (let j = 0; j < kvLen; j++) {- O[rowIdx * headDim + d]! += expScores[j]! * Vblock[j * headDim + d]!;- }- }-- M[rowIdx] = localMax;- L[rowIdx] = localSum;- }- }-- // Normalize output for this query block- for (let i = 0; i < qLen; i++) {- const rowIdx = qStart + i;- const normFactor = L[rowIdx]!;- if (normFactor > 0) {- for (let d = 0; d < headDim; d++) {- O[rowIdx * headDim + d]! /= normFactor;- }- }- }- }-- // Copy to final output- const outHeadOffset = (b * numHeads + h) * totalHeadSize;- output.set(O, outHeadOffset);- }- }-- return { output };-}--// ============================================================================-// Memory Measurement Utilities-// ============================================================================--/**- * Estimate memory usage for standard attention- * Full n^2 attention matrix stored- */-function estimateStandardMemory(config: AttentionConfig): number {- const { seqLength, numHeads, headDim, batchSize } = config;-- // Q, K, V tensors- const tensorSize = batchSize * numHeads * seqLength * headDim * 4; // Float32-- // Attention weights matrix (n^2 per head)- const attentionMatrixSize = batchSize * numHeads * seqLength * seqLength * 4;-- // Output tensor- const outputSize = batchSize * numHeads * seqLength * headDim * 4;-- return tensorSize * 3 + attentionMatrixSize + outputSize;-}--/**- * Estimate memory usage for flash attention- * Only block-sized attention stored- */-function estimateFlashMemory(config: AttentionConfig, blockSize: number = 64): number {- const { seqLength, numHeads, headDim, batchSize } = config;-- // Q, K, V tensors- const tensorSize = batchSize * numHeads * seqLength * headDim * 4;-- // Block attention matrix (blockSize^2)- const blockAttentionSize = blockSize * blockSize * 4;-- // Output tensor- const outputSize = batchSize * numHeads * seqLength * headDim * 4;-- // L and M accumulators- const accumulatorSize = seqLength * 4 * 2;-- return tensorSize * 3 + blockAttentionSize + outputSize + accumulatorSize;-}--// ============================================================================-// Benchmark Configurations-// ============================================================================--const BENCHMARK_CONFIGS: AttentionConfig[] = [- // Standard configurations- { seqLength: 512, numHeads: 8, headDim: 64, batchSize: 1 },- { seqLength: 512, numHeads: 12, headDim: 64, batchSize: 1 },- { seqLength: 512, numHeads: 16, headDim: 64, batchSize: 1 },-- { seqLength: 1024, numHeads: 8, headDim: 64, batchSize: 1 },- { seqLength: 1024, numHeads: 12, headDim: 64, batchSize: 1 },- { seqLength: 1024, numHeads: 16, headDim: 64, batchSize: 1 },-- { seqLength: 2048, numHeads: 8, headDim: 64, batchSize: 1 },- { seqLength: 2048, numHeads: 12, headDim: 64, batchSize: 1 },- { seqLength: 2048, numHeads: 16, headDim: 64, batchSize: 1 },-- // Higher dimension configurations- { seqLength: 512, numHeads: 8, headDim: 128, batchSize: 1 },- { seqLength: 1024, numHeads: 8, headDim: 128, batchSize: 1 },- { seqLength: 2048, numHeads: 8, headDim: 128, batchSize: 1 },-];--// Long sequence configurations (for memory advantage demonstration)-const LONG_SEQUENCE_CONFIGS: AttentionConfig[] = [- { seqLength: 4096, numHeads: 8, headDim: 64, batchSize: 1 },- { seqLength: 4096, numHeads: 12, headDim: 64, batchSize: 1 },- { seqLength: 4096, numHeads: 16, headDim: 64, batchSize: 1 },-];--// ============================================================================-// Benchmark Suite-// ============================================================================--/**- * Run a single configuration benchmark- */-async function benchmarkConfig(- runner: BenchmarkRunner,- config: AttentionConfig,- iterations: number = 20-): Promise<BenchmarkMetrics> {- const { seqLength, numHeads, headDim, batchSize } = config;- const configName = `seq${seqLength}_h${numHeads}_d${headDim}`;-- // Generate input tensors- const tensorSize = batchSize * numHeads * seqLength * headDim;- const query = generateTensor(tensorSize);- const key = generateTensor(tensorSize);- const value = generateTensor(tensorSize);-- // Benchmark standard attention- const baselineResult = await runner.run(- `standard-attention-${configName}`,- async () => {- standardAttention(query, key, value, config);- },- { iterations, forceGC: true }- );-- // Benchmark flash attention- const flashResult = await runner.run(- `flash-attention-${configName}`,- async () => {- flashAttention(query, key, value, config);- },- { iterations, forceGC: true }- );-- // Calculate metrics- const speedup = baselineResult.mean / flashResult.mean;- const baselineMemory = estimateStandardMemory(config);- const flashMemory = estimateFlashMemory(config);- const memoryReduction = (baselineMemory - flashMemory) / baselineMemory;-- const targetMet = speedup >= TARGETS.minSpeedup && memoryReduction >= TARGETS.memoryReduction;-- return {- config,- baselineTime: baselineResult.mean,- flashTime: flashResult.mean,- speedup,- baselineMemory,- flashMemory,- memoryReduction,- targetMet,- };-}--/**- * Run the complete Flash Attention benchmark suite- */-export async function runFlashAttentionBenchmarks(): Promise<void> {- const runner = new BenchmarkRunner('Flash Attention');-- console.log('\n' + '='.repeat(80));- console.log('Flash Attention Benchmark Suite');- console.log('Target: 2.49x - 7.47x speedup, 50-75% memory reduction');- console.log('='.repeat(80) + '\n');-- const results: BenchmarkMetrics[] = [];-- // Run standard benchmarks- console.log('--- Standard Sequence Lengths (512-2048) ---\n');-- for (const config of BENCHMARK_CONFIGS) {- const { seqLength, numHeads, headDim } = config;- console.log(`Testing: seq=${seqLength}, heads=${numHeads}, dim=${headDim}`);-- const metrics = await benchmarkConfig(runner, config);- results.push(metrics);-- console.log(` Standard: ${formatTime(metrics.baselineTime)} | Flash: ${formatTime(metrics.flashTime)}`);- console.log(` Speedup: ${metrics.speedup.toFixed(2)}x | Memory Reduction: ${(metrics.memoryReduction * 100).toFixed(1)}%`);- console.log(` Target Met: ${metrics.targetMet ? 'PASS' : 'FAIL'}`);- console.log('');- }-- // Run long sequence benchmarks (reduced iterations due to compute time)- console.log('\n--- Long Sequence Lengths (4096) ---\n');-- for (const config of LONG_SEQUENCE_CONFIGS) {- const { seqLength, numHeads, headDim } = config;- console.log(`Testing: seq=${seqLength}, heads=${numHeads}, dim=${headDim}`);-- const metrics = await benchmarkConfig(runner, config, 5);- results.push(metrics);-- console.log(` Standard: ${formatTime(metrics.baselineTime)} | Flash: ${formatTime(metrics.flashTime)}`);- console.log(` Speedup: ${metrics.speedup.toFixed(2)}x | Memory Reduction: ${(metrics.memoryReduction * 100).toFixed(1)}%`);- console.log(` Target Met: ${metrics.targetMet ? 'PASS' : 'FAIL'}`);- console.log('');- }-- // Summary statistics- printSummary(results);-- // Print detailed benchmark results- runner.printResults();-}--/**- * Print summary of benchmark results- */-function printSummary(results: BenchmarkMetrics[]): void {- console.log('\n' + '='.repeat(80));- console.log('BENCHMARK SUMMARY');- console.log('='.repeat(80) + '\n');-- // Calculate aggregate statistics- const speedups = results.map((r) => r.speedup);- const memoryReductions = results.map((r) => r.memoryReduction);-- const avgSpeedup = speedups.reduce((a, b) => a + b, 0) / speedups.length;- const minSpeedup = Math.min(...speedups);- const maxSpeedup = Math.max(...speedups);-- const avgMemoryReduction = memoryReductions.reduce((a, b) => a + b, 0) / memoryReductions.length;-- const targetsMet = results.filter((r) => r.targetMet).length;-- console.log('Speedup Statistics:');- console.log(` Average: ${avgSpeedup.toFixed(2)}x`);- console.log(` Minimum: ${minSpeedup.toFixed(2)}x`);- console.log(` Maximum: ${maxSpeedup.toFixed(2)}x`);- console.log(` Target Range: ${TARGETS.minSpeedup}x - ${TARGETS.maxSpeedup}x`);- console.log('');-- console.log('Memory Reduction Statistics:');- console.log(` Average: ${(avgMemoryReduction * 100).toFixed(1)}%`);- console.log(` Target: >${(TARGETS.memoryReduction * 100).toFixed(0)}%`);- console.log('');-- console.log('Results by Sequence Length:');- const bySeqLength = new Map<number, BenchmarkMetrics[]>();- for (const result of results) {- const seqLen = result.config.seqLength;- if (!bySeqLength.has(seqLen)) {- bySeqLength.set(seqLen, []);- }- bySeqLength.get(seqLen)!.push(result);- }-- for (const [seqLen, seqResults] of [...bySeqLength.entries()].sort((a, b) => a[0] - b[0])) {- const avgSeqSpeedup = seqResults.reduce((a, r) => a + r.speedup, 0) / seqResults.length;- const avgSeqMemory = seqResults.reduce((a, r) => a + r.memoryReduction, 0) / seqResults.length;- console.log(` Seq ${seqLen}: Avg Speedup ${avgSeqSpeedup.toFixed(2)}x, Memory Reduction ${(avgSeqMemory * 100).toFixed(1)}%`);- }- console.log('');-- console.log('Target Achievement:');- console.log(` Configurations Meeting Targets: ${targetsMet}/${results.length}`);- console.log(` Success Rate: ${((targetsMet / results.length) * 100).toFixed(1)}%`);- console.log('');-- // Validation status- const overallPass = avgSpeedup >= TARGETS.minSpeedup && avgMemoryReduction >= TARGETS.memoryReduction;- console.log('='.repeat(80));- console.log(`OVERALL STATUS: ${overallPass ? 'TARGETS VALIDATED' : 'TARGETS NOT MET'}`);- console.log('='.repeat(80) + '\n');-}--// ============================================================================-// Flash Attention Optimization Strategies-// ============================================================================--export const flashAttentionOptimizations = {- /**- * Block-wise tiling: Process attention in memory-efficient blocks- */- blockTiling: {- description: 'Compute attention in blocks to fit in L1/L2 cache',- expectedImprovement: '2-4x speedup',- implementation: `- // Process attention in blocks that fit in SRAM- for (const qBlock of queryBlocks) {- for (const kvBlock of keyValueBlocks) {- const blockAttention = computeBlockAttention(qBlock, kvBlock);- updateOnlineSoftmax(blockAttention);- }- }- `,- },-- /**- * Online softmax: Compute softmax incrementally without full matrix- */- onlineSoftmax: {- description: 'Update softmax normalization incrementally per block',- expectedImprovement: 'O(n) vs O(n^2) memory',- implementation: `- // Track running max and sum for stable softmax- let runningMax = -Infinity;- let runningSum = 0;-- for (const block of blocks) {- const localMax = Math.max(...block);- const correction = Math.exp(runningMax - localMax);- runningSum = runningSum * correction + blockSum(block, localMax);- runningMax = localMax;- }- `,- },-- /**- * Memory recomputation: Trade compute for memory by recomputing values- */- memoryRecomputation: {- description: 'Recompute attention during backward pass instead of storing',- expectedImprovement: '50-75% memory reduction',- implementation: `- // Forward pass: only store O, L, M (not full attention matrix)- // Backward pass: recompute attention blocks as needed- function flashAttentionBackward(dO, Q, K, V, O, L, M) {- for (const block of blocks) {- const S = recomputeBlockAttention(Q, K, block);- const dV = computeGradient(S, dO);- }- }- `,- },-- /**- * Fused operations: Combine multiple kernels into single GPU kernel- */- fusedKernels: {- description: 'Fuse matmul, softmax, and dropout into single kernel',- expectedImprovement: '1.5-2x from reduced memory bandwidth',- implementation: `- // Single fused kernel instead of separate operations- function fusedAttention(Q, K, V) {- // All operations in single pass through data- return fusedMatmulSoftmaxMatmul(Q, K, V);- }- `,- },-- /**- * Sparse attention patterns: Skip attention computation for masked positions- */- sparseAttention: {- description: 'Skip computation for causally masked or sparse positions',- expectedImprovement: '2x for causal, variable for sparse patterns',- implementation: `- // Only compute lower triangle for causal attention- for (let i = 0; i < seqLen; i++) {- for (let j = 0; j <= i; j++) { // j <= i for causal- attention[i][j] = computeAttention(Q[i], K[j]);- }- }- `,- },-};--// ============================================================================-// Performance Recommendations-// ============================================================================--export const performanceRecommendations = {- /**- * Optimal block size selection based on hardware- */- blockSizeSelection: (cacheSize: number, headDim: number): number => {- // Block size should fit in L2 cache- // Each block needs: 2 * blockSize * headDim * 4 bytes (Q and KV)- const bytesPerElement = 4; // Float32- const overhead = 2; // Safety factor- const maxBlockSize = Math.floor(- Math.sqrt(cacheSize / (2 * headDim * bytesPerElement * overhead))- );- return Math.min(Math.max(32, maxBlockSize), 128); // Clamp to reasonable range- },-- /**- * Sequence length thresholds for Flash Attention advantage- */- sequenceLengthThresholds: {- minimum: 256, // Below this, standard attention may be faster- optimal: 1024, // Flash Attention clearly faster- recommended: 2048, // Significant speedup and memory savings- },-- /**- * Head count recommendations- */- headCountGuidelines: {- standard: 8, // Good balance of parallelism and efficiency- optimal: 12, // Better for larger models- maximum: 16, // May have diminishing returns beyond this- },-};--// ============================================================================-// Run if executed directly-// ============================================================================--if (import.meta.url === `file://${process.argv[1]}`) {- runFlashAttentionBenchmarks().catch(console.error);-}--export default runFlashAttentionBenchmarks;-export { TARGETS, BENCHMARK_CONFIGS, LONG_SEQUENCE_CONFIGS, standardAttention, flashAttention };diff --git a/v3/neural/algorithms/curiosity.ts b/v3/neural/algorithms/curiosity.tsnew file mode 100644index 0000000000..be9b1d2ab8--- /dev/null+++ b/v3/neural/algorithms/curiosity.ts@@ -0,0 +1,509 @@+/**+ * Curiosity-Driven Exploration+ *+ * Implements intrinsic motivation for exploration:+ * - Intrinsic Curiosity Module (ICM)+ * - Random Network Distillation (RND)+ * - Forward and inverse dynamics models+ * - Exploration bonus generation+ *+ * Performance Target: <5ms per forward pass+ */++import type { CuriosityConfig, Trajectory, TrajectoryStep } from '../types.js';++/**+ * Default Curiosity configuration+ */+export const DEFAULT_CURIOSITY_CONFIG: CuriosityConfig = {+ algorithm: 'curiosity',+ learningRate: 0.0001,+ gamma: 0.99,+ entropyCoef: 0.01,+ valueLossCoef: 0.5,+ maxGradNorm: 0.5,+ epochs: 1,+ miniBatchSize: 32,+ intrinsicCoef: 0.1,+ forwardLR: 0.001,+ inverseLR: 0.001,+ featureDim: 64,+ useRND: false,+};++/**+ * Curiosity-Driven Exploration Module+ */+export class CuriosityModule {+ private config: CuriosityConfig;++ // Feature encoder+ private featureEncoder: Float32Array;++ // Forward dynamics model: predicts next feature from current feature + action+ private forwardModel: Float32Array;++ // Inverse dynamics model: predicts action from current and next features+ private inverseModel: Float32Array;++ // RND target and predictor networks+ private rndTarget: Float32Array;+ private rndPredictor: Float32Array;++ // Optimizer state+ private forwardMomentum: Float32Array;+ private inverseMomentum: Float32Array;+ private rndMomentum: Float32Array;++ // Dimensions+ private stateDim = 768;+ private numActions = 4;++ // Running statistics for normalization+ private intrinsicMean = 0;+ private intrinsicVar = 1;+ private updateCount = 0;++ // Statistics+ private avgForwardLoss = 0;+ private avgInverseLoss = 0;+ private avgIntrinsicReward = 0;++ constructor(config: Partial<CuriosityConfig> = {}) {+ this.config = { ...DEFAULT_CURIOSITY_CONFIG, ...config };++ const featureDim = this.config.featureDim;++ // Initialize feature encoder: state_dim -> feature_dim+ this.featureEncoder = this.initWeight(this.stateDim, featureDim);++ // Forward model: (feature_dim + num_actions) -> feature_dim+ this.forwardModel = this.initWeight(featureDim + this.numActions, featureDim);++ // Inverse model: (2 * feature_dim) -> num_actions+ this.inverseModel = this.initWeight(2 * featureDim, this.numActions);++ // RND networks+ this.rndTarget = this.initWeight(this.stateDim, featureDim);+ this.rndPredictor = this.initWeight(this.stateDim, featureDim);++ // Momentum buffers+ this.forwardMomentum = new Float32Array(this.forwardModel.length);+ this.inverseMomentum = new Float32Array(this.inverseModel.length);+ this.rndMomentum = new Float32Array(this.rndPredictor.length);+ }++ /**+ * Compute intrinsic reward for a transition+ */+ computeIntrinsicReward(+ state: Float32Array,+ action: string,+ nextState: Float32Array+ ): number {+ if (this.config.useRND) {+ return this.computeRNDReward(nextState);+ } else {+ return this.computeICMReward(state, action, nextState);+ }+ }++ /**+ * Compute ICM-based intrinsic reward (prediction error)+ */+ computeICMReward(+ state: Float32Array,+ action: string,+ nextState: Float32Array+ ): number {+ const startTime = performance.now();++ // Encode states to features+ const stateFeature = this.encodeState(state);+ const nextStateFeature = this.encodeState(nextState);++ // Predict next state feature+ const actionIdx = this.hashAction(action);+ const predictedFeature = this.forwardPredict(stateFeature, actionIdx);++ // Compute prediction error as intrinsic reward+ let error = 0;+ for (let i = 0; i < this.config.featureDim; i++) {+ error += (predictedFeature[i] - nextStateFeature[i]) ** 2;+ }++ // Normalize intrinsic reward+ const intrinsic = this.normalizeIntrinsic(error);++ const elapsed = performance.now() - startTime;+ if (elapsed > 5) {+ console.warn(`ICM reward exceeded target: ${elapsed.toFixed(2)}ms > 5ms`);+ }++ return intrinsic * this.config.intrinsicCoef;+ }++ /**+ * Compute RND-based intrinsic reward+ */+ computeRNDReward(state: Float32Array): number {+ const startTime = performance.now();++ // Target network output (fixed random features)+ const targetOutput = this.rndForward(state, this.rndTarget);++ // Predictor network output (trained to match target)+ const predictorOutput = this.rndForward(state, this.rndPredictor);++ // Compute prediction error+ let error = 0;+ for (let i = 0; i < this.config.featureDim; i++) {+ error += (predictorOutput[i] - targetOutput[i]) ** 2;+ }++ // Normalize+ const intrinsic = this.normalizeIntrinsic(error);++ const elapsed = performance.now() - startTime;+ if (elapsed > 5) {+ console.warn(`RND reward exceeded target: ${elapsed.toFixed(2)}ms > 5ms`);+ }++ return intrinsic * this.config.intrinsicCoef;+ }++ /**+ * Update curiosity models from trajectory+ */+ update(trajectory: Trajectory): { forwardLoss: number; inverseLoss: number } {+ const startTime = performance.now();++ if (trajectory.steps.length < 2) {+ return { forwardLoss: 0, inverseLoss: 0 };+ }++ let totalForwardLoss = 0;+ let totalInverseLoss = 0;+ let count = 0;++ for (let i = 0; i < trajectory.steps.length - 1; i++) {+ const step = trajectory.steps[i];+ const nextStep = trajectory.steps[i + 1];++ const stateFeature = this.encodeState(step.stateAfter);+ const nextStateFeature = this.encodeState(nextStep.stateAfter);+ const actionIdx = this.hashAction(step.action);++ // Update forward model+ const forwardLoss = this.updateForwardModel(stateFeature, actionIdx, nextStateFeature);+ totalForwardLoss += forwardLoss;++ // Update inverse model+ const inverseLoss = this.updateInverseModel(stateFeature, nextStateFeature, actionIdx);+ totalInverseLoss += inverseLoss;++ // Update RND predictor if using RND+ if (this.config.useRND) {+ this.updateRNDPredictor(nextStep.stateAfter);+ }++ count++;+ }++ this.updateCount++;+ this.avgForwardLoss = count > 0 ? totalForwardLoss / count : 0;+ this.avgInverseLoss = count > 0 ? totalInverseLoss / count : 0;++ const elapsed = performance.now() - startTime;+ if (elapsed > 10) {+ console.warn(`Curiosity update exceeded target: ${elapsed.toFixed(2)}ms > 10ms`);+ }++ return {+ forwardLoss: this.avgForwardLoss,+ inverseLoss: this.avgInverseLoss,+ };+ }++ /**+ * Add intrinsic rewards to trajectory+ */+ augmentTrajectory(trajectory: Trajectory): Trajectory {+ const augmented = { ...trajectory, steps: [...trajectory.steps] };++ for (let i = 0; i < augmented.steps.length - 1; i++) {+ const step = augmented.steps[i];+ const nextStep = augmented.steps[i + 1];++ const intrinsic = this.computeIntrinsicReward(+ step.stateAfter,+ step.action,+ nextStep.stateAfter+ );++ // Augment reward+ augmented.steps[i] = {+ ...step,+ reward: step.reward + intrinsic,+ };+ }++ return augmented;+ }++ /**+ * Get statistics+ */+ getStats(): Record<string, number> {+ return {+ updateCount: this.updateCount,+ avgForwardLoss: this.avgForwardLoss,+ avgInverseLoss: this.avgInverseLoss,+ avgIntrinsicReward: this.avgIntrinsicReward,+ intrinsicMean: this.intrinsicMean,+ intrinsicStd: Math.sqrt(this.intrinsicVar),+ };+ }++ // ==========================================================================+ // Private Methods+ // ==========================================================================++ private initWeight(inputDim: number, outputDim: number): Float32Array {+ const weight = new Float32Array(inputDim * outputDim);+ const scale = Math.sqrt(2 / inputDim);+ for (let i = 0; i < weight.length; i++) {+ weight[i] = (Math.random() - 0.5) * scale;+ }+ return weight;+ }++ private encodeState(state: Float32Array): Float32Array {+ const featureDim = this.config.featureDim;+ const feature = new Float32Array(featureDim);++ for (let f = 0; f < featureDim; f++) {+ let sum = 0;+ for (let s = 0; s < Math.min(state.length, this.stateDim); s++) {+ sum += state[s] * this.featureEncoder[s * featureDim + f];+ }+ feature[f] = Math.max(0, sum); // ReLU+ }++ return feature;+ }++ private forwardPredict(stateFeature: Float32Array, action: number): Float32Array {+ const featureDim = this.config.featureDim;+ const inputDim = featureDim + this.numActions;+ const predicted = new Float32Array(featureDim);++ // Concatenate feature and one-hot action+ const input = new Float32Array(inputDim);+ input.set(stateFeature);+ input[featureDim + action] = 1;++ // Forward pass+ for (let f = 0; f < featureDim; f++) {+ let sum = 0;+ for (let i = 0; i < inputDim; i++) {+ sum += input[i] * this.forwardModel[i * featureDim + f];+ }+ predicted[f] = sum;+ }++ return predicted;+ }++ private inversePredict(+ stateFeature: Float32Array,+ nextStateFeature: Float32Array+ ): Float32Array {+ const featureDim = this.config.featureDim;+ const logits = new Float32Array(this.numActions);++ // Concatenate features+ const input = new Float32Array(2 * featureDim);+ input.set(stateFeature);+ input.set(nextStateFeature, featureDim);++ // Forward pass+ for (let a = 0; a < this.numActions; a++) {+ let sum = 0;+ for (let i = 0; i < 2 * featureDim; i++) {+ sum += input[i] * this.inverseModel[i * this.numActions + a];+ }+ logits[a] = sum;+ }++ return this.softmax(logits);+ }++ private rndForward(state: Float32Array, weights: Float32Array): Float32Array {+ const featureDim = this.config.featureDim;+ const output = new Float32Array(featureDim);++ for (let f = 0; f < featureDim; f++) {+ let sum = 0;+ for (let s = 0; s < Math.min(state.length, this.stateDim); s++) {+ sum += state[s] * weights[s * featureDim + f];+ }+ output[f] = Math.max(0, sum); // ReLU+ }++ return output;+ }++ private updateForwardModel(+ stateFeature: Float32Array,+ action: number,+ targetFeature: Float32Array+ ): number {+ const featureDim = this.config.featureDim;+ const inputDim = featureDim + this.numActions;+ const lr = this.config.forwardLR;+ const beta = 0.9;++ // Forward pass+ const predicted = this.forwardPredict(stateFeature, action);++ // Compute loss and gradient+ let loss = 0;+ const grad = new Float32Array(predicted.length);++ for (let f = 0; f < featureDim; f++) {+ const diff = predicted[f] - targetFeature[f];+ loss += diff * diff;+ grad[f] = 2 * diff;+ }++ // Backprop to weights+ const input = new Float32Array(inputDim);+ input.set(stateFeature);+ input[featureDim + action] = 1;++ for (let i = 0; i < inputDim; i++) {+ for (let f = 0; f < featureDim; f++) {+ const weightGrad = input[i] * grad[f];+ const idx = i * featureDim + f;+ this.forwardMomentum[idx] = beta * this.forwardMomentum[idx] + (1 - beta) * weightGrad;+ this.forwardModel[idx] -= lr * this.forwardMomentum[idx];+ }+ }++ return loss;+ }++ private updateInverseModel(+ stateFeature: Float32Array,+ nextStateFeature: Float32Array,+ targetAction: number+ ): number {+ const featureDim = this.config.featureDim;+ const lr = this.config.inverseLR;+ const beta = 0.9;++ // Forward pass+ const probs = this.inversePredict(stateFeature, nextStateFeature);++ // Cross-entropy loss+ const loss = -Math.log(probs[targetAction] + 1e-8);++ // Gradient+ const grad = new Float32Array(this.numActions);+ for (let a = 0; a < this.numActions; a++) {+ grad[a] = probs[a] - (a === targetAction ? 1 : 0);+ }++ // Backprop to weights+ const input = new Float32Array(2 * featureDim);+ input.set(stateFeature);+ input.set(nextStateFeature, featureDim);++ for (let i = 0; i < 2 * featureDim; i++) {+ for (let a = 0; a < this.numActions; a++) {+ const weightGrad = input[i] * grad[a];+ const idx = i * this.numActions + a;+ this.inverseMomentum[idx] = beta * this.inverseMomentum[idx] + (1 - beta) * weightGrad;+ this.inverseModel[idx] -= lr * this.inverseMomentum[idx];+ }+ }++ return loss;+ }++ private updateRNDPredictor(state: Float32Array): void {+ const featureDim = this.config.featureDim;+ const lr = this.config.learningRate;+ const beta = 0.9;++ // Target output (fixed)+ const targetOutput = this.rndForward(state, this.rndTarget);++ // Predictor output+ const predictorOutput = this.rndForward(state, this.rndPredictor);++ // Gradient+ const grad = new Float32Array(featureDim);+ for (let f = 0; f < featureDim; f++) {+ grad[f] = 2 * (predictorOutput[f] - targetOutput[f]);+ }++ // Update predictor weights+ for (let s = 0; s < Math.min(state.length, this.stateDim); s++) {+ for (let f = 0; f < featureDim; f++) {+ if (predictorOutput[f] > 0) { // ReLU gradient+ const weightGrad = state[s] * grad[f];+ const idx = s * featureDim + f;+ this.rndMomentum[idx] = beta * this.rndMomentum[idx] + (1 - beta) * weightGrad;+ this.rndPredictor[idx] -= lr * this.rndMomentum[idx];+ }+ }+ }+ }++ private normalizeIntrinsic(raw: number): number {+ // Update running statistics+ const alpha = 0.01;+ this.intrinsicMean = (1 - alpha) * this.intrinsicMean + alpha * raw;+ this.intrinsicVar = (1 - alpha) * this.intrinsicVar + alpha * (raw - this.intrinsicMean) ** 2;++ // Normalize+ const normalized = (raw - this.intrinsicMean) / (Math.sqrt(this.intrinsicVar) + 1e-8);++ // Clip to reasonable range+ return Math.max(-5, Math.min(5, normalized));+ }++ private softmax(logits: Float32Array): Float32Array {+ const max = Math.max(...logits);+ const exps = new Float32Array(logits.length);+ let sum = 0;++ for (let i = 0; i < logits.length; i++) {+ exps[i] = Math.exp(logits[i] - max);+ sum += exps[i];+ }++ for (let i = 0; i < exps.length; i++) {+ exps[i] /= sum;+ }++ return exps;+ }++ private hashAction(action: string): number {+ let hash = 0;+ for (let i = 0; i < action.length; i++) {+ hash = (hash * 31 + action.charCodeAt(i)) % this.numActions;+ }+ return hash;+ }+}++/**+ * Factory function+ */+export function createCuriosity(config?: Partial<CuriosityConfig>): CuriosityModule {+ return new CuriosityModule(config);+}diff --git a/v3/neural/algorithms/index.ts b/v3/neural/algorithms/index.tsnew file mode 100644index 0000000000..c12d9795ed--- /dev/null+++ b/v3/neural/algorithms/index.ts@@ -0,0 +1,121 @@+/**+ * RL Algorithms Index+ *+ * Exports all reinforcement learning algorithm implementations.+ */++// PPO - Proximal Policy Optimization+export {+ PPOAlgorithm,+ createPPO,+ DEFAULT_PPO_CONFIG,+} from './ppo.js';+export type { PPOConfig } from '../types.js';++// DQN - Deep Q-Network+export {+ DQNAlgorithm,+ createDQN,+ DEFAULT_DQN_CONFIG,+} from './dqn.js';+export type { DQNConfig } from '../types.js';++// A2C - Advantage Actor-Critic+export {+ A2CAlgorithm,+ createA2C,+ DEFAULT_A2C_CONFIG,+} from './a2c.js';+export type { A2CConfig } from './a2c.js';++// Decision Transformer+export {+ DecisionTransformer,+ createDecisionTransformer,+ DEFAULT_DT_CONFIG,+} from './decision-transformer.js';+export type { DecisionTransformerConfig } from '../types.js';++// Q-Learning (Tabular)+export {+ QLearning,+ createQLearning,+ DEFAULT_QLEARNING_CONFIG,+} from './q-learning.js';+export type { QLearningConfig } from './q-learning.js';++// SARSA+export {+ SARSAAlgorithm,+ createSARSA,+ DEFAULT_SARSA_CONFIG,+} from './sarsa.js';+export type { SARSAConfig } from './sarsa.js';++// Curiosity-Driven Exploration+export {+ CuriosityModule,+ createCuriosity,+ DEFAULT_CURIOSITY_CONFIG,+} from './curiosity.js';+export type { CuriosityConfig } from '../types.js';++/**+ * Algorithm factory+ */+import type { RLAlgorithm, RLConfig } from '../types.js';+import { createPPO } from './ppo.js';+import { createDQN } from './dqn.js';+import { createA2C } from './a2c.js';+import { createDecisionTransformer } from './decision-transformer.js';+import { createQLearning } from './q-learning.js';+import { createSARSA } from './sarsa.js';+import { createCuriosity } from './curiosity.js';++/**+ * Create an RL algorithm by name+ */+export function createAlgorithm(algorithm: RLAlgorithm, config?: Partial<RLConfig>): unknown {+ switch (algorithm) {+ case 'ppo':+ return createPPO(config);+ case 'dqn':+ return createDQN(config);+ case 'a2c':+ return createA2C(config);+ case 'decision-transformer':+ return createDecisionTransformer(config);+ case 'q-learning':+ return createQLearning(config);+ case 'sarsa':+ return createSARSA(config);+ case 'curiosity':+ return createCuriosity(config);+ default:+ throw new Error(`Unknown algorithm: ${algorithm}`);+ }+}++/**+ * Get default configuration for an algorithm+ */+export function getDefaultConfig(algorithm: RLAlgorithm): RLConfig {+ switch (algorithm) {+ case 'ppo':+ return { ...DEFAULT_PPO_CONFIG };+ case 'dqn':+ return { ...DEFAULT_DQN_CONFIG };+ case 'a2c':+ return { ...DEFAULT_A2C_CONFIG };+ case 'decision-transformer':+ return { ...DEFAULT_DT_CONFIG };+ case 'q-learning':+ return { ...DEFAULT_QLEARNING_CONFIG };+ case 'sarsa':+ return { ...DEFAULT_SARSA_CONFIG };+ case 'curiosity':+ return { ...DEFAULT_CURIOSITY_CONFIG };+ default:+ throw new Error(`Unknown algorithm: ${algorithm}`);+ }+}