Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

checkpoint: File edit:

rUv2604030cc8a5
diff --git a/.agentic-flow/intelligence.json b/.agentic-flow/intelligence.json
index eb119b7071..2617f8be47 100644
--- a/.agentic-flow/intelligence.json
+++ b/.agentic-flow/intelligence.json
@@ -146,7 +146,7 @@
"test-engineer": 0.6
},
"command:": {
- "success": 0.999823035654572
+ "success": 0.9999059538913014
}
},
"sequences": {},
@@ -456,8 +456,8 @@
},
"errorPatterns": [],
"metrics": {
- "totalRoutes": 153,
- "successfulRoutes": 153,
+ "totalRoutes": 163,
+ "successfulRoutes": 163,
"routingHistory": [
{
"timestamp": "2026-01-04T15:51:13.039Z",
@@ -884,6 +884,30 @@
"task": "edit:",
"agent": "unknown",
"success": true
+ },
+ {
+ "timestamp": "2026-01-04T17:35:28.457Z",
+ "task": "edit:",
+ "agent": "unknown",
+ "success": true
+ },
+ {
+ "timestamp": "2026-01-04T17:35:37.509Z",
+ "task": "edit:",
+ "agent": "unknown",
+ "success": true
+ },
+ {
+ "timestamp": "2026-01-04T17:35:51.985Z",
+ "task": "edit:",
+ "agent": "unknown",
+ "success": true
+ },
+ {
+ "timestamp": "2026-01-04T17:36:52.479Z",
+ "task": "edit:",
+ "agent": "unknown",
+ "success": true
}
]
},
diff --git a/.claude/helpers/swarm-monitor.sh b/.claude/helpers/swarm-monitor.sh
index c38419b9c3..af47a91f8e 100755
--- a/.claude/helpers/swarm-monitor.sh
+++ b/.claude/helpers/swarm-monitor.sh
@@ -75,7 +75,7 @@ update_activity_metrics() {
# Update active agents in metrics
if [ -f "$UPDATE_SCRIPT" ]; then
- "$UPDATE_SCRIPT" agents "$agent_count" >/dev/null 2>&1
+ "$UPDATE_SCRIPT" agent "$agent_count" >/dev/null 2>&1
fi
# Update integration status based on activity
diff --git a/.claude/settings.json b/.claude/settings.json
index 32f8b9326f..35e595cc29 100644
--- a/.claude/settings.json
+++ b/.claude/settings.json
@@ -116,6 +116,11 @@
"type": "command",
"timeout": 3000,
"command": "npx agentic-flow@alpha hooks post-command \"$TOOL_INPUT_command\" --success --learn --performance-metrics --v3-mode 2>/dev/null || true"
+ },
+ {
+ "type": "command",
+ "timeout": 2000,
+ "command": ".claude/helpers/swarm-monitor.sh check 2>/dev/null || true"
}
]
},
@@ -127,6 +132,11 @@
"timeout": 2000,
"command": "npx agentic-flow@alpha hooks metrics --agent-performance --swarm-efficiency 2>/dev/null || true"
},
+ {
+ "type": "command",
+ "timeout": 3000,
+ "command": ".claude/helpers/swarm-monitor.sh check 2>/dev/null || true"
+ },
{
"type": "command",
"timeout": 4000,
@@ -229,6 +239,7 @@
"v3ConfigValidator": ".claude/helpers/validate-v3-config.sh",
"v3QuickStatus": ".claude/helpers/v3-quick-status.sh",
"v3CheckpointManager": ".claude/helpers/checkpoint-manager.sh",
+ "v3SwarmMonitor": ".claude/helpers/swarm-monitor.sh",
"v3Master": ".claude/helpers/v3.sh"
}
},
diff --git a/v3/benchmarks/attention/memory-efficiency.bench.ts b/v3/benchmarks/attention/memory-efficiency.bench.ts
new file mode 100644
index 0000000000..280f7a4023
--- /dev/null
+++ b/v3/benchmarks/attention/memory-efficiency.bench.ts
@@ -0,0 +1,569 @@
+/**
+ * Attention Memory Efficiency Benchmark
+ *
+ * Target: 50-75% memory reduction
+ *
+ * Measures memory efficiency of different attention implementations
+ * and optimization strategies.
+ */
+
+import { benchmark, BenchmarkRunner, formatTime, formatBytes } from '../framework/benchmark.js';
+
+// ============================================================================
+// Memory Tracking
+// ============================================================================
+
+interface MemorySnapshot {
+ heapUsed: number;
+ heapTotal: number;
+ external: number;
+ arrayBuffers: number;
+ rss: number;
+}
+
+function takeMemorySnapshot(): MemorySnapshot {
+ const mem = process.memoryUsage();
+ return {
+ heapUsed: mem.heapUsed,
+ heapTotal: mem.heapTotal,
+ external: mem.external,
+ arrayBuffers: mem.arrayBuffers,
+ rss: mem.rss,
+ };
+}
+
+function calculateMemoryDelta(before: MemorySnapshot, after: MemorySnapshot): number {
+ return after.heapUsed - before.heapUsed;
+}
+
+// ============================================================================
+// Attention Implementations for Memory Testing
+// ============================================================================
+
+/**
+ * Standard attention - stores full attention matrix
+ */
+function standardAttention(
+ query: Float32Array,
+ key: Float32Array,
+ value: Float32Array,
+ seqLength: number,
+ headDim: number
+): { output: Float32Array; attentionMatrix: Float32Array } {
+ const scale = 1 / Math.sqrt(headDim);
+
+ // Full attention matrix - O(n^2) memory
+ const attentionMatrix = new Float32Array(seqLength * seqLength);
+
+ // Compute scores
+ for (let i = 0; i < seqLength; i++) {
+ for (let j = 0; j < seqLength; j++) {
+ let dot = 0;
+ for (let k = 0; k < headDim; k++) {
+ dot += query[i * headDim + k]! * key[j * headDim + k]!;
+ }
+ attentionMatrix[i * seqLength + j] = dot * scale;
+ }
+ }
+
+ // Softmax
+ for (let i = 0; i < seqLength; i++) {
+ let max = -Infinity;
+ for (let j = 0; j < seqLength; j++) {
+ max = Math.max(max, attentionMatrix[i * seqLength + j]!);
+ }
+
+ let sum = 0;
+ for (let j = 0; j < seqLength; j++) {
+ const exp = Math.exp(attentionMatrix[i * seqLength + j]! - max);
+ attentionMatrix[i * seqLength + j] = exp;
+ sum += exp;
+ }
+
+ for (let j = 0; j < seqLength; j++) {
+ attentionMatrix[i * seqLength + j]! /= sum;
+ }
+ }
+
+ // Output
+ const output = new Float32Array(seqLength * headDim);
+ for (let i = 0; i < seqLength; i++) {
+ for (let k = 0; k < headDim; k++) {
+ let sum = 0;
+ for (let j = 0; j < seqLength; j++) {
+ sum += attentionMatrix[i * seqLength + j]! * value[j * headDim + k]!;
+ }
+ output[i * headDim + k] = sum;
+ }
+ }
+
+ return { output, attentionMatrix };
+}
+
+/**
+ * Memory-efficient attention - no full matrix storage
+ */
+function memoryEfficientAttention(
+ query: Float32Array,
+ key: Float32Array,
+ value: Float32Array,
+ seqLength: number,
+ headDim: number
+): { output: Float32Array } {
+ const scale = 1 / Math.sqrt(headDim);
+ const output = new Float32Array(seqLength * headDim);
+
+ // Process row by row - O(n) memory for scores
+ const rowScores = new Float32Array(seqLength);
+
+ for (let i = 0; i < seqLength; i++) {
+ // Compute scores for this row
+ let max = -Infinity;
+ for (let j = 0; j < seqLength; j++) {
+ let dot = 0;
+ for (let k = 0; k < headDim; k++) {
+ dot += query[i * headDim + k]! * key[j * headDim + k]!;
+ }
+ rowScores[j] = dot * scale;
+ max = Math.max(max, rowScores[j]!);
+ }
+
+ // Softmax
+ let sum = 0;
+ for (let j = 0; j < seqLength; j++) {
+ rowScores[j] = Math.exp(rowScores[j]! - max);
+ sum += rowScores[j]!;
+ }
+ for (let j = 0; j < seqLength; j++) {
+ rowScores[j]! /= sum;
+ }
+
+ // Compute output for this row
+ for (let k = 0; k < headDim; k++) {
+ let val = 0;
+ for (let j = 0; j < seqLength; j++) {
+ val += rowScores[j]! * value[j * headDim + k]!;
+ }
+ output[i * headDim + k] = val;
+ }
+ }
+
+ return { output };
+}
+
+/**
+ * Chunked attention - process in blocks
+ */
+function chunkedAttention(
+ query: Float32Array,
+ key: Float32Array,
+ value: Float32Array,
+ seqLength: number,
+ headDim: number,
+ chunkSize: number = 64
+): { output: Float32Array } {
+ const scale = 1 / Math.sqrt(headDim);
+ const output = new Float32Array(seqLength * headDim);
+ const numChunks = Math.ceil(seqLength / chunkSize);
+
+ // Chunk buffer - O(chunkSize^2) memory
+ const chunkScores = new Float32Array(chunkSize * seqLength);
+ const rowMax = new Float32Array(chunkSize).fill(-Infinity);
+ const rowSum = new Float32Array(chunkSize).fill(0);
+
+ for (let ci = 0; ci < numChunks; ci++) {
+ const iStart = ci * chunkSize;
+ const iEnd = Math.min(iStart + chunkSize, seqLength);
+ const iSize = iEnd - iStart;
+
+ // Reset accumulators
+ rowMax.fill(-Infinity);
+ rowSum.fill(0);
+ output.fill(0, iStart * headDim, iEnd * headDim);
+
+ for (let cj = 0; cj < numChunks; cj++) {
+ const jStart = cj * chunkSize;
+ const jEnd = Math.min(jStart + chunkSize, seqLength);
+ const jSize = jEnd - jStart;
+
+ // Compute chunk scores
+ for (let i = 0; i < iSize; i++) {
+ for (let j = 0; j < jSize; j++) {
+ let dot = 0;
+ for (let k = 0; k < headDim; k++) {
+ dot += query[(iStart + i) * headDim + k]! * key[(jStart + j) * headDim + k]!;
+ }
+ chunkScores[i * seqLength + jStart + j] = dot * scale;
+ }
+ }
+
+ // Online softmax update
+ for (let i = 0; i < iSize; i++) {
+ const prevMax = rowMax[i]!;
+
+ // Find new max
+ for (let j = 0; j < jSize; j++) {
+ rowMax[i] = Math.max(rowMax[i]!, chunkScores[i * seqLength + jStart + j]!);
+ }
+
+ // Rescale previous
+ if (prevMax !== -Infinity && prevMax !== rowMax[i]) {
+ const rescale = Math.exp(prevMax - rowMax[i]!);
+ rowSum[i]! *= rescale;
+ for (let k = 0; k < headDim; k++) {
+ output[(iStart + i) * headDim + k]! *= rescale;
+ }
+ }
+
+ // Add new exponentials
+ for (let j = 0; j < jSize; j++) {
+ const exp = Math.exp(chunkScores[i * seqLength + jStart + j]! - rowMax[i]!);
+ chunkScores[i * seqLength + jStart + j] = exp;
+ rowSum[i]! += exp;
+ }
+
+ // Accumulate output
+ for (let k = 0; k < headDim; k++) {
+ for (let j = 0; j < jSize; j++) {
+ output[(iStart + i) * headDim + k]! +=
+ chunkScores[i * seqLength + jStart + j]! * value[(jStart + j) * headDim + k]!;
+ }
+ }
+ }
+ }
+
+ // Final normalization
+ for (let i = 0; i < iSize; i++) {
+ for (let k = 0; k < headDim; k++) {
+ output[(iStart + i) * headDim + k]! /= rowSum[i]!;
+ }
+ }
+ }
+
+ return { output };
+}
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+function generateRandomTensor(size: number): Float32Array {
+ const tensor = new Float32Array(size);
+ for (let i = 0; i < size; i++) {
+ tensor[i] = Math.random() * 2 - 1;
+ }
+ return tensor;
+}
+
+// ============================================================================
+// Benchmark Suite
+// ============================================================================
+
+export async function runMemoryEfficiencyBenchmarks(): Promise<void> {
+ const runner = new BenchmarkRunner('Attention Memory Efficiency');
+
+ console.log('\n--- Attention Memory Efficiency Benchmarks ---\n');
+
+ // Test configurations
+ const seqLengths = [128, 256, 512, 1024];
+ const headDim = 64;
+
+ // Memory scaling comparison
+ console.log('--- Memory Scaling by Sequence Length ---\n');
+
+ const memoryResults: Array<{
+ seqLength: number;
+ standard: number;
+ efficient: number;
+ chunked: number;
+ reduction: number;
+ }> = [];
+
+ for (const seqLength of seqLengths) {
+ console.log(`Sequence Length: ${seqLength}`);
+
+ const size = seqLength * headDim;
+ const query = generateRandomTensor(size);
+ const key = generateRandomTensor(size);
+ const value = generateRandomTensor(size);
+
+ // Standard attention memory
+ if (typeof global.gc === 'function') global.gc();
+ const standardBefore = takeMemorySnapshot();
+ const standardResult = standardAttention(query, key, value, seqLength, headDim);
+ const standardAfter = takeMemorySnapshot();
+ const standardMem = calculateMemoryDelta(standardBefore, standardAfter);
+ void standardResult;
+
+ // Memory-efficient attention
+ if (typeof global.gc === 'function') global.gc();
+ const efficientBefore = takeMemorySnapshot();
+ const efficientResult = memoryEfficientAttention(query, key, value, seqLength, headDim);
+ const efficientAfter = takeMemorySnapshot();
+ const efficientMem = calculateMemoryDelta(efficientBefore, efficientAfter);
+ void efficientResult;
+
+ // Chunked attention
+ if (typeof global.gc === 'function') global.gc();
+ const chunkedBefore = takeMemorySnapshot();
+ const chunkedResult = chunkedAttention(query, key, value, seqLength, headDim);
+ const chunkedAfter = takeMemorySnapshot();
+ const chunkedMem = calculateMemoryDelta(chunkedBefore, chunkedAfter);
+ void chunkedResult;
+
+ const reduction = ((standardMem - efficientMem) / standardMem) * 100;
+
+ memoryResults.push({
+ seqLength,
+ standard: standardMem,
+ efficient: efficientMem,
+ chunked: chunkedMem,
+ reduction,
+ });
+
+ console.log(` Standard: ${formatBytes(standardMem)}`);
+ console.log(` Efficient: ${formatBytes(efficientMem)}`);
+ console.log(` Chunked: ${formatBytes(chunkedMem)}`);
+ console.log(` Reduction: ${reduction.toFixed(1)}%`);
+ console.log('');
+ }
+
+ // Theoretical memory comparison
+ console.log('--- Theoretical Memory Analysis ---\n');
+
+ for (const seqLength of seqLengths) {
+ const bytesPerFloat = 4;
+
+ // Standard: stores full n x n attention matrix
+ const standardTheory = seqLength * seqLength * bytesPerFloat;
+
+ // Efficient: stores only one row at a time
+ const efficientTheory = seqLength * bytesPerFloat;
+
+ // Chunked: stores chunk x n scores
+ const chunkSize = 64;
+ const chunkedTheory = chunkSize * seqLength * bytesPerFloat;
+
+ console.log(`Seq ${seqLength}:`);
+ console.log(` Standard: ${formatBytes(standardTheory)} (n^2)`);
+ console.log(` Efficient: ${formatBytes(efficientTheory)} (n)`);
+ console.log(` Chunked: ${formatBytes(chunkedTheory)} (chunk * n)`);
+ console.log(` Theoretical reduction: ${((1 - efficientTheory / standardTheory) * 100).toFixed(1)}%`);
+ console.log('');
+ }
+
+ // Performance vs Memory tradeoff
+ console.log('--- Performance vs Memory Tradeoff ---\n');
+
+ const tradeoffConfig = { seqLength: 512, headDim: 64 };
+ const size = tradeoffConfig.seqLength * tradeoffConfig.headDim;
+ const q = generateRandomTensor(size);
+ const k = generateRandomTensor(size);
+ const v = generateRandomTensor(size);
+
+ // Standard performance
+ const standardPerfResult = await runner.run(
+ 'standard-attention-perf',
+ async () => {
+ standardAttention(q, k, v, tradeoffConfig.seqLength, tradeoffConfig.headDim);
+ },
+ { iterations: 20 }
+ );
+
+ console.log(`Standard Performance: ${formatTime(standardPerfResult.mean)}`);
+
+ // Efficient performance
+ const efficientPerfResult = await runner.run(
+ 'efficient-attention-perf',
+ async () => {
+ memoryEfficientAttention(q, k, v, tradeoffConfig.seqLength, tradeoffConfig.headDim);
+ },
+ { iterations: 20 }
+ );
+
+ console.log(`Memory-Efficient Performance: ${formatTime(efficientPerfResult.mean)}`);
+
+ // Chunked performance with different chunk sizes
+ const chunkSizes = [32, 64, 128, 256];
+
+ for (const chunkSize of chunkSizes) {
+ const chunkedPerfResult = await runner.run(
+ `chunked-attention-chunk${chunkSize}`,
+ async () => {
+ chunkedAttention(q, k, v, tradeoffConfig.seqLength, tradeoffConfig.headDim, chunkSize);
+ },
+ { iterations: 20 }
+ );
+
+ console.log(`Chunked (size=${chunkSize}): ${formatTime(chunkedPerfResult.mean)}`);
+ }
+
+ // Multi-head memory analysis
+ console.log('\n--- Multi-Head Memory Analysis ---\n');
+
+ const numHeads = [4, 8, 16, 32];
+ const mhaSeqLength = 256;
+
+ for (const heads of numHeads) {
+ const mhaSize = mhaSeqLength * headDim;
+
+ // Standard MHA memory
+ const standardMHAMem = mhaSeqLength * mhaSeqLength * 4 * heads; // attention matrices
+ const qkvMem = mhaSize * 4 * 3 * heads; // QKV storage
+
+ // GQA memory (shared KV)
+ const gqaKVHeads = heads / 4;
+ const gqaMem = mhaSeqLength * mhaSeqLength * 4 * heads + // attention matrices (same)
+ mhaSize * 4 * heads + // Q storage
+ mhaSize * 4 * 2 * gqaKVHeads; // shared KV
+
+ // MQA memory (single KV)
+ const mqaMem = mhaSeqLength * mhaSeqLength * 4 * heads + // attention matrices
+ mhaSize * 4 * heads + // Q storage
+ mhaSize * 4 * 2; // single KV
+
+ console.log(`${heads} heads:`);
+ console.log(` Standard MHA: ${formatBytes(standardMHAMem + qkvMem)}`);
+ console.log(` GQA (${gqaKVHeads} KV): ${formatBytes(gqaMem)}`);
+ console.log(` MQA (1 KV): ${formatBytes(mqaMem)}`);
+ console.log(` MQA reduction: ${(((standardMHAMem + qkvMem) - mqaMem) / (standardMHAMem + qkvMem) * 100).toFixed(1)}%`);
+ console.log('');
+ }
+
+ // Summary
+ console.log('--- Summary ---\n');
+
+ console.log('Memory Reduction Achieved:');
+ for (const result of memoryResults) {
+ const targetMet = result.reduction >= 50;
+ console.log(
+ ` Seq ${result.seqLength}: ${result.reduction.toFixed(1)}% ${targetMet ? '(TARGET MET)' : ''}`
+ );
+ }
+
+ console.log('\nPerformance Comparison (seq=512):');
+ console.log(` Standard: ${formatTime(standardPerfResult.mean)}`);
+ console.log(` Efficient: ${formatTime(efficientPerfResult.mean)}`);
+
+ // Print full results
+ runner.printResults();
+}
+
+// ============================================================================
+// Memory Efficiency Optimization Strategies
+// ============================================================================
+
+export const memoryOptimizations = {
+ /**
+ * Online softmax computation
+ */
+ onlineSoftmax: {
+ description: 'Compute softmax in streaming fashion without storing all values',
+ expectedImprovement: 'O(n) instead of O(n^2) for softmax',
+ implementation: `
+ class OnlineSoftmax {
+ private max = -Infinity;
+ private sum = 0;
+ private count = 0;
+
+ add(value: number): void {
+ if (value > this.max) {
+ this.sum *= Math.exp(this.max - value);
+ this.max = value;
+ }
+ this.sum += Math.exp(value - this.max);
+ this.count++;
+ }
+
+ normalize(value: number): number {
+ return Math.exp(value - this.max) / this.sum;
+ }
+ }
+ `,
+ },
+
+ /**
+ * Gradient checkpointing
+ */
+ gradientCheckpointing: {
+ description: 'Recompute attention during backward pass instead of storing',
+ expectedImprovement: 'O(1) memory for activations',
+ implementation: `
+ function checkpointedAttention(q, k, v) {
+ const output = computeAttention(q, k, v);
+
+ function backward(gradOutput) {
+ // Recompute attention weights during backward
+ const attnWeights = recomputeAttention(q, k);
+ return computeGradients(gradOutput, attnWeights, q, k, v);
+ }
+
+ return { output, backward };
+ }
+ `,
+ },
+
+ /**
+ * Sparse attention patterns
+ */
+ sparseAttention: {
+ description: 'Only compute attention for relevant positions',
+ expectedImprovement: 'O(n * k) instead of O(n^2) where k << n',
+ implementation: `
+ function sparseAttention(q, k, v, pattern: 'local' | 'strided' | 'block') {
+ const sparseMask = generateSparsePattern(q.length, pattern);
+ return computeAttentionWithMask(q, k, v, sparseMask);
+ }
+ `,
+ },
+
+ /**
+ * Quantization
+ */
+ quantization: {
+ description: 'Use lower precision for attention computation',
+ expectedImprovement: '2-4x memory reduction',
+ implementation: `
+ function quantizedAttention(q, k, v) {
+ // Quantize to int8
+ const qInt8 = quantizeToInt8(q);
+ const kInt8 = quantizeToInt8(k);
+
+ // Compute in int8
+ const scores = computeInt8Attention(qInt8, kInt8);
+
+ // Dequantize for output
+ return dequantizeAndApply(scores, v);
+ }
+ `,
+ },
+
+ /**
+ * Memory pooling
+ */
+ memoryPooling: {
+ description: 'Reuse memory buffers across forward passes',
+ expectedImprovement: 'Eliminates allocation overhead',
+ implementation: `
+ class AttentionMemoryPool {
+ private scoreBuffer: Float32Array;
+ private outputBuffer: Float32Array;
+
+ forward(q, k, v) {
+ // Reuse pre-allocated buffers
+ computeScores(q, k, this.scoreBuffer);
+ applySoftmax(this.scoreBuffer);
+ computeOutput(this.scoreBuffer, v, this.outputBuffer);
+ return this.outputBuffer;
+ }
+ }
+ `,
+ },
+};
+
+// Run if executed directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+ runMemoryEfficiencyBenchmarks().catch(console.error);
+}
+
+export default runMemoryEfficiencyBenchmarks;
diff --git a/v3/neural/algorithms/q-learning.ts b/v3/neural/algorithms/q-learning.ts
new file mode 100644
index 0000000000..956dc22a67
--- /dev/null
+++ b/v3/neural/algorithms/q-learning.ts
@@ -0,0 +1,333 @@
+/**
+ * Tabular Q-Learning
+ *
+ * Classic Q-learning algorithm with:
+ * - Epsilon-greedy exploration
+ * - State hashing for continuous states
+ * - Eligibility traces (optional)
+ * - Experience replay
+ *
+ * Suitable for smaller state spaces or discretized environments.
+ * Performance Target: <1ms per update
+ */
+
+import type { Trajectory, RLConfig } from '../types.js';
+
+/**
+ * Q-Learning configuration
+ */
+export interface QLearningConfig extends RLConfig {
+ algorithm: 'q-learning';
+ explorationInitial: number;
+ explorationFinal: number;
+ explorationDecay: number;
+ maxStates: number;
+ useEligibilityTraces: boolean;
+ traceDecay: number;
+}
+
+/**
+ * Default Q-Learning configuration
+ */
+export const DEFAULT_QLEARNING_CONFIG: QLearningConfig = {
+ algorithm: 'q-learning',
+ learningRate: 0.1,
+ gamma: 0.99,
+ entropyCoef: 0,
+ valueLossCoef: 1,
+ maxGradNorm: 1,
+ epochs: 1,
+ miniBatchSize: 1,
+ explorationInitial: 1.0,
+ explorationFinal: 0.01,
+ explorationDecay: 10000,
+ maxStates: 10000,
+ useEligibilityTraces: false,
+ traceDecay: 0.9,
+};
+
+/**
+ * Q-table entry
+ */
+interface QEntry {
+ qValues: Float32Array;
+ visits: number;
+ lastUpdate: number;
+}
+
+/**
+ * Q-Learning Algorithm Implementation
+ */
+export class QLearning {
+ private config: QLearningConfig;
+
+ // Q-table
+ private qTable: Map<string, QEntry> = new Map();
+
+ // Exploration
+ private epsilon: number;
+ private stepCount = 0;
+
+ // Number of actions
+ private numActions = 4;
+
+ // Eligibility traces
+ private traces: Map<string, Float32Array> = new Map();
+
+ // Statistics
+ private updateCount = 0;
+ private avgTDError = 0;
+
+ constructor(config: Partial<QLearningConfig> = {}) {
+ this.config = { ...DEFAULT_QLEARNING_CONFIG, ...config };
+ this.epsilon = this.config.explorationInitial;
+ }
+
+ /**
+ * Update Q-values from trajectory
+ */
+ update(trajectory: Trajectory): { tdError: number } {
+ const startTime = performance.now();
+
+ if (trajectory.steps.length === 0) {
+ return { tdError: 0 };
+ }
+
+ let totalTDError = 0;
+
+ // Reset eligibility traces for new trajectory
+ if (this.config.useEligibilityTraces) {
+ this.traces.clear();
+ }
+
+ for (let i = 0; i < trajectory.steps.length; i++) {
+ const step = trajectory.steps[i];
+ const stateKey = this.hashState(step.stateBefore);
+ const action = this.hashAction(step.action);
+
+ // Get or create Q-entry
+ const qEntry = this.getOrCreateEntry(stateKey);
+
+ // Current Q-value
+ const currentQ = qEntry.qValues[action];
+
+ // Compute target Q-value
+ let targetQ: number;
+ if (i === trajectory.steps.length - 1) {
+ // Terminal state
+ targetQ = step.reward;
+ } else {
+ const nextStateKey = this.hashState(step.stateAfter);
+ const nextEntry = this.getOrCreateEntry(nextStateKey);
+ const maxNextQ = Math.max(...nextEntry.qValues);
+ targetQ = step.reward + this.config.gamma * maxNextQ;
+ }
+
+ // TD error
+ const tdError = targetQ - currentQ;
+ totalTDError += Math.abs(tdError);
+
+ if (this.config.useEligibilityTraces) {
+ // Update eligibility trace
+ this.updateTrace(stateKey, action);
+
+ // Update all states with traces
+ this.updateWithTraces(tdError);
+ } else {
+ // Simple Q-learning update
+ qEntry.qValues[action] += this.config.learningRate * tdError;
+ qEntry.visits++;
+ qEntry.lastUpdate = Date.now();
+ }
+ }
+
+ // Decay exploration
+ this.stepCount += trajectory.steps.length;
+ this.epsilon = Math.max(
+ this.config.explorationFinal,
+ this.config.explorationInitial - this.stepCount / this.config.explorationDecay
+ );
+
+ // Prune Q-table if too large
+ if (this.qTable.size > this.config.maxStates) {
+ this.pruneQTable();
+ }
+
+ this.updateCount++;
+ this.avgTDError = totalTDError / trajectory.steps.length;
+
+ const elapsed = performance.now() - startTime;
+ if (elapsed > 1) {
+ console.warn(`Q-learning update exceeded target: ${elapsed.toFixed(2)}ms > 1ms`);
+ }
+
+ return { tdError: this.avgTDError };
+ }
+
+ /**
+ * Get action using epsilon-greedy policy
+ */
+ getAction(state: Float32Array, explore: boolean = true): number {
+ if (explore && Math.random() < this.epsilon) {
+ return Math.floor(Math.random() * this.numActions);
+ }
+
+ const stateKey = this.hashState(state);
+ const entry = this.qTable.get(stateKey);
+
+ if (!entry) {
+ return Math.floor(Math.random() * this.numActions);
+ }
+
+ return this.argmax(entry.qValues);
+ }
+
+ /**
+ * Get Q-values for a state
+ */
+ getQValues(state: Float32Array): Float32Array {
+ const stateKey = this.hashState(state);
+ const entry = this.qTable.get(stateKey);
+
+ if (!entry) {
+ return new Float32Array(this.numActions);
+ }
+
+ return new Float32Array(entry.qValues);
+ }
+
+ /**
+ * Get statistics
+ */
+ getStats(): Record<string, number> {
+ return {
+ updateCount: this.updateCount,
+ qTableSize: this.qTable.size,
+ epsilon: this.epsilon,
+ avgTDError: this.avgTDError,
+ stepCount: this.stepCount,
+ };
+ }
+
+ /**
+ * Reset Q-table
+ */
+ reset(): void {
+ this.qTable.clear();
+ this.traces.clear();
+ this.epsilon = this.config.explorationInitial;
+ this.stepCount = 0;
+ this.updateCount = 0;
+ this.avgTDError = 0;
+ }
+
+ // ==========================================================================
+ // Private Methods
+ // ==========================================================================
+
+ private hashState(state: Float32Array): string {
+ // Discretize state by binning values
+ const bins = 10;
+ const parts: number[] = [];
+
+ // Use first 8 dimensions for hashing
+ for (let i = 0; i < Math.min(8, state.length); i++) {
+ const normalized = (state[i] + 1) / 2; // Assume [-1, 1] range
+ const bin = Math.floor(Math.max(0, Math.min(bins - 1, normalized * bins)));
+ parts.push(bin);
+ }
+
+ return parts.join(',');
+ }
+
+ private hashAction(action: string): number {
+ let hash = 0;
+ for (let i = 0; i < action.length; i++) {
+ hash = (hash * 31 + action.charCodeAt(i)) % this.numActions;
+ }
+ return hash;
+ }
+
+ private getOrCreateEntry(stateKey: string): QEntry {
+ let entry = this.qTable.get(stateKey);
+
+ if (!entry) {
+ entry = {
+ qValues: new Float32Array(this.numActions),
+ visits: 0,
+ lastUpdate: Date.now(),
+ };
+ this.qTable.set(stateKey, entry);
+ }
+
+ return entry;
+ }
+
+ private updateTrace(stateKey: string, action: number): void {
+ // Decay all existing traces
+ for (const [key, trace] of this.traces) {
+ for (let a = 0; a < this.numActions; a++) {
+ trace[a] *= this.config.gamma * this.config.traceDecay;
+ }
+
+ // Remove near-zero traces
+ const maxTrace = Math.max(...trace);
+ if (maxTrace < 0.001) {
+ this.traces.delete(key);
+ }
+ }
+
+ // Set trace for current state-action
+ let trace = this.traces.get(stateKey);
+ if (!trace) {
+ trace = new Float32Array(this.numActions);
+ this.traces.set(stateKey, trace);
+ }
+ trace[action] = 1.0;
+ }
+
+ private updateWithTraces(tdError: number): void {
+ const lr = this.config.learningRate;
+
+ for (const [stateKey, trace] of this.traces) {
+ const entry = this.qTable.get(stateKey);
+ if (entry) {
+ for (let a = 0; a < this.numActions; a++) {
+ entry.qValues[a] += lr * tdError * trace[a];
+ }
+ entry.visits++;
+ entry.lastUpdate = Date.now();
+ }
+ }
+ }
+
+ private pruneQTable(): void {
+ // Remove least recently used states
+ const entries = Array.from(this.qTable.entries())
+ .sort((a, b) => a[1].lastUpdate - b[1].lastUpdate);
+
+ const toRemove = entries.length - Math.floor(this.config.maxStates * 0.8);
+ for (let i = 0; i < toRemove; i++) {
+ this.qTable.delete(entries[i][0]);
+ }
+ }
+
+ private argmax(values: Float32Array): number {
+ let maxIdx = 0;
+ let maxVal = values[0];
+ for (let i = 1; i < values.length; i++) {
+ if (values[i] > maxVal) {
+ maxVal = values[i];
+ maxIdx = i;
+ }
+ }
+ return maxIdx;
+ }
+}
+
+/**
+ * Factory function
+ */
+export function createQLearning(config?: Partial<QLearningConfig>): QLearning {
+ return new QLearning(config);
+}