🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
checkpoint: File edit:
rUv2604030cc8a5
diff --git a/.agentic-flow/intelligence.json b/.agentic-flow/intelligence.jsonindex eb119b7071..2617f8be47 100644--- a/.agentic-flow/intelligence.json+++ b/.agentic-flow/intelligence.json@@ -146,7 +146,7 @@"test-engineer": 0.6},"command:": {- "success": 0.999823035654572+ "success": 0.9999059538913014}},"sequences": {},@@ -456,8 +456,8 @@},"errorPatterns": [],"metrics": {- "totalRoutes": 153,- "successfulRoutes": 153,+ "totalRoutes": 163,+ "successfulRoutes": 163,"routingHistory": [{"timestamp": "2026-01-04T15:51:13.039Z",@@ -884,6 +884,30 @@"task": "edit:","agent": "unknown","success": true+ },+ {+ "timestamp": "2026-01-04T17:35:28.457Z",+ "task": "edit:",+ "agent": "unknown",+ "success": true+ },+ {+ "timestamp": "2026-01-04T17:35:37.509Z",+ "task": "edit:",+ "agent": "unknown",+ "success": true+ },+ {+ "timestamp": "2026-01-04T17:35:51.985Z",+ "task": "edit:",+ "agent": "unknown",+ "success": true+ },+ {+ "timestamp": "2026-01-04T17:36:52.479Z",+ "task": "edit:",+ "agent": "unknown",+ "success": true}]},diff --git a/.claude/helpers/swarm-monitor.sh b/.claude/helpers/swarm-monitor.shindex c38419b9c3..af47a91f8e 100755--- a/.claude/helpers/swarm-monitor.sh+++ b/.claude/helpers/swarm-monitor.sh@@ -75,7 +75,7 @@ update_activity_metrics() {# Update active agents in metricsif [ -f "$UPDATE_SCRIPT" ]; then- "$UPDATE_SCRIPT" agents "$agent_count" >/dev/null 2>&1+ "$UPDATE_SCRIPT" agent "$agent_count" >/dev/null 2>&1fi# Update integration status based on activitydiff --git a/.claude/settings.json b/.claude/settings.jsonindex 32f8b9326f..35e595cc29 100644--- a/.claude/settings.json+++ b/.claude/settings.json@@ -116,6 +116,11 @@"type": "command","timeout": 3000,"command": "npx agentic-flow@alpha hooks post-command \"$TOOL_INPUT_command\" --success --learn --performance-metrics --v3-mode 2>/dev/null || true"+ },+ {+ "type": "command",+ "timeout": 2000,+ "command": ".claude/helpers/swarm-monitor.sh check 2>/dev/null || true"}]},@@ -127,6 +132,11 @@"timeout": 2000,"command": "npx agentic-flow@alpha hooks metrics --agent-performance --swarm-efficiency 2>/dev/null || true"},+ {+ "type": "command",+ "timeout": 3000,+ "command": ".claude/helpers/swarm-monitor.sh check 2>/dev/null || true"+ },{"type": "command","timeout": 4000,@@ -229,6 +239,7 @@"v3ConfigValidator": ".claude/helpers/validate-v3-config.sh","v3QuickStatus": ".claude/helpers/v3-quick-status.sh","v3CheckpointManager": ".claude/helpers/checkpoint-manager.sh",+ "v3SwarmMonitor": ".claude/helpers/swarm-monitor.sh","v3Master": ".claude/helpers/v3.sh"}},diff --git a/v3/benchmarks/attention/memory-efficiency.bench.ts b/v3/benchmarks/attention/memory-efficiency.bench.tsnew file mode 100644index 0000000000..280f7a4023--- /dev/null+++ b/v3/benchmarks/attention/memory-efficiency.bench.ts@@ -0,0 +1,569 @@+/**+ * Attention Memory Efficiency Benchmark+ *+ * Target: 50-75% memory reduction+ *+ * Measures memory efficiency of different attention implementations+ * and optimization strategies.+ */++import { benchmark, BenchmarkRunner, formatTime, formatBytes } from '../framework/benchmark.js';++// ============================================================================+// Memory Tracking+// ============================================================================++interface MemorySnapshot {+ heapUsed: number;+ heapTotal: number;+ external: number;+ arrayBuffers: number;+ rss: number;+}++function takeMemorySnapshot(): MemorySnapshot {+ const mem = process.memoryUsage();+ return {+ heapUsed: mem.heapUsed,+ heapTotal: mem.heapTotal,+ external: mem.external,+ arrayBuffers: mem.arrayBuffers,+ rss: mem.rss,+ };+}++function calculateMemoryDelta(before: MemorySnapshot, after: MemorySnapshot): number {+ return after.heapUsed - before.heapUsed;+}++// ============================================================================+// Attention Implementations for Memory Testing+// ============================================================================++/**+ * Standard attention - stores full attention matrix+ */+function standardAttention(+ query: Float32Array,+ key: Float32Array,+ value: Float32Array,+ seqLength: number,+ headDim: number+): { output: Float32Array; attentionMatrix: Float32Array } {+ const scale = 1 / Math.sqrt(headDim);++ // Full attention matrix - O(n^2) memory+ const attentionMatrix = new Float32Array(seqLength * seqLength);++ // Compute scores+ for (let i = 0; i < seqLength; i++) {+ for (let j = 0; j < seqLength; j++) {+ let dot = 0;+ for (let k = 0; k < headDim; k++) {+ dot += query[i * headDim + k]! * key[j * headDim + k]!;+ }+ attentionMatrix[i * seqLength + j] = dot * scale;+ }+ }++ // Softmax+ for (let i = 0; i < seqLength; i++) {+ let max = -Infinity;+ for (let j = 0; j < seqLength; j++) {+ max = Math.max(max, attentionMatrix[i * seqLength + j]!);+ }++ let sum = 0;+ for (let j = 0; j < seqLength; j++) {+ const exp = Math.exp(attentionMatrix[i * seqLength + j]! - max);+ attentionMatrix[i * seqLength + j] = exp;+ sum += exp;+ }++ for (let j = 0; j < seqLength; j++) {+ attentionMatrix[i * seqLength + j]! /= sum;+ }+ }++ // Output+ const output = new Float32Array(seqLength * headDim);+ for (let i = 0; i < seqLength; i++) {+ for (let k = 0; k < headDim; k++) {+ let sum = 0;+ for (let j = 0; j < seqLength; j++) {+ sum += attentionMatrix[i * seqLength + j]! * value[j * headDim + k]!;+ }+ output[i * headDim + k] = sum;+ }+ }++ return { output, attentionMatrix };+}++/**+ * Memory-efficient attention - no full matrix storage+ */+function memoryEfficientAttention(+ query: Float32Array,+ key: Float32Array,+ value: Float32Array,+ seqLength: number,+ headDim: number+): { output: Float32Array } {+ const scale = 1 / Math.sqrt(headDim);+ const output = new Float32Array(seqLength * headDim);++ // Process row by row - O(n) memory for scores+ const rowScores = new Float32Array(seqLength);++ for (let i = 0; i < seqLength; i++) {+ // Compute scores for this row+ let max = -Infinity;+ for (let j = 0; j < seqLength; j++) {+ let dot = 0;+ for (let k = 0; k < headDim; k++) {+ dot += query[i * headDim + k]! * key[j * headDim + k]!;+ }+ rowScores[j] = dot * scale;+ max = Math.max(max, rowScores[j]!);+ }++ // Softmax+ let sum = 0;+ for (let j = 0; j < seqLength; j++) {+ rowScores[j] = Math.exp(rowScores[j]! - max);+ sum += rowScores[j]!;+ }+ for (let j = 0; j < seqLength; j++) {+ rowScores[j]! /= sum;+ }++ // Compute output for this row+ for (let k = 0; k < headDim; k++) {+ let val = 0;+ for (let j = 0; j < seqLength; j++) {+ val += rowScores[j]! * value[j * headDim + k]!;+ }+ output[i * headDim + k] = val;+ }+ }++ return { output };+}++/**+ * Chunked attention - process in blocks+ */+function chunkedAttention(+ query: Float32Array,+ key: Float32Array,+ value: Float32Array,+ seqLength: number,+ headDim: number,+ chunkSize: number = 64+): { output: Float32Array } {+ const scale = 1 / Math.sqrt(headDim);+ const output = new Float32Array(seqLength * headDim);+ const numChunks = Math.ceil(seqLength / chunkSize);++ // Chunk buffer - O(chunkSize^2) memory+ const chunkScores = new Float32Array(chunkSize * seqLength);+ const rowMax = new Float32Array(chunkSize).fill(-Infinity);+ const rowSum = new Float32Array(chunkSize).fill(0);++ for (let ci = 0; ci < numChunks; ci++) {+ const iStart = ci * chunkSize;+ const iEnd = Math.min(iStart + chunkSize, seqLength);+ const iSize = iEnd - iStart;++ // Reset accumulators+ rowMax.fill(-Infinity);+ rowSum.fill(0);+ output.fill(0, iStart * headDim, iEnd * headDim);++ for (let cj = 0; cj < numChunks; cj++) {+ const jStart = cj * chunkSize;+ const jEnd = Math.min(jStart + chunkSize, seqLength);+ const jSize = jEnd - jStart;++ // Compute chunk scores+ for (let i = 0; i < iSize; i++) {+ for (let j = 0; j < jSize; j++) {+ let dot = 0;+ for (let k = 0; k < headDim; k++) {+ dot += query[(iStart + i) * headDim + k]! * key[(jStart + j) * headDim + k]!;+ }+ chunkScores[i * seqLength + jStart + j] = dot * scale;+ }+ }++ // Online softmax update+ for (let i = 0; i < iSize; i++) {+ const prevMax = rowMax[i]!;++ // Find new max+ for (let j = 0; j < jSize; j++) {+ rowMax[i] = Math.max(rowMax[i]!, chunkScores[i * seqLength + jStart + j]!);+ }++ // Rescale previous+ if (prevMax !== -Infinity && prevMax !== rowMax[i]) {+ const rescale = Math.exp(prevMax - rowMax[i]!);+ rowSum[i]! *= rescale;+ for (let k = 0; k < headDim; k++) {+ output[(iStart + i) * headDim + k]! *= rescale;+ }+ }++ // Add new exponentials+ for (let j = 0; j < jSize; j++) {+ const exp = Math.exp(chunkScores[i * seqLength + jStart + j]! - rowMax[i]!);+ chunkScores[i * seqLength + jStart + j] = exp;+ rowSum[i]! += exp;+ }++ // Accumulate output+ for (let k = 0; k < headDim; k++) {+ for (let j = 0; j < jSize; j++) {+ output[(iStart + i) * headDim + k]! +=+ chunkScores[i * seqLength + jStart + j]! * value[(jStart + j) * headDim + k]!;+ }+ }+ }+ }++ // Final normalization+ for (let i = 0; i < iSize; i++) {+ for (let k = 0; k < headDim; k++) {+ output[(iStart + i) * headDim + k]! /= rowSum[i]!;+ }+ }+ }++ return { output };+}++// ============================================================================+// Helper Functions+// ============================================================================++function generateRandomTensor(size: number): Float32Array {+ const tensor = new Float32Array(size);+ for (let i = 0; i < size; i++) {+ tensor[i] = Math.random() * 2 - 1;+ }+ return tensor;+}++// ============================================================================+// Benchmark Suite+// ============================================================================++export async function runMemoryEfficiencyBenchmarks(): Promise<void> {+ const runner = new BenchmarkRunner('Attention Memory Efficiency');++ console.log('\n--- Attention Memory Efficiency Benchmarks ---\n');++ // Test configurations+ const seqLengths = [128, 256, 512, 1024];+ const headDim = 64;++ // Memory scaling comparison+ console.log('--- Memory Scaling by Sequence Length ---\n');++ const memoryResults: Array<{+ seqLength: number;+ standard: number;+ efficient: number;+ chunked: number;+ reduction: number;+ }> = [];++ for (const seqLength of seqLengths) {+ console.log(`Sequence Length: ${seqLength}`);++ const size = seqLength * headDim;+ const query = generateRandomTensor(size);+ const key = generateRandomTensor(size);+ const value = generateRandomTensor(size);++ // Standard attention memory+ if (typeof global.gc === 'function') global.gc();+ const standardBefore = takeMemorySnapshot();+ const standardResult = standardAttention(query, key, value, seqLength, headDim);+ const standardAfter = takeMemorySnapshot();+ const standardMem = calculateMemoryDelta(standardBefore, standardAfter);+ void standardResult;++ // Memory-efficient attention+ if (typeof global.gc === 'function') global.gc();+ const efficientBefore = takeMemorySnapshot();+ const efficientResult = memoryEfficientAttention(query, key, value, seqLength, headDim);+ const efficientAfter = takeMemorySnapshot();+ const efficientMem = calculateMemoryDelta(efficientBefore, efficientAfter);+ void efficientResult;++ // Chunked attention+ if (typeof global.gc === 'function') global.gc();+ const chunkedBefore = takeMemorySnapshot();+ const chunkedResult = chunkedAttention(query, key, value, seqLength, headDim);+ const chunkedAfter = takeMemorySnapshot();+ const chunkedMem = calculateMemoryDelta(chunkedBefore, chunkedAfter);+ void chunkedResult;++ const reduction = ((standardMem - efficientMem) / standardMem) * 100;++ memoryResults.push({+ seqLength,+ standard: standardMem,+ efficient: efficientMem,+ chunked: chunkedMem,+ reduction,+ });++ console.log(` Standard: ${formatBytes(standardMem)}`);+ console.log(` Efficient: ${formatBytes(efficientMem)}`);+ console.log(` Chunked: ${formatBytes(chunkedMem)}`);+ console.log(` Reduction: ${reduction.toFixed(1)}%`);+ console.log('');+ }++ // Theoretical memory comparison+ console.log('--- Theoretical Memory Analysis ---\n');++ for (const seqLength of seqLengths) {+ const bytesPerFloat = 4;++ // Standard: stores full n x n attention matrix+ const standardTheory = seqLength * seqLength * bytesPerFloat;++ // Efficient: stores only one row at a time+ const efficientTheory = seqLength * bytesPerFloat;++ // Chunked: stores chunk x n scores+ const chunkSize = 64;+ const chunkedTheory = chunkSize * seqLength * bytesPerFloat;++ console.log(`Seq ${seqLength}:`);+ console.log(` Standard: ${formatBytes(standardTheory)} (n^2)`);+ console.log(` Efficient: ${formatBytes(efficientTheory)} (n)`);+ console.log(` Chunked: ${formatBytes(chunkedTheory)} (chunk * n)`);+ console.log(` Theoretical reduction: ${((1 - efficientTheory / standardTheory) * 100).toFixed(1)}%`);+ console.log('');+ }++ // Performance vs Memory tradeoff+ console.log('--- Performance vs Memory Tradeoff ---\n');++ const tradeoffConfig = { seqLength: 512, headDim: 64 };+ const size = tradeoffConfig.seqLength * tradeoffConfig.headDim;+ const q = generateRandomTensor(size);+ const k = generateRandomTensor(size);+ const v = generateRandomTensor(size);++ // Standard performance+ const standardPerfResult = await runner.run(+ 'standard-attention-perf',+ async () => {+ standardAttention(q, k, v, tradeoffConfig.seqLength, tradeoffConfig.headDim);+ },+ { iterations: 20 }+ );++ console.log(`Standard Performance: ${formatTime(standardPerfResult.mean)}`);++ // Efficient performance+ const efficientPerfResult = await runner.run(+ 'efficient-attention-perf',+ async () => {+ memoryEfficientAttention(q, k, v, tradeoffConfig.seqLength, tradeoffConfig.headDim);+ },+ { iterations: 20 }+ );++ console.log(`Memory-Efficient Performance: ${formatTime(efficientPerfResult.mean)}`);++ // Chunked performance with different chunk sizes+ const chunkSizes = [32, 64, 128, 256];++ for (const chunkSize of chunkSizes) {+ const chunkedPerfResult = await runner.run(+ `chunked-attention-chunk${chunkSize}`,+ async () => {+ chunkedAttention(q, k, v, tradeoffConfig.seqLength, tradeoffConfig.headDim, chunkSize);+ },+ { iterations: 20 }+ );++ console.log(`Chunked (size=${chunkSize}): ${formatTime(chunkedPerfResult.mean)}`);+ }++ // Multi-head memory analysis+ console.log('\n--- Multi-Head Memory Analysis ---\n');++ const numHeads = [4, 8, 16, 32];+ const mhaSeqLength = 256;++ for (const heads of numHeads) {+ const mhaSize = mhaSeqLength * headDim;++ // Standard MHA memory+ const standardMHAMem = mhaSeqLength * mhaSeqLength * 4 * heads; // attention matrices+ const qkvMem = mhaSize * 4 * 3 * heads; // QKV storage++ // GQA memory (shared KV)+ const gqaKVHeads = heads / 4;+ const gqaMem = mhaSeqLength * mhaSeqLength * 4 * heads + // attention matrices (same)+ mhaSize * 4 * heads + // Q storage+ mhaSize * 4 * 2 * gqaKVHeads; // shared KV++ // MQA memory (single KV)+ const mqaMem = mhaSeqLength * mhaSeqLength * 4 * heads + // attention matrices+ mhaSize * 4 * heads + // Q storage+ mhaSize * 4 * 2; // single KV++ console.log(`${heads} heads:`);+ console.log(` Standard MHA: ${formatBytes(standardMHAMem + qkvMem)}`);+ console.log(` GQA (${gqaKVHeads} KV): ${formatBytes(gqaMem)}`);+ console.log(` MQA (1 KV): ${formatBytes(mqaMem)}`);+ console.log(` MQA reduction: ${(((standardMHAMem + qkvMem) - mqaMem) / (standardMHAMem + qkvMem) * 100).toFixed(1)}%`);+ console.log('');+ }++ // Summary+ console.log('--- Summary ---\n');++ console.log('Memory Reduction Achieved:');+ for (const result of memoryResults) {+ const targetMet = result.reduction >= 50;+ console.log(+ ` Seq ${result.seqLength}: ${result.reduction.toFixed(1)}% ${targetMet ? '(TARGET MET)' : ''}`+ );+ }++ console.log('\nPerformance Comparison (seq=512):');+ console.log(` Standard: ${formatTime(standardPerfResult.mean)}`);+ console.log(` Efficient: ${formatTime(efficientPerfResult.mean)}`);++ // Print full results+ runner.printResults();+}++// ============================================================================+// Memory Efficiency Optimization Strategies+// ============================================================================++export const memoryOptimizations = {+ /**+ * Online softmax computation+ */+ onlineSoftmax: {+ description: 'Compute softmax in streaming fashion without storing all values',+ expectedImprovement: 'O(n) instead of O(n^2) for softmax',+ implementation: `+ class OnlineSoftmax {+ private max = -Infinity;+ private sum = 0;+ private count = 0;++ add(value: number): void {+ if (value > this.max) {+ this.sum *= Math.exp(this.max - value);+ this.max = value;+ }+ this.sum += Math.exp(value - this.max);+ this.count++;+ }++ normalize(value: number): number {+ return Math.exp(value - this.max) / this.sum;+ }+ }+ `,+ },++ /**+ * Gradient checkpointing+ */+ gradientCheckpointing: {+ description: 'Recompute attention during backward pass instead of storing',+ expectedImprovement: 'O(1) memory for activations',+ implementation: `+ function checkpointedAttention(q, k, v) {+ const output = computeAttention(q, k, v);++ function backward(gradOutput) {+ // Recompute attention weights during backward+ const attnWeights = recomputeAttention(q, k);+ return computeGradients(gradOutput, attnWeights, q, k, v);+ }++ return { output, backward };+ }+ `,+ },++ /**+ * Sparse attention patterns+ */+ sparseAttention: {+ description: 'Only compute attention for relevant positions',+ expectedImprovement: 'O(n * k) instead of O(n^2) where k << n',+ implementation: `+ function sparseAttention(q, k, v, pattern: 'local' | 'strided' | 'block') {+ const sparseMask = generateSparsePattern(q.length, pattern);+ return computeAttentionWithMask(q, k, v, sparseMask);+ }+ `,+ },++ /**+ * Quantization+ */+ quantization: {+ description: 'Use lower precision for attention computation',+ expectedImprovement: '2-4x memory reduction',+ implementation: `+ function quantizedAttention(q, k, v) {+ // Quantize to int8+ const qInt8 = quantizeToInt8(q);+ const kInt8 = quantizeToInt8(k);++ // Compute in int8+ const scores = computeInt8Attention(qInt8, kInt8);++ // Dequantize for output+ return dequantizeAndApply(scores, v);+ }+ `,+ },++ /**+ * Memory pooling+ */+ memoryPooling: {+ description: 'Reuse memory buffers across forward passes',+ expectedImprovement: 'Eliminates allocation overhead',+ implementation: `+ class AttentionMemoryPool {+ private scoreBuffer: Float32Array;+ private outputBuffer: Float32Array;++ forward(q, k, v) {+ // Reuse pre-allocated buffers+ computeScores(q, k, this.scoreBuffer);+ applySoftmax(this.scoreBuffer);+ computeOutput(this.scoreBuffer, v, this.outputBuffer);+ return this.outputBuffer;+ }+ }+ `,+ },+};++// Run if executed directly+if (import.meta.url === `file://${process.argv[1]}`) {+ runMemoryEfficiencyBenchmarks().catch(console.error);+}++export default runMemoryEfficiencyBenchmarks;diff --git a/v3/neural/algorithms/q-learning.ts b/v3/neural/algorithms/q-learning.tsnew file mode 100644index 0000000000..956dc22a67--- /dev/null+++ b/v3/neural/algorithms/q-learning.ts@@ -0,0 +1,333 @@+/**+ * Tabular Q-Learning+ *+ * Classic Q-learning algorithm with:+ * - Epsilon-greedy exploration+ * - State hashing for continuous states+ * - Eligibility traces (optional)+ * - Experience replay+ *+ * Suitable for smaller state spaces or discretized environments.+ * Performance Target: <1ms per update+ */++import type { Trajectory, RLConfig } from '../types.js';++/**+ * Q-Learning configuration+ */+export interface QLearningConfig extends RLConfig {+ algorithm: 'q-learning';+ explorationInitial: number;+ explorationFinal: number;+ explorationDecay: number;+ maxStates: number;+ useEligibilityTraces: boolean;+ traceDecay: number;+}++/**+ * Default Q-Learning configuration+ */+export const DEFAULT_QLEARNING_CONFIG: QLearningConfig = {+ algorithm: 'q-learning',+ learningRate: 0.1,+ gamma: 0.99,+ entropyCoef: 0,+ valueLossCoef: 1,+ maxGradNorm: 1,+ epochs: 1,+ miniBatchSize: 1,+ explorationInitial: 1.0,+ explorationFinal: 0.01,+ explorationDecay: 10000,+ maxStates: 10000,+ useEligibilityTraces: false,+ traceDecay: 0.9,+};++/**+ * Q-table entry+ */+interface QEntry {+ qValues: Float32Array;+ visits: number;+ lastUpdate: number;+}++/**+ * Q-Learning Algorithm Implementation+ */+export class QLearning {+ private config: QLearningConfig;++ // Q-table+ private qTable: Map<string, QEntry> = new Map();++ // Exploration+ private epsilon: number;+ private stepCount = 0;++ // Number of actions+ private numActions = 4;++ // Eligibility traces+ private traces: Map<string, Float32Array> = new Map();++ // Statistics+ private updateCount = 0;+ private avgTDError = 0;++ constructor(config: Partial<QLearningConfig> = {}) {+ this.config = { ...DEFAULT_QLEARNING_CONFIG, ...config };+ this.epsilon = this.config.explorationInitial;+ }++ /**+ * Update Q-values from trajectory+ */+ update(trajectory: Trajectory): { tdError: number } {+ const startTime = performance.now();++ if (trajectory.steps.length === 0) {+ return { tdError: 0 };+ }++ let totalTDError = 0;++ // Reset eligibility traces for new trajectory+ if (this.config.useEligibilityTraces) {+ this.traces.clear();+ }++ for (let i = 0; i < trajectory.steps.length; i++) {+ const step = trajectory.steps[i];+ const stateKey = this.hashState(step.stateBefore);+ const action = this.hashAction(step.action);++ // Get or create Q-entry+ const qEntry = this.getOrCreateEntry(stateKey);++ // Current Q-value+ const currentQ = qEntry.qValues[action];++ // Compute target Q-value+ let targetQ: number;+ if (i === trajectory.steps.length - 1) {+ // Terminal state+ targetQ = step.reward;+ } else {+ const nextStateKey = this.hashState(step.stateAfter);+ const nextEntry = this.getOrCreateEntry(nextStateKey);+ const maxNextQ = Math.max(...nextEntry.qValues);+ targetQ = step.reward + this.config.gamma * maxNextQ;+ }++ // TD error+ const tdError = targetQ - currentQ;+ totalTDError += Math.abs(tdError);++ if (this.config.useEligibilityTraces) {+ // Update eligibility trace+ this.updateTrace(stateKey, action);++ // Update all states with traces+ this.updateWithTraces(tdError);+ } else {+ // Simple Q-learning update+ qEntry.qValues[action] += this.config.learningRate * tdError;+ qEntry.visits++;+ qEntry.lastUpdate = Date.now();+ }+ }++ // Decay exploration+ this.stepCount += trajectory.steps.length;+ this.epsilon = Math.max(+ this.config.explorationFinal,+ this.config.explorationInitial - this.stepCount / this.config.explorationDecay+ );++ // Prune Q-table if too large+ if (this.qTable.size > this.config.maxStates) {+ this.pruneQTable();+ }++ this.updateCount++;+ this.avgTDError = totalTDError / trajectory.steps.length;++ const elapsed = performance.now() - startTime;+ if (elapsed > 1) {+ console.warn(`Q-learning update exceeded target: ${elapsed.toFixed(2)}ms > 1ms`);+ }++ return { tdError: this.avgTDError };+ }++ /**+ * Get action using epsilon-greedy policy+ */+ getAction(state: Float32Array, explore: boolean = true): number {+ if (explore && Math.random() < this.epsilon) {+ return Math.floor(Math.random() * this.numActions);+ }++ const stateKey = this.hashState(state);+ const entry = this.qTable.get(stateKey);++ if (!entry) {+ return Math.floor(Math.random() * this.numActions);+ }++ return this.argmax(entry.qValues);+ }++ /**+ * Get Q-values for a state+ */+ getQValues(state: Float32Array): Float32Array {+ const stateKey = this.hashState(state);+ const entry = this.qTable.get(stateKey);++ if (!entry) {+ return new Float32Array(this.numActions);+ }++ return new Float32Array(entry.qValues);+ }++ /**+ * Get statistics+ */+ getStats(): Record<string, number> {+ return {+ updateCount: this.updateCount,+ qTableSize: this.qTable.size,+ epsilon: this.epsilon,+ avgTDError: this.avgTDError,+ stepCount: this.stepCount,+ };+ }++ /**+ * Reset Q-table+ */+ reset(): void {+ this.qTable.clear();+ this.traces.clear();+ this.epsilon = this.config.explorationInitial;+ this.stepCount = 0;+ this.updateCount = 0;+ this.avgTDError = 0;+ }++ // ==========================================================================+ // Private Methods+ // ==========================================================================++ private hashState(state: Float32Array): string {+ // Discretize state by binning values+ const bins = 10;+ const parts: number[] = [];++ // Use first 8 dimensions for hashing+ for (let i = 0; i < Math.min(8, state.length); i++) {+ const normalized = (state[i] + 1) / 2; // Assume [-1, 1] range+ const bin = Math.floor(Math.max(0, Math.min(bins - 1, normalized * bins)));+ parts.push(bin);+ }++ return parts.join(',');+ }++ private hashAction(action: string): number {+ let hash = 0;+ for (let i = 0; i < action.length; i++) {+ hash = (hash * 31 + action.charCodeAt(i)) % this.numActions;+ }+ return hash;+ }++ private getOrCreateEntry(stateKey: string): QEntry {+ let entry = this.qTable.get(stateKey);++ if (!entry) {+ entry = {+ qValues: new Float32Array(this.numActions),+ visits: 0,+ lastUpdate: Date.now(),+ };+ this.qTable.set(stateKey, entry);+ }++ return entry;+ }++ private updateTrace(stateKey: string, action: number): void {+ // Decay all existing traces+ for (const [key, trace] of this.traces) {+ for (let a = 0; a < this.numActions; a++) {+ trace[a] *= this.config.gamma * this.config.traceDecay;+ }++ // Remove near-zero traces+ const maxTrace = Math.max(...trace);+ if (maxTrace < 0.001) {+ this.traces.delete(key);+ }+ }++ // Set trace for current state-action+ let trace = this.traces.get(stateKey);+ if (!trace) {+ trace = new Float32Array(this.numActions);+ this.traces.set(stateKey, trace);+ }+ trace[action] = 1.0;+ }++ private updateWithTraces(tdError: number): void {+ const lr = this.config.learningRate;++ for (const [stateKey, trace] of this.traces) {+ const entry = this.qTable.get(stateKey);+ if (entry) {+ for (let a = 0; a < this.numActions; a++) {+ entry.qValues[a] += lr * tdError * trace[a];+ }+ entry.visits++;+ entry.lastUpdate = Date.now();+ }+ }+ }++ private pruneQTable(): void {+ // Remove least recently used states+ const entries = Array.from(this.qTable.entries())+ .sort((a, b) => a[1].lastUpdate - b[1].lastUpdate);++ const toRemove = entries.length - Math.floor(this.config.maxStates * 0.8);+ for (let i = 0; i < toRemove; i++) {+ this.qTable.delete(entries[i][0]);+ }+ }++ private argmax(values: Float32Array): number {+ let maxIdx = 0;+ let maxVal = values[0];+ for (let i = 1; i < values.length; i++) {+ if (values[i] > maxVal) {+ maxVal = values[i];+ maxIdx = i;+ }+ }+ return maxIdx;+ }+}++/**+ * Factory function+ */+export function createQLearning(config?: Partial<QLearningConfig>): QLearning {+ return new QLearning(config);+}