Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

checkpoint: File edit:

rUvc61c1b030ae1
diff --git a/.claude/statusline.sh b/.claude/statusline.sh
index d6d330299a..a0cc65470d 100755
--- a/.claude/statusline.sh
+++ b/.claude/statusline.sh
@@ -86,6 +86,38 @@ if [ -f "package.json" ]; then
fi
fi
+# REAL-TIME SWARM DETECTION
+# Count active agentic-flow processes
+ACTIVE_PROCESSES=$(ps aux 2>/dev/null | grep -E "(agentic-flow|claude-flow)" | grep -v grep | wc -l)
+
+# Detect running swarm agents dynamically
+if [ "$ACTIVE_PROCESSES" -gt 0 ]; then
+ # Update active agents count from real processes
+ # Each agentic-flow process might represent an agent or coordination process
+ DYNAMIC_AGENTS=$(ps aux 2>/dev/null | grep -E "agentic-flow.*agent" | grep -v grep | wc -l)
+
+ # If we have agentic-flow processes but no specific agents, use a heuristic
+ if [ "$DYNAMIC_AGENTS" -eq 0 ] && [ "$ACTIVE_PROCESSES" -gt 0 ]; then
+ # Assume some processes are coordination/management, others are agents
+ DYNAMIC_AGENTS=$((ACTIVE_PROCESSES / 2))
+ if [ "$DYNAMIC_AGENTS" -eq 0 ] && [ "$ACTIVE_PROCESSES" -gt 0 ]; then
+ DYNAMIC_AGENTS=1
+ fi
+ fi
+
+ # Override static value with dynamic detection
+ AGENTS_ACTIVE="$DYNAMIC_AGENTS"
+
+ # Update integration status to show active when processes are running
+ INTEGRATION_STATUS="●"
+fi
+
+# Check for MCP server processes
+MCP_ACTIVE=$(ps aux 2>/dev/null | grep -E "mcp.*start" | grep -v grep | wc -l)
+if [ "$MCP_ACTIVE" -gt 0 ]; then
+ INTEGRATION_STATUS="●"
+fi
+
# Colorful domain status indicators
COMPLETED_DOMAIN="${BRIGHT_GREEN}●${RESET}"
PENDING_DOMAIN="${DIM}○${RESET}"
diff --git a/v3/__tests__/unit/swarm/unified-coordinator.test.ts b/v3/__tests__/unit/swarm/unified-coordinator.test.ts
new file mode 100644
index 0000000000..47c6fdf6d3
--- /dev/null
+++ b/v3/__tests__/unit/swarm/unified-coordinator.test.ts
@@ -0,0 +1,952 @@
+/**
+ * V3 Claude-Flow Unified Coordinator Unit Tests
+ *
+ * London School TDD - Behavior Verification
+ * Tests 15-agent swarm coordination (ADR-003)
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { createMock, type MockedInterface, InteractionRecorder } from '../../helpers/create-mock';
+import { createSwarmTestInstance, type V3AgentType, type SwarmTask } from '../../helpers/swarm-instance';
+import { swarmConfigs, createSwarmConfig } from '../../fixtures/configurations';
+import { agentConfigs, create15AgentSwarmConfig } from '../../fixtures/agents';
+
+/**
+ * Unified coordinator interface (to be implemented per ADR-003)
+ */
+interface IUnifiedCoordinator {
+ initialize(config: SwarmConfig): Promise<void>;
+ addAgent(agent: AgentInfo): Promise<void>;
+ removeAgent(agentId: string): Promise<void>;
+ coordinate(task: CoordinationTask): Promise<CoordinationResult>;
+ shutdown(): Promise<void>;
+ getTopology(): SwarmTopology;
+ getAgentStatus(agentId: string): AgentStatus | null;
+}
+
+/**
+ * Task distributor interface (collaborator)
+ */
+interface ITaskDistributor {
+ distribute(task: CoordinationTask, agents: AgentInfo[]): Promise<TaskAssignment[]>;
+ rebalance(assignments: TaskAssignment[]): Promise<TaskAssignment[]>;
+}
+
+/**
+ * Consensus manager interface (collaborator)
+ */
+interface IConsensusManager {
+ propose(proposal: Proposal): Promise<boolean>;
+ getLeader(): string | null;
+ isLeader(agentId: string): boolean;
+}
+
+/**
+ * Communication channel interface (collaborator)
+ */
+interface ICommunicationChannel {
+ broadcast(message: SwarmMessage): Promise<void>;
+ send(agentId: string, message: SwarmMessage): Promise<void>;
+ subscribe(handler: MessageHandler): void;
+}
+
+/**
+ * Event publisher interface (collaborator)
+ */
+interface IEventPublisher {
+ publish(event: DomainEvent): Promise<void>;
+}
+
+interface SwarmConfig {
+ topology: 'hierarchical' | 'mesh' | 'adaptive' | 'hierarchical-mesh';
+ maxAgents: number;
+ consensusProtocol: 'raft' | 'pbft' | 'gossip';
+}
+
+interface AgentInfo {
+ id: string;
+ type: V3AgentType;
+ capabilities: string[];
+ priority: number;
+}
+
+interface CoordinationTask {
+ id: string;
+ type: string;
+ payload: unknown;
+ requiredCapabilities: string[];
+ priority: number;
+}
+
+interface CoordinationResult {
+ taskId: string;
+ success: boolean;
+ results: TaskResult[];
+ duration: number;
+ participatingAgents: string[];
+}
+
+interface TaskAssignment {
+ agentId: string;
+ taskId: string;
+ subtask: unknown;
+}
+
+interface TaskResult {
+ agentId: string;
+ success: boolean;
+ output?: unknown;
+ error?: Error;
+}
+
+interface Proposal {
+ type: string;
+ payload: unknown;
+ proposerId: string;
+}
+
+interface SwarmMessage {
+ type: string;
+ payload: unknown;
+ from: string;
+ to: string | 'broadcast';
+}
+
+type MessageHandler = (message: SwarmMessage) => Promise<void>;
+
+type SwarmTopology = 'hierarchical' | 'mesh' | 'adaptive' | 'hierarchical-mesh';
+
+type AgentStatus = 'idle' | 'busy' | 'offline';
+
+interface DomainEvent {
+ type: string;
+ payload: unknown;
+ timestamp: Date;
+}
+
+/**
+ * Unified coordinator implementation for testing
+ */
+class UnifiedCoordinator implements IUnifiedCoordinator {
+ private agents: Map<string, AgentInfo> = new Map();
+ private agentStatus: Map<string, AgentStatus> = new Map();
+ private initialized = false;
+ private config: SwarmConfig | null = null;
+
+ constructor(
+ private readonly distributor: ITaskDistributor,
+ private readonly consensus: IConsensusManager,
+ private readonly channel: ICommunicationChannel,
+ private readonly eventPublisher: IEventPublisher
+ ) {}
+
+ async initialize(config: SwarmConfig): Promise<void> {
+ if (this.initialized) {
+ throw new Error('Coordinator already initialized');
+ }
+
+ this.config = config;
+
+ // Subscribe to messages
+ this.channel.subscribe(async (message) => {
+ await this.handleMessage(message);
+ });
+
+ // Publish initialization event
+ await this.eventPublisher.publish({
+ type: 'CoordinatorInitialized',
+ payload: { topology: config.topology, maxAgents: config.maxAgents },
+ timestamp: new Date(),
+ });
+
+ this.initialized = true;
+ }
+
+ async addAgent(agent: AgentInfo): Promise<void> {
+ if (!this.initialized) {
+ throw new Error('Coordinator not initialized');
+ }
+
+ if (this.agents.size >= this.config!.maxAgents) {
+ throw new Error('Maximum agent limit reached');
+ }
+
+ this.agents.set(agent.id, agent);
+ this.agentStatus.set(agent.id, 'idle');
+
+ // Broadcast agent joined
+ await this.channel.broadcast({
+ type: 'AgentJoined',
+ payload: { agentId: agent.id, type: agent.type },
+ from: 'coordinator',
+ to: 'broadcast',
+ });
+
+ await this.eventPublisher.publish({
+ type: 'AgentAddedToSwarm',
+ payload: { agentId: agent.id, agentType: agent.type },
+ timestamp: new Date(),
+ });
+ }
+
+ async removeAgent(agentId: string): Promise<void> {
+ if (!this.agents.has(agentId)) {
+ throw new Error(`Agent not found: ${agentId}`);
+ }
+
+ this.agents.delete(agentId);
+ this.agentStatus.delete(agentId);
+
+ // Broadcast agent left
+ await this.channel.broadcast({
+ type: 'AgentLeft',
+ payload: { agentId },
+ from: 'coordinator',
+ to: 'broadcast',
+ });
+
+ await this.eventPublisher.publish({
+ type: 'AgentRemovedFromSwarm',
+ payload: { agentId },
+ timestamp: new Date(),
+ });
+ }
+
+ async coordinate(task: CoordinationTask): Promise<CoordinationResult> {
+ if (!this.initialized) {
+ throw new Error('Coordinator not initialized');
+ }
+
+ const startTime = Date.now();
+
+ // Get available agents
+ const availableAgents = this.getAvailableAgents(task.requiredCapabilities);
+
+ if (availableAgents.length === 0) {
+ throw new Error('No available agents with required capabilities');
+ }
+
+ // For hierarchical topology, propose task through consensus
+ if (this.config!.topology.includes('hierarchical')) {
+ const approved = await this.consensus.propose({
+ type: 'TaskCoordination',
+ payload: task,
+ proposerId: 'coordinator',
+ });
+
+ if (!approved) {
+ throw new Error('Task coordination not approved by consensus');
+ }
+ }
+
+ // Distribute task
+ const assignments = await this.distributor.distribute(task, availableAgents);
+
+ // Mark agents as busy
+ for (const assignment of assignments) {
+ this.agentStatus.set(assignment.agentId, 'busy');
+ }
+
+ // Notify agents
+ for (const assignment of assignments) {
+ await this.channel.send(assignment.agentId, {
+ type: 'TaskAssigned',
+ payload: assignment,
+ from: 'coordinator',
+ to: assignment.agentId,
+ });
+ }
+
+ // Simulate task execution results
+ const results: TaskResult[] = assignments.map((a) => ({
+ agentId: a.agentId,
+ success: true,
+ }));
+
+ // Mark agents as idle
+ for (const assignment of assignments) {
+ this.agentStatus.set(assignment.agentId, 'idle');
+ }
+
+ const duration = Date.now() - startTime;
+
+ await this.eventPublisher.publish({
+ type: 'TaskCoordinationCompleted',
+ payload: { taskId: task.id, duration, participatingAgents: assignments.map((a) => a.agentId) },
+ timestamp: new Date(),
+ });
+
+ return {
+ taskId: task.id,
+ success: true,
+ results,
+ duration,
+ participatingAgents: assignments.map((a) => a.agentId),
+ };
+ }
+
+ async shutdown(): Promise<void> {
+ // Notify all agents
+ await this.channel.broadcast({
+ type: 'CoordinatorShutdown',
+ payload: {},
+ from: 'coordinator',
+ to: 'broadcast',
+ });
+
+ // Clear state
+ this.agents.clear();
+ this.agentStatus.clear();
+ this.initialized = false;
+
+ await this.eventPublisher.publish({
+ type: 'CoordinatorShutdown',
+ payload: {},
+ timestamp: new Date(),
+ });
+ }
+
+ getTopology(): SwarmTopology {
+ return this.config?.topology ?? 'mesh';
+ }
+
+ getAgentStatus(agentId: string): AgentStatus | null {
+ return this.agentStatus.get(agentId) ?? null;
+ }
+
+ private getAvailableAgents(requiredCapabilities: string[]): AgentInfo[] {
+ const available: AgentInfo[] = [];
+
+ for (const [agentId, agent] of this.agents) {
+ const status = this.agentStatus.get(agentId);
+
+ if (status === 'idle') {
+ const hasCapabilities = requiredCapabilities.every((cap) =>
+ agent.capabilities.includes(cap)
+ );
+
+ if (hasCapabilities || requiredCapabilities.length === 0) {
+ available.push(agent);
+ }
+ }
+ }
+
+ return available.sort((a, b) => b.priority - a.priority);
+ }
+
+ private async handleMessage(message: SwarmMessage): Promise<void> {
+ // Handle incoming messages
+ if (message.type === 'AgentStatusUpdate') {
+ const { agentId, status } = message.payload as { agentId: string; status: AgentStatus };
+ this.agentStatus.set(agentId, status);
+ }
+ }
+}
+
+describe('UnifiedCoordinator', () => {
+ let mockDistributor: MockedInterface<ITaskDistributor>;
+ let mockConsensus: MockedInterface<IConsensusManager>;
+ let mockChannel: MockedInterface<ICommunicationChannel>;
+ let mockEventPublisher: MockedInterface<IEventPublisher>;
+ let coordinator: UnifiedCoordinator;
+ let interactionRecorder: InteractionRecorder;
+
+ beforeEach(() => {
+ mockDistributor = createMock<ITaskDistributor>();
+ mockConsensus = createMock<IConsensusManager>();
+ mockChannel = createMock<ICommunicationChannel>();
+ mockEventPublisher = createMock<IEventPublisher>();
+ interactionRecorder = new InteractionRecorder();
+
+ // Configure default mock behavior
+ mockDistributor.distribute.mockResolvedValue([]);
+ mockDistributor.rebalance.mockResolvedValue([]);
+
+ mockConsensus.propose.mockResolvedValue(true);
+ mockConsensus.getLeader.mockReturnValue('leader-agent');
+ mockConsensus.isLeader.mockReturnValue(false);
+
+ mockChannel.broadcast.mockResolvedValue(undefined);
+ mockChannel.send.mockResolvedValue(undefined);
+ mockChannel.subscribe.mockReturnValue(undefined);
+
+ mockEventPublisher.publish.mockResolvedValue(undefined);
+
+ // Track interactions
+ interactionRecorder.track('distributor', mockDistributor);
+ interactionRecorder.track('consensus', mockConsensus);
+ interactionRecorder.track('channel', mockChannel);
+ interactionRecorder.track('events', mockEventPublisher);
+
+ coordinator = new UnifiedCoordinator(
+ mockDistributor,
+ mockConsensus,
+ mockChannel,
+ mockEventPublisher
+ );
+ });
+
+ describe('initialize', () => {
+ it('should subscribe to communication channel', async () => {
+ // Given
+ const config: SwarmConfig = {
+ topology: 'hierarchical-mesh',
+ maxAgents: 15,
+ consensusProtocol: 'raft',
+ };
+
+ // When
+ await coordinator.initialize(config);
+
+ // Then
+ expect(mockChannel.subscribe).toHaveBeenCalled();
+ });
+
+ it('should publish initialization event', async () => {
+ // Given
+ const config: SwarmConfig = {
+ topology: 'hierarchical-mesh',
+ maxAgents: 15,
+ consensusProtocol: 'raft',
+ };
+
+ // When
+ await coordinator.initialize(config);
+
+ // Then
+ expect(mockEventPublisher.publish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'CoordinatorInitialized',
+ payload: expect.objectContaining({
+ topology: 'hierarchical-mesh',
+ maxAgents: 15,
+ }),
+ })
+ );
+ });
+
+ it('should throw error if already initialized', async () => {
+ // Given
+ const config: SwarmConfig = {
+ topology: 'mesh',
+ maxAgents: 10,
+ consensusProtocol: 'gossip',
+ };
+ await coordinator.initialize(config);
+
+ // When/Then
+ await expect(coordinator.initialize(config)).rejects.toThrow(
+ 'Coordinator already initialized'
+ );
+ });
+ });
+
+ describe('addAgent', () => {
+ beforeEach(async () => {
+ await coordinator.initialize({
+ topology: 'hierarchical-mesh',
+ maxAgents: 15,
+ consensusProtocol: 'raft',
+ });
+ });
+
+ it('should throw error if not initialized', async () => {
+ // Given
+ const uninitializedCoordinator = new UnifiedCoordinator(
+ mockDistributor,
+ mockConsensus,
+ mockChannel,
+ mockEventPublisher
+ );
+
+ const agent: AgentInfo = {
+ id: 'agent-1',
+ type: 'coder',
+ capabilities: ['coding'],
+ priority: 50,
+ };
+
+ // When/Then
+ await expect(uninitializedCoordinator.addAgent(agent)).rejects.toThrow(
+ 'Coordinator not initialized'
+ );
+ });
+
+ it('should broadcast agent joined message', async () => {
+ // Given
+ const agent: AgentInfo = {
+ id: 'agent-1',
+ type: 'queen-coordinator',
+ capabilities: ['orchestration'],
+ priority: 100,
+ };
+
+ // When
+ await coordinator.addAgent(agent);
+
+ // Then
+ expect(mockChannel.broadcast).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'AgentJoined',
+ payload: expect.objectContaining({
+ agentId: 'agent-1',
+ }),
+ })
+ );
+ });
+
+ it('should publish AgentAddedToSwarm event', async () => {
+ // Given
+ const agent: AgentInfo = {
+ id: 'agent-1',
+ type: 'security-architect',
+ capabilities: ['security-design'],
+ priority: 90,
+ };
+
+ // When
+ await coordinator.addAgent(agent);
+
+ // Then
+ expect(mockEventPublisher.publish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'AgentAddedToSwarm',
+ payload: expect.objectContaining({
+ agentId: 'agent-1',
+ agentType: 'security-architect',
+ }),
+ })
+ );
+ });
+
+ it('should set agent status to idle', async () => {
+ // Given
+ const agent: AgentInfo = {
+ id: 'agent-1',
+ type: 'coder',
+ capabilities: ['coding'],
+ priority: 50,
+ };
+
+ // When
+ await coordinator.addAgent(agent);
+
+ // Then
+ expect(coordinator.getAgentStatus('agent-1')).toBe('idle');
+ });
+
+ it('should throw error when max agents reached', async () => {
+ // Given - Add 15 agents to reach max
+ for (let i = 0; i < 15; i++) {
+ await coordinator.addAgent({
+ id: `agent-${i}`,
+ type: 'coder',
+ capabilities: ['coding'],
+ priority: 50,
+ });
+ }
+
+ // When/Then
+ await expect(
+ coordinator.addAgent({
+ id: 'agent-16',
+ type: 'coder',
+ capabilities: ['coding'],
+ priority: 50,
+ })
+ ).rejects.toThrow('Maximum agent limit reached');
+ });
+ });
+
+ describe('removeAgent', () => {
+ beforeEach(async () => {
+ await coordinator.initialize({
+ topology: 'hierarchical-mesh',
+ maxAgents: 15,
+ consensusProtocol: 'raft',
+ });
+ await coordinator.addAgent({
+ id: 'agent-1',
+ type: 'coder',
+ capabilities: ['coding'],
+ priority: 50,
+ });
+ });
+
+ it('should throw error for non-existent agent', async () => {
+ // When/Then
+ await expect(coordinator.removeAgent('non-existent')).rejects.toThrow(
+ 'Agent not found: non-existent'
+ );
+ });
+
+ it('should broadcast agent left message', async () => {
+ // When
+ await coordinator.removeAgent('agent-1');
+
+ // Then
+ expect(mockChannel.broadcast).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'AgentLeft',
+ payload: { agentId: 'agent-1' },
+ })
+ );
+ });
+
+ it('should publish AgentRemovedFromSwarm event', async () => {
+ // When
+ await coordinator.removeAgent('agent-1');
+
+ // Then
+ expect(mockEventPublisher.publish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'AgentRemovedFromSwarm',
+ payload: { agentId: 'agent-1' },
+ })
+ );
+ });
+
+ it('should clear agent status', async () => {
+ // When
+ await coordinator.removeAgent('agent-1');
+
+ // Then
+ expect(coordinator.getAgentStatus('agent-1')).toBeNull();
+ });
+ });
+
+ describe('coordinate', () => {
+ beforeEach(async () => {
+ await coordinator.initialize({
+ topology: 'hierarchical-mesh',
+ maxAgents: 15,
+ consensusProtocol: 'raft',
+ });
+ await coordinator.addAgent({
+ id: 'agent-1',
+ type: 'coder',
+ capabilities: ['coding', 'debugging'],
+ priority: 50,
+ });
+ await coordinator.addAgent({
+ id: 'agent-2',
+ type: 'tester',
+ capabilities: ['testing'],
+ priority: 60,
+ });
+ });
+
+ it('should throw error if not initialized', async () => {
+ // Given
+ const uninitializedCoordinator = new UnifiedCoordinator(
+ mockDistributor,
+ mockConsensus,
+ mockChannel,
+ mockEventPublisher
+ );
+
+ const task: CoordinationTask = {
+ id: 'task-1',
+ type: 'implementation',
+ payload: {},
+ requiredCapabilities: ['coding'],
+ priority: 50,
+ };
+
+ // When/Then
+ await expect(uninitializedCoordinator.coordinate(task)).rejects.toThrow(
+ 'Coordinator not initialized'
+ );
+ });
+
+ it('should propose task through consensus for hierarchical topology', async () => {
+ // Given
+ const task: CoordinationTask = {
+ id: 'task-1',
+ type: 'implementation',
+ payload: {},
+ requiredCapabilities: [],
+ priority: 50,
+ };
+ mockDistributor.distribute.mockResolvedValue([
+ { agentId: 'agent-1', taskId: 'task-1', subtask: {} },
+ ]);
+
+ // When
+ await coordinator.coordinate(task);
+
+ // Then
+ expect(mockConsensus.propose).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'TaskCoordination',
+ payload: task,
+ })
+ );
+ });
+
+ it('should throw error if consensus rejects', async () => {
+ // Given
+ const task: CoordinationTask = {
+ id: 'task-1',
+ type: 'implementation',
+ payload: {},
+ requiredCapabilities: [],
+ priority: 50,
+ };
+ mockConsensus.propose.mockResolvedValue(false);
+
+ // When/Then
+ await expect(coordinator.coordinate(task)).rejects.toThrow(
+ 'Task coordination not approved by consensus'
+ );
+ });
+
+ it('should distribute task to available agents', async () => {
+ // Given
+ const task: CoordinationTask = {
+ id: 'task-1',
+ type: 'implementation',
+ payload: {},
+ requiredCapabilities: ['coding'],
+ priority: 50,
+ };
+ mockDistributor.distribute.mockResolvedValue([
+ { agentId: 'agent-1', taskId: 'task-1', subtask: {} },
+ ]);
+
+ // When
+ await coordinator.coordinate(task);
+
+ // Then
+ expect(mockDistributor.distribute).toHaveBeenCalledWith(
+ task,
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'agent-1' }),
+ ])
+ );
+ });
+
+ it('should throw error if no agents with required capabilities', async () => {
+ // Given
+ const task: CoordinationTask = {
+ id: 'task-1',
+ type: 'implementation',
+ payload: {},
+ requiredCapabilities: ['non-existent-capability'],
+ priority: 50,
+ };
+
+ // When/Then
+ await expect(coordinator.coordinate(task)).rejects.toThrow(
+ 'No available agents with required capabilities'
+ );
+ });
+
+ it('should send task assignment to each agent', async () => {
+ // Given
+ const task: CoordinationTask = {
+ id: 'task-1',
+ type: 'implementation',
+ payload: {},
+ requiredCapabilities: [],
+ priority: 50,
+ };
+ mockDistributor.distribute.mockResolvedValue([
+ { agentId: 'agent-1', taskId: 'task-1', subtask: { part: 1 } },
+ { agentId: 'agent-2', taskId: 'task-1', subtask: { part: 2 } },
+ ]);
+
+ // When
+ await coordinator.coordinate(task);
+
+ // Then
+ expect(mockChannel.send).toHaveBeenCalledWith(
+ 'agent-1',
+ expect.objectContaining({
+ type: 'TaskAssigned',
+ })
+ );
+ expect(mockChannel.send).toHaveBeenCalledWith(
+ 'agent-2',
+ expect.objectContaining({
+ type: 'TaskAssigned',
+ })
+ );
+ });
+
+ it('should return coordination result', async () => {
+ // Given
+ const task: CoordinationTask = {
+ id: 'task-1',
+ type: 'implementation',
+ payload: {},
+ requiredCapabilities: [],
+ priority: 50,
+ };
+ mockDistributor.distribute.mockResolvedValue([
+ { agentId: 'agent-1', taskId: 'task-1', subtask: {} },
+ ]);
+
+ // When
+ const result = await coordinator.coordinate(task);
+
+ // Then
+ expect(result.taskId).toBe('task-1');
+ expect(result.success).toBe(true);
+ expect(result.participatingAgents).toContain('agent-1');
+ expect(result.duration).toBeGreaterThanOrEqual(0);
+ });
+
+ it('should publish TaskCoordinationCompleted event', async () => {
+ // Given
+ const task: CoordinationTask = {
+ id: 'task-1',
+ type: 'implementation',
+ payload: {},
+ requiredCapabilities: [],
+ priority: 50,
+ };
+ mockDistributor.distribute.mockResolvedValue([
+ { agentId: 'agent-1', taskId: 'task-1', subtask: {} },
+ ]);
+
+ // When
+ await coordinator.coordinate(task);
+
+ // Then
+ expect(mockEventPublisher.publish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'TaskCoordinationCompleted',
+ payload: expect.objectContaining({
+ taskId: 'task-1',
+ }),
+ })
+ );
+ });
+
+ it('should prioritize higher priority agents', async () => {
+ // Given
+ const task: CoordinationTask = {
+ id: 'task-1',
+ type: 'implementation',
+ payload: {},
+ requiredCapabilities: [],
+ priority: 50,
+ };
+
+ let capturedAgents: AgentInfo[] = [];
+ mockDistributor.distribute.mockImplementation(async (_, agents) => {
+ capturedAgents = agents;
+ return [{ agentId: agents[0].id, taskId: 'task-1', subtask: {} }];
+ });
+
+ // When
+ await coordinator.coordinate(task);
+
+ // Then - agent-2 has higher priority (60 vs 50)
+ expect(capturedAgents[0].id).toBe('agent-2');
+ });
+ });
+
+ describe('shutdown', () => {
+ beforeEach(async () => {
+ await coordinator.initialize({
+ topology: 'mesh',
+ maxAgents: 10,
+ consensusProtocol: 'gossip',
+ });
+ });
+
+ it('should broadcast shutdown message', async () => {
+ // When
+ await coordinator.shutdown();
+
+ // Then
+ expect(mockChannel.broadcast).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'CoordinatorShutdown',
+ })
+ );
+ });
+
+ it('should publish shutdown event', async () => {
+ // When
+ await coordinator.shutdown();
+
+ // Then
+ expect(mockEventPublisher.publish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: 'CoordinatorShutdown',
+ })
+ );
+ });
+
+ it('should clear all agents', async () => {
+ // Given
+ await coordinator.addAgent({
+ id: 'agent-1',
+ type: 'coder',
+ capabilities: ['coding'],
+ priority: 50,
+ });
+
+ // When
+ await coordinator.shutdown();
+
+ // Then
+ expect(coordinator.getAgentStatus('agent-1')).toBeNull();
+ });
+ });
+
+ describe('getTopology', () => {
+ it('should return configured topology', async () => {
+ // Given
+ await coordinator.initialize({
+ topology: 'hierarchical-mesh',
+ maxAgents: 15,
+ consensusProtocol: 'raft',
+ });
+
+ // When
+ const topology = coordinator.getTopology();
+
+ // Then
+ expect(topology).toBe('hierarchical-mesh');
+ });
+
+ it('should return mesh as default before initialization', () => {
+ // When
+ const topology = coordinator.getTopology();
+
+ // Then
+ expect(topology).toBe('mesh');
+ });
+ });
+
+ describe('15-agent swarm coordination (ADR-003)', () => {
+ it('should support up to 15 agents', async () => {
+ // Given
+ await coordinator.initialize({
+ topology: 'hierarchical-mesh',
+ maxAgents: 15,
+ consensusProtocol: 'raft',
+ });
+
+ const agentConfigurations = create15AgentSwarmConfig();
+
+ // When
+ for (let i = 0; i < 15; i++) {
+ await coordinator.addAgent({
+ id: `agent-${i}`,
+ type: agentConfigurations[i].type,
+ capabilities: agentConfigurations[i].capabilities,
+ priority: agentConfigurations[i].priority ?? 50,
+ });
+ }
+
+ // Then - all 15 agents should have idle status
+ for (let i = 0; i < 15; i++) {
+ expect(coordinator.getAgentStatus(`agent-${i}`)).toBe('idle');
+ }
+ });
+ });
+});
diff --git a/v3/neural/algorithms/a2c.ts b/v3/neural/algorithms/a2c.ts
new file mode 100644
index 0000000000..79c93047a7
--- /dev/null
+++ b/v3/neural/algorithms/a2c.ts
@@ -0,0 +1,478 @@
+/**
+ * Advantage Actor-Critic (A2C)
+ *
+ * Implements synchronous A2C algorithm with:
+ * - Shared actor-critic network
+ * - N-step returns
+ * - Entropy regularization
+ * - Advantage normalization
+ *
+ * Performance Target: <10ms per update step
+ */
+
+import type {
+ RLConfig,
+ Trajectory,
+ TrajectoryStep,
+} from '../types.js';
+
+/**
+ * A2C configuration
+ */
+export interface A2CConfig extends RLConfig {
+ algorithm: 'a2c';
+ nSteps: number;
+ useGAE: boolean;
+ gaeLambda: number;
+}
+
+/**
+ * Default A2C configuration
+ */
+export const DEFAULT_A2C_CONFIG: A2CConfig = {
+ algorithm: 'a2c',
+ learningRate: 0.0007,
+ gamma: 0.99,
+ entropyCoef: 0.01,
+ valueLossCoef: 0.5,
+ maxGradNorm: 0.5,
+ epochs: 1,
+ miniBatchSize: 32,
+ nSteps: 5,
+ useGAE: true,
+ gaeLambda: 0.95,
+};
+
+/**
+ * A2C experience entry
+ */
+interface A2CExperience {
+ state: Float32Array;
+ action: number;
+ reward: number;
+ value: number;
+ logProb: number;
+ entropy: number;
+}
+
+/**
+ * A2C Algorithm Implementation
+ */
+export class A2CAlgorithm {
+ private config: A2CConfig;
+
+ // Shared network weights
+ private sharedWeights: Float32Array;
+ private policyHead: Float32Array;
+ private valueHead: Float32Array;
+
+ // Optimizer state
+ private sharedMomentum: Float32Array;
+ private policyMomentum: Float32Array;
+ private valueMomentum: Float32Array;
+
+ // Experience buffer for n-step
+ private buffer: A2CExperience[] = [];
+
+ // Dimensions
+ private inputDim = 768;
+ private hiddenDim = 64;
+ private numActions = 4;
+
+ // Statistics
+ private updateCount = 0;
+ private avgPolicyLoss = 0;
+ private avgValueLoss = 0;
+ private avgEntropy = 0;
+
+ constructor(config: Partial<A2CConfig> = {}) {
+ this.config = { ...DEFAULT_A2C_CONFIG, ...config };
+
+ // Initialize network
+ const scale = Math.sqrt(2 / this.inputDim);
+ this.sharedWeights = new Float32Array(this.inputDim * this.hiddenDim);
+ this.policyHead = new Float32Array(this.hiddenDim * this.numActions);
+ this.valueHead = new Float32Array(this.hiddenDim);
+
+ for (let i = 0; i < this.sharedWeights.length; i++) {
+ this.sharedWeights[i] = (Math.random() - 0.5) * scale;
+ }
+ for (let i = 0; i < this.policyHead.length; i++) {
+ this.policyHead[i] = (Math.random() - 0.5) * 0.1;
+ }
+ for (let i = 0; i < this.valueHead.length; i++) {
+ this.valueHead[i] = (Math.random() - 0.5) * 0.1;
+ }
+
+ // Initialize momentum
+ this.sharedMomentum = new Float32Array(this.sharedWeights.length);
+ this.policyMomentum = new Float32Array(this.policyHead.length);
+ this.valueMomentum = new Float32Array(this.valueHead.length);
+ }
+
+ /**
+ * Add experience from trajectory
+ */
+ addExperience(trajectory: Trajectory): void {
+ for (const step of trajectory.steps) {
+ const { probs, value, entropy } = this.evaluate(step.stateAfter);
+ const action = this.hashAction(step.action);
+
+ this.buffer.push({
+ state: step.stateAfter,
+ action,
+ reward: step.reward,
+ value,
+ logProb: Math.log(probs[action] + 1e-8),
+ entropy,
+ });
+ }
+ }
+
+ /**
+ * Perform A2C update
+ * Target: <10ms
+ */
+ update(): { policyLoss: number; valueLoss: number; entropy: number } {
+ const startTime = performance.now();
+
+ if (this.buffer.length < this.config.nSteps) {
+ return { policyLoss: 0, valueLoss: 0, entropy: 0 };
+ }
+
+ // Compute returns and advantages
+ const returns = this.computeReturns();
+ const advantages = this.computeAdvantages(returns);
+
+ // Initialize gradients
+ const sharedGrad = new Float32Array(this.sharedWeights.length);
+ const policyGrad = new Float32Array(this.policyHead.length);
+ const valueGrad = new Float32Array(this.valueHead.length);
+
+ let totalPolicyLoss = 0;
+ let totalValueLoss = 0;
+ let totalEntropy = 0;
+
+ // Process all experiences
+ for (let i = 0; i < this.buffer.length; i++) {
+ const exp = this.buffer[i];
+ const advantage = advantages[i];
+ const return_ = returns[i];
+
+ // Get current policy and value
+ const { probs, value, hidden } = this.forwardWithHidden(exp.state);
+ const logProb = Math.log(probs[exp.action] + 1e-8);
+
+ // Policy loss
+ const policyLoss = -logProb * advantage;
+ totalPolicyLoss += policyLoss;
+
+ // Value loss
+ const valueLoss = (value - return_) ** 2;
+ totalValueLoss += valueLoss;
+
+ // Entropy
+ let entropy = 0;
+ for (const p of probs) {
+ if (p > 0) entropy -= p * Math.log(p);
+ }
+ totalEntropy += entropy;
+
+ // Accumulate gradients
+ this.accumulateGradients(
+ sharedGrad, policyGrad, valueGrad,
+ exp.state, hidden, exp.action,
+ advantage, value - return_
+ );
+ }
+
+ // Add entropy bonus to policy gradient
+ for (let i = 0; i < policyGrad.length; i++) {
+ policyGrad[i] -= this.config.entropyCoef * totalEntropy / this.buffer.length;
+ }
+
+ // Apply gradients
+ this.applyGradients(sharedGrad, policyGrad, valueGrad, this.buffer.length);
+
+ // Clear buffer
+ this.buffer = [];
+ this.updateCount++;
+
+ this.avgPolicyLoss = totalPolicyLoss / this.buffer.length || 0;
+ this.avgValueLoss = totalValueLoss / this.buffer.length || 0;
+ this.avgEntropy = totalEntropy / this.buffer.length || 0;
+
+ const elapsed = performance.now() - startTime;
+ if (elapsed > 10) {
+ console.warn(`A2C update exceeded target: ${elapsed.toFixed(2)}ms > 10ms`);
+ }
+
+ return {
+ policyLoss: this.avgPolicyLoss,
+ valueLoss: this.avgValueLoss,
+ entropy: this.avgEntropy,
+ };
+ }
+
+ /**
+ * Get action from policy
+ */
+ getAction(state: Float32Array): { action: number; value: number } {
+ const { probs, value } = this.evaluate(state);
+ const action = this.sampleAction(probs);
+ return { action, value };
+ }
+
+ /**
+ * Get statistics
+ */
+ getStats(): Record<string, number> {
+ return {
+ updateCount: this.updateCount,
+ bufferSize: this.buffer.length,
+ avgPolicyLoss: this.avgPolicyLoss,
+ avgValueLoss: this.avgValueLoss,
+ avgEntropy: this.avgEntropy,
+ };
+ }
+
+ // ==========================================================================
+ // Private Methods
+ // ==========================================================================
+
+ private evaluate(state: Float32Array): { probs: Float32Array; value: number; entropy: number } {
+ const { probs, value } = this.forward(state);
+
+ let entropy = 0;
+ for (const p of probs) {
+ if (p > 0) entropy -= p * Math.log(p);
+ }
+
+ return { probs, value, entropy };
+ }
+
+ private forward(state: Float32Array): { probs: Float32Array; value: number } {
+ // Shared hidden layer
+ const hidden = new Float32Array(this.hiddenDim);
+ for (let h = 0; h < this.hiddenDim; h++) {
+ let sum = 0;
+ for (let i = 0; i < Math.min(state.length, this.inputDim); i++) {
+ sum += state[i] * this.sharedWeights[i * this.hiddenDim + h];
+ }
+ hidden[h] = Math.max(0, sum); // ReLU
+ }
+
+ // Policy head
+ const logits = new Float32Array(this.numActions);
+ for (let a = 0; a < this.numActions; a++) {
+ let sum = 0;
+ for (let h = 0; h < this.hiddenDim; h++) {
+ sum += hidden[h] * this.policyHead[h * this.numActions + a];
+ }
+ logits[a] = sum;
+ }
+ const probs = this.softmax(logits);
+
+ // Value head
+ let value = 0;
+ for (let h = 0; h < this.hiddenDim; h++) {
+ value += hidden[h] * this.valueHead[h];
+ }
+
+ return { probs, value };
+ }
+
+ private forwardWithHidden(state: Float32Array): { probs: Float32Array; value: number; hidden: Float32Array } {
+ const hidden = new Float32Array(this.hiddenDim);
+ for (let h = 0; h < this.hiddenDim; h++) {
+ let sum = 0;
+ for (let i = 0; i < Math.min(state.length, this.inputDim); i++) {
+ sum += state[i] * this.sharedWeights[i * this.hiddenDim + h];
+ }
+ hidden[h] = Math.max(0, sum);
+ }
+
+ const logits = new Float32Array(this.numActions);
+ for (let a = 0; a < this.numActions; a++) {
+ let sum = 0;
+ for (let h = 0; h < this.hiddenDim; h++) {
+ sum += hidden[h] * this.policyHead[h * this.numActions + a];
+ }
+ logits[a] = sum;
+ }
+ const probs = this.softmax(logits);
+
+ let value = 0;
+ for (let h = 0; h < this.hiddenDim; h++) {
+ value += hidden[h] * this.valueHead[h];
+ }
+
+ return { probs, value, hidden };
+ }
+
+ private computeReturns(): number[] {
+ const returns = new Array(this.buffer.length).fill(0);
+ let cumReturn = 0;
+
+ // Bootstrap from last value if not terminal
+ if (this.buffer.length > 0) {
+ cumReturn = this.buffer[this.buffer.length - 1].value;
+ }
+
+ for (let t = this.buffer.length - 1; t >= 0; t--) {
+ cumReturn = this.buffer[t].reward + this.config.gamma * cumReturn;
+ returns[t] = cumReturn;
+ }
+
+ return returns;
+ }
+
+ private computeAdvantages(returns: number[]): number[] {
+ if (this.config.useGAE) {
+ return this.computeGAE();
+ }
+
+ // Simple advantage: return - value
+ const advantages = new Array(this.buffer.length).fill(0);
+ for (let i = 0; i < this.buffer.length; i++) {
+ advantages[i] = returns[i] - this.buffer[i].value;
+ }
+
+ // Normalize
+ const mean = advantages.reduce((a, b) => a + b, 0) / advantages.length;
+ const std = Math.sqrt(
+ advantages.reduce((a, b) => a + (b - mean) ** 2, 0) / advantages.length
+ ) + 1e-8;
+
+ return advantages.map(a => (a - mean) / std);
+ }
+
+ private computeGAE(): number[] {
+ const advantages = new Array(this.buffer.length).fill(0);
+ let lastGae = 0;
+
+ for (let t = this.buffer.length - 1; t >= 0; t--) {
+ const nextValue = t < this.buffer.length - 1
+ ? this.buffer[t + 1].value
+ : 0;
+ const delta = this.buffer[t].reward + this.config.gamma * nextValue - this.buffer[t].value;
+ lastGae = delta + this.config.gamma * this.config.gaeLambda * lastGae;
+ advantages[t] = lastGae;
+ }
+
+ // Normalize
+ const mean = advantages.reduce((a, b) => a + b, 0) / advantages.length;
+ const std = Math.sqrt(
+ advantages.reduce((a, b) => a + (b - mean) ** 2, 0) / advantages.length
+ ) + 1e-8;
+
+ return advantages.map(a => (a - mean) / std);
+ }
+
+ private accumulateGradients(
+ sharedGrad: Float32Array,
+ policyGrad: Float32Array,
+ valueGrad: Float32Array,
+ state: Float32Array,
+ hidden: Float32Array,
+ action: number,
+ advantage: number,
+ valueError: number
+ ): void {
+ // Policy gradient
+ for (let h = 0; h < this.hiddenDim; h++) {
+ policyGrad[h * this.numActions + action] += hidden[h] * advantage;
+ }
+
+ // Value gradient
+ for (let h = 0; h < this.hiddenDim; h++) {
+ valueGrad[h] += hidden[h] * valueError * this.config.valueLossCoef;
+ }
+
+ // Shared layer gradient (backprop through both heads)
+ for (let h = 0; h < this.hiddenDim; h++) {
+ if (hidden[h] > 0) { // ReLU gradient
+ const policySignal = advantage * this.policyHead[h * this.numActions + action];
+ const valueSignal = valueError * this.valueHead[h] * this.config.valueLossCoef;
+ const totalSignal = policySignal + valueSignal;
+
+ for (let i = 0; i < Math.min(state.length, this.inputDim); i++) {
+ sharedGrad[i * this.hiddenDim + h] += state[i] * totalSignal;
+ }
+ }
+ }
+ }
+
+ private applyGradients(
+ sharedGrad: Float32Array,
+ policyGrad: Float32Array,
+ valueGrad: Float32Array,
+ batchSize: number
+ ): void {
+ const lr = this.config.learningRate / batchSize;
+ const beta = 0.9;
+
+ // Apply to shared weights
+ for (let i = 0; i < this.sharedWeights.length; i++) {
+ const grad = Math.max(Math.min(sharedGrad[i], this.config.maxGradNorm), -this.config.maxGradNorm);
+ this.sharedMomentum[i] = beta * this.sharedMomentum[i] + (1 - beta) * grad;
+ this.sharedWeights[i] -= lr * this.sharedMomentum[i];
+ }
+
+ // Apply to policy head
+ for (let i = 0; i < this.policyHead.length; i++) {
+ const grad = Math.max(Math.min(policyGrad[i], this.config.maxGradNorm), -this.config.maxGradNorm);
+ this.policyMomentum[i] = beta * this.policyMomentum[i] + (1 - beta) * grad;
+ this.policyHead[i] -= lr * this.policyMomentum[i];
+ }
+
+ // Apply to value head
+ for (let i = 0; i < this.valueHead.length; i++) {
+ const grad = Math.max(Math.min(valueGrad[i], this.config.maxGradNorm), -this.config.maxGradNorm);
+ this.valueMomentum[i] = beta * this.valueMomentum[i] + (1 - beta) * grad;
+ this.valueHead[i] -= lr * this.valueMomentum[i];
+ }
+ }
+
+ private softmax(logits: Float32Array): Float32Array {
+ const max = Math.max(...logits);
+ const exps = new Float32Array(logits.length);
+ let sum = 0;
+
+ for (let i = 0; i < logits.length; i++) {
+ exps[i] = Math.exp(logits[i] - max);
+ sum += exps[i];
+ }
+
+ for (let i = 0; i < exps.length; i++) {
+ exps[i] /= sum;
+ }
+
+ return exps;
+ }
+
+ private sampleAction(probs: Float32Array): number {
+ const r = Math.random();
+ let cumSum = 0;
+ for (let i = 0; i < probs.length; i++) {
+ cumSum += probs[i];
+ if (r < cumSum) return i;
+ }
+ return probs.length - 1;
+ }
+
+ private hashAction(action: string): number {
+ let hash = 0;
+ for (let i = 0; i < action.length; i++) {
+ hash = (hash * 31 + action.charCodeAt(i)) % this.numActions;
+ }
+ return hash;
+ }
+}
+
+/**
+ * Factory function
+ */
+export function createA2C(config?: Partial<A2CConfig>): A2CAlgorithm {
+ return new A2CAlgorithm(config);
+}
diff --git a/v3/neural/algorithms/dqn.ts b/v3/neural/algorithms/dqn.ts
new file mode 100644
index 0000000000..8666a1a7dc
--- /dev/null
+++ b/v3/neural/algorithms/dqn.ts
@@ -0,0 +1,382 @@
+/**
+ * Deep Q-Network (DQN)
+ *
+ * Implements DQN with enhancements:
+ * - Experience replay
+ * - Target network
+ * - Double DQN (optional)
+ * - Dueling architecture (optional)
+ * - Epsilon-greedy exploration
+ *
+ * Performance Target: <10ms per update step
+ */
+
+import type {
+ DQNConfig,
+ Trajectory,
+ TrajectoryStep,
+} from '../types.js';
+
+/**
+ * Default DQN configuration
+ */
+export const DEFAULT_DQN_CONFIG: DQNConfig = {
+ algorithm: 'dqn',
+ learningRate: 0.0001,
+ gamma: 0.99,
+ entropyCoef: 0,
+ valueLossCoef: 1,
+ maxGradNorm: 10,
+ epochs: 1,
+ miniBatchSize: 32,
+ bufferSize: 10000,
+ explorationInitial: 1.0,
+ explorationFinal: 0.01,
+ explorationDecay: 10000,
+ targetUpdateFreq: 100,
+ doubleDQN: true,
+ duelingNetwork: false,
+};
+
+/**
+ * Experience for replay buffer
+ */
+interface DQNExperience {
+ state: Float32Array;
+ action: number;
+ reward: number;
+ nextState: Float32Array;
+ done: boolean;
+}
+
+/**
+ * DQN Algorithm Implementation
+ */
+export class DQNAlgorithm {
+ private config: DQNConfig;
+
+ // Q-network weights
+ private qWeights: Float32Array[];
+ private targetWeights: Float32Array[];
+
+ // Optimizer state
+ private qMomentum: Float32Array[];
+
+ // Replay buffer (circular)
+ private buffer: DQNExperience[] = [];
+ private bufferIdx = 0;
+
+ // Exploration
+ private epsilon: number;
+ private stepCount = 0;
+
+ // Number of actions
+ private numActions = 4;
+ private inputDim = 768;
+
+ // Statistics
+ private updateCount = 0;
+ private avgLoss = 0;
+
+ constructor(config: Partial<DQNConfig> = {}) {
+ this.config = { ...DEFAULT_DQN_CONFIG, ...config };
+ this.epsilon = this.config.explorationInitial;
+
+ // Initialize Q-network (2 hidden layers)
+ this.qWeights = this.initializeNetwork();
+ this.targetWeights = this.copyNetwork(this.qWeights);
+ this.qMomentum = this.qWeights.map(w => new Float32Array(w.length));
+ }
+
+ /**
+ * Add experience from trajectory
+ */
+ addExperience(trajectory: Trajectory): void {
+ for (let i = 0; i < trajectory.steps.length; i++) {
+ const step = trajectory.steps[i];
+ const nextStep = i < trajectory.steps.length - 1
+ ? trajectory.steps[i + 1]
+ : null;
+
+ const experience: DQNExperience = {
+ state: step.stateBefore,
+ action: this.hashAction(step.action),
+ reward: step.reward,
+ nextState: step.stateAfter,
+ done: nextStep === null,
+ };
+
+ // Add to circular buffer
+ if (this.buffer.length < this.config.bufferSize) {
+ this.buffer.push(experience);
+ } else {
+ this.buffer[this.bufferIdx] = experience;
+ }
+ this.bufferIdx = (this.bufferIdx + 1) % this.config.bufferSize;
+ }
+ }
+
+ /**
+ * Perform DQN update
+ * Target: <10ms
+ */
+ update(): { loss: number; epsilon: number } {
+ const startTime = performance.now();
+
+ if (this.buffer.length < this.config.miniBatchSize) {
+ return { loss: 0, epsilon: this.epsilon };
+ }
+
+ // Sample mini-batch
+ const batch = this.sampleBatch();
+
+ // Compute TD targets
+ let totalLoss = 0;
+ const gradients = this.qWeights.map(w => new Float32Array(w.length));
+
+ for (const exp of batch) {
+ // Current Q-values
+ const qValues = this.forward(exp.state, this.qWeights);
+ const currentQ = qValues[exp.action];
+
+ // Target Q-value
+ let targetQ: number;
+ if (exp.done) {
+ targetQ = exp.reward;
+ } else {
+ if (this.config.doubleDQN) {
+ // Double DQN: use online network to select action, target to evaluate
+ const nextQOnline = this.forward(exp.nextState, this.qWeights);
+ const bestAction = this.argmax(nextQOnline);
+ const nextQTarget = this.forward(exp.nextState, this.targetWeights);
+ targetQ = exp.reward + this.config.gamma * nextQTarget[bestAction];
+ } else {
+ // Standard DQN
+ const nextQ = this.forward(exp.nextState, this.targetWeights);
+ targetQ = exp.reward + this.config.gamma * Math.max(...nextQ);
+ }
+ }
+
+ // TD error
+ const tdError = targetQ - currentQ;
+ const loss = tdError * tdError;
+ totalLoss += loss;
+
+ // Accumulate gradients
+ this.accumulateGradients(gradients, exp.state, exp.action, tdError);
+ }
+
+ // Apply gradients
+ this.applyGradients(gradients, batch.length);
+
+ // Update target network periodically
+ this.stepCount++;
+ if (this.stepCount % this.config.targetUpdateFreq === 0) {
+ this.targetWeights = this.copyNetwork(this.qWeights);
+ }
+
+ // Decay exploration
+ this.epsilon = Math.max(
+ this.config.explorationFinal,
+ this.config.explorationInitial - this.stepCount / this.config.explorationDecay
+ );
+
+ this.updateCount++;
+ this.avgLoss = totalLoss / batch.length;
+
+ const elapsed = performance.now() - startTime;
+ if (elapsed > 10) {
+ console.warn(`DQN update exceeded target: ${elapsed.toFixed(2)}ms > 10ms`);
+ }
+
+ return {
+ loss: this.avgLoss,
+ epsilon: this.epsilon,
+ };
+ }
+
+ /**
+ * Get action using epsilon-greedy
+ */
+ getAction(state: Float32Array, explore: boolean = true): number {
+ if (explore && Math.random() < this.epsilon) {
+ return Math.floor(Math.random() * this.numActions);
+ }
+
+ const qValues = this.forward(state, this.qWeights);
+ return this.argmax(qValues);
+ }
+
+ /**
+ * Get Q-values for a state
+ */
+ getQValues(state: Float32Array): Float32Array {
+ return this.forward(state, this.qWeights);
+ }
+
+ /**
+ * Get statistics
+ */
+ getStats(): Record<string, number> {
+ return {
+ updateCount: this.updateCount,
+ bufferSize: this.buffer.length,
+ epsilon: this.epsilon,
+ avgLoss: this.avgLoss,
+ stepCount: this.stepCount,
+ };
+ }
+
+ // ==========================================================================
+ // Private Methods
+ // ==========================================================================
+
+ private initializeNetwork(): Float32Array[] {
+ // Simple 2-layer network: input -> hidden -> output
+ const hiddenDim = 64;
+ const weights: Float32Array[] = [];
+
+ // Layer 1: input_dim -> hidden
+ const w1 = new Float32Array(this.inputDim * hiddenDim);
+ const scale1 = Math.sqrt(2 / this.inputDim);
+ for (let i = 0; i < w1.length; i++) {
+ w1[i] = (Math.random() - 0.5) * scale1;
+ }
+ weights.push(w1);
+
+ // Layer 2: hidden -> num_actions
+ const w2 = new Float32Array(hiddenDim * this.numActions);
+ const scale2 = Math.sqrt(2 / hiddenDim);
+ for (let i = 0; i < w2.length; i++) {
+ w2[i] = (Math.random() - 0.5) * scale2;
+ }
+ weights.push(w2);
+
+ return weights;
+ }
+
+ private copyNetwork(weights: Float32Array[]): Float32Array[] {
+ return weights.map(w => new Float32Array(w));
+ }
+
+ private forward(state: Float32Array, weights: Float32Array[]): Float32Array {
+ const hiddenDim = 64;
+
+ // Layer 1: ReLU(W1 * x)
+ const hidden = new Float32Array(hiddenDim);
+ for (let h = 0; h < hiddenDim; h++) {
+ let sum = 0;
+ for (let i = 0; i < Math.min(state.length, this.inputDim); i++) {
+ sum += state[i] * weights[0][i * hiddenDim + h];
+ }
+ hidden[h] = Math.max(0, sum); // ReLU
+ }
+
+ // Layer 2: W2 * hidden (no activation for Q-values)
+ const output = new Float32Array(this.numActions);
+ for (let a = 0; a < this.numActions; a++) {
+ let sum = 0;
+ for (let h = 0; h < hiddenDim; h++) {
+ sum += hidden[h] * weights[1][h * this.numActions + a];
+ }
+ output[a] = sum;
+ }
+
+ return output;
+ }
+
+ private accumulateGradients(
+ gradients: Float32Array[],
+ state: Float32Array,
+ action: number,
+ tdError: number
+ ): void {
+ const hiddenDim = 64;
+
+ // Forward pass to get hidden activations
+ const hidden = new Float32Array(hiddenDim);
+ for (let h = 0; h < hiddenDim; h++) {
+ let sum = 0;
+ for (let i = 0; i < Math.min(state.length, this.inputDim); i++) {
+ sum += state[i] * this.qWeights[0][i * hiddenDim + h];
+ }
+ hidden[h] = Math.max(0, sum);
+ }
+
+ // Gradient for layer 2 (only for selected action)
+ for (let h = 0; h < hiddenDim; h++) {
+ gradients[1][h * this.numActions + action] += hidden[h] * tdError;
+ }
+
+ // Gradient for layer 1 (backprop through ReLU)
+ for (let h = 0; h < hiddenDim; h++) {
+ if (hidden[h] > 0) { // ReLU gradient
+ const grad = tdError * this.qWeights[1][h * this.numActions + action];
+ for (let i = 0; i < Math.min(state.length, this.inputDim); i++) {
+ gradients[0][i * hiddenDim + h] += state[i] * grad;
+ }
+ }
+ }
+ }
+
+ private applyGradients(gradients: Float32Array[], batchSize: number): void {
+ const lr = this.config.learningRate / batchSize;
+ const beta = 0.9;
+
+ for (let layer = 0; layer < gradients.length; layer++) {
+ for (let i = 0; i < gradients[layer].length; i++) {
+ // Gradient clipping
+ const grad = Math.max(
+ Math.min(gradients[layer][i], this.config.maxGradNorm),
+ -this.config.maxGradNorm
+ );
+
+ // Momentum update
+ this.qMomentum[layer][i] = beta * this.qMomentum[layer][i] + (1 - beta) * grad;
+ this.qWeights[layer][i] += lr * this.qMomentum[layer][i];
+ }
+ }
+ }
+
+ private sampleBatch(): DQNExperience[] {
+ const batch: DQNExperience[] = [];
+ const indices = new Set<number>();
+
+ while (indices.size < this.config.miniBatchSize && indices.size < this.buffer.length) {
+ indices.add(Math.floor(Math.random() * this.buffer.length));
+ }
+
+ for (const idx of indices) {
+ batch.push(this.buffer[idx]);
+ }
+
+ return batch;
+ }
+
+ private hashAction(action: string): number {
+ let hash = 0;
+ for (let i = 0; i < action.length; i++) {
+ hash = (hash * 31 + action.charCodeAt(i)) % this.numActions;
+ }
+ return hash;
+ }
+
+ private argmax(values: Float32Array): number {
+ let maxIdx = 0;
+ let maxVal = values[0];
+ for (let i = 1; i < values.length; i++) {
+ if (values[i] > maxVal) {
+ maxVal = values[i];
+ maxIdx = i;
+ }
+ }
+ return maxIdx;
+ }
+}
+
+/**
+ * Factory function
+ */
+export function createDQN(config?: Partial<DQNConfig>): DQNAlgorithm {
+ return new DQNAlgorithm(config);
+}