🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
Checkpoint: File edits
rUv84f03cb39f98
diff --git a/agents/architect.yaml b/agents/architect.yamlindex 0c1fa90234..9955239bfd 100644--- a/agents/architect.yaml+++ b/agents/architect.yaml@@ -8,4 +8,4 @@ capabilities:optimizations:- context-caching- memory-persistence-createdAt: "2026-01-07T23:33:50.680Z"+createdAt: "2026-01-08T00:04:31.596Z"diff --git a/agents/coder.yaml b/agents/coder.yamlindex 29229d9fe3..69d6026d2f 100644--- a/agents/coder.yaml+++ b/agents/coder.yaml@@ -8,4 +8,4 @@ capabilities:optimizations:- flash-attention- token-reduction-createdAt: "2026-01-07T23:33:50.680Z"+createdAt: "2026-01-08T00:04:31.596Z"diff --git a/agents/reviewer.yaml b/agents/reviewer.yamlindex 238daee714..3889da0b82 100644--- a/agents/reviewer.yaml+++ b/agents/reviewer.yaml@@ -7,4 +7,4 @@ capabilities:- best-practicesoptimizations:- incremental-analysis-createdAt: "2026-01-07T23:33:50.681Z"+createdAt: "2026-01-08T00:04:31.596Z"diff --git a/agents/security-architect.yaml b/agents/security-architect.yamlindex 85d55d957a..425571c2bf 100644--- a/agents/security-architect.yaml+++ b/agents/security-architect.yaml@@ -7,4 +7,4 @@ capabilities:- security-reviewoptimizations:- pattern-matching-createdAt: "2026-01-07T23:33:50.680Z"+createdAt: "2026-01-08T00:04:31.596Z"diff --git a/agents/tester.yaml b/agents/tester.yamlindex ac3d57c506..2b7f9dbcdf 100644--- a/agents/tester.yaml+++ b/agents/tester.yaml@@ -7,4 +7,4 @@ capabilities:- coverageoptimizations:- parallel-execution-createdAt: "2026-01-07T23:33:50.680Z"+createdAt: "2026-01-08T00:04:31.596Z"diff --git a/v3/@claude-flow/cli/src/services/container-worker-pool.ts b/v3/@claude-flow/cli/src/services/container-worker-pool.tsnew file mode 100644index 0000000000..6b2cb248c6--- /dev/null+++ b/v3/@claude-flow/cli/src/services/container-worker-pool.ts@@ -0,0 +1,742 @@+/**+ * Container Worker Pool+ * Docker-based worker pool for high-throughput headless execution.+ *+ * ADR-020: Headless Worker Integration Architecture - Phase 3+ * - Manages pool of Docker containers for isolated worker execution+ * - Supports dynamic scaling based on workload+ * - Provides container lifecycle management+ * - Integrates with WorkerQueue for task distribution+ *+ * Key Features:+ * - Container pooling with configurable size+ * - Health checking and auto-recovery+ * - Resource limits (CPU, memory)+ * - Volume mounting for workspace access+ * - Network isolation per worker type+ */++import { EventEmitter } from 'events';+import { spawn, execSync, type ChildProcess } from 'child_process';+import { existsSync, mkdirSync, writeFileSync, readFileSync } from 'fs';+import { join } from 'path';+import type { HeadlessWorkerType, HeadlessExecutionResult, SandboxMode } from './headless-worker-executor.js';++// ============================================+// Type Definitions+// ============================================++/**+ * Container state+ */+export type ContainerState = 'creating' | 'ready' | 'busy' | 'unhealthy' | 'terminated';++/**+ * Container info+ */+export interface ContainerInfo {+ id: string;+ name: string;+ state: ContainerState;+ createdAt: Date;+ lastUsedAt?: Date;+ workerType?: HeadlessWorkerType;+ executionCount: number;+ healthCheckFailures: number;+ pid?: number;+}++/**+ * Container pool configuration+ */+export interface ContainerPoolConfig {+ /** Maximum number of containers in the pool */+ maxContainers: number;++ /** Minimum number of containers to keep warm */+ minContainers: number;++ /** Docker image to use */+ image: string;++ /** Container resource limits */+ resources: {+ cpus: string;+ memory: string;+ };++ /** Health check interval in ms */+ healthCheckIntervalMs: number;++ /** Container idle timeout in ms */+ idleTimeoutMs: number;++ /** Workspace volume mount path */+ workspacePath: string;++ /** State persistence path */+ statePath: string;++ /** Network name for container isolation */+ network?: string;++ /** Environment variables for containers */+ env?: Record<string, string>;++ /** Default sandbox mode */+ defaultSandbox: SandboxMode;+}++/**+ * Container execution options+ */+export interface ContainerExecutionOptions {+ workerType: HeadlessWorkerType;+ prompt: string;+ contextPatterns?: string[];+ sandbox?: SandboxMode;+ model?: string;+ timeoutMs?: number;+}++/**+ * Pool status+ */+export interface ContainerPoolStatus {+ totalContainers: number;+ readyContainers: number;+ busyContainers: number;+ unhealthyContainers: number;+ queuedTasks: number;+ containers: ContainerInfo[];+ dockerAvailable: boolean;+ lastHealthCheck?: Date;+}++// ============================================+// Constants+// ============================================++const DEFAULT_CONFIG: ContainerPoolConfig = {+ maxContainers: 3,+ minContainers: 1,+ image: 'ghcr.io/ruvnet/claude-flow-headless:latest',+ resources: {+ cpus: '2',+ memory: '4g',+ },+ healthCheckIntervalMs: 30000,+ idleTimeoutMs: 300000, // 5 minutes+ workspacePath: '/workspace',+ statePath: '.claude-flow/container-pool',+ defaultSandbox: 'strict',+};++// ============================================+// ContainerWorkerPool Class+// ============================================++/**+ * ContainerWorkerPool - Manages Docker containers for headless worker execution+ */+export class ContainerWorkerPool extends EventEmitter {+ private config: ContainerPoolConfig;+ private projectRoot: string;+ private containers: Map<string, ContainerInfo> = new Map();+ private taskQueue: Array<{+ options: ContainerExecutionOptions;+ resolve: (result: HeadlessExecutionResult) => void;+ reject: (error: Error) => void;+ queuedAt: Date;+ }> = [];+ private healthCheckTimer?: NodeJS.Timeout;+ private idleCheckTimer?: NodeJS.Timeout;+ private dockerAvailable: boolean | null = null;+ private initialized = false;++ constructor(projectRoot: string, config?: Partial<ContainerPoolConfig>) {+ super();+ this.projectRoot = projectRoot;+ this.config = { ...DEFAULT_CONFIG, ...config };++ // Ensure state directory exists+ const stateDir = join(projectRoot, this.config.statePath);+ if (!existsSync(stateDir)) {+ mkdirSync(stateDir, { recursive: true });+ }+ }++ // ============================================+ // Public API+ // ============================================++ /**+ * Initialize the container pool+ */+ async initialize(): Promise<boolean> {+ if (this.initialized) {+ return true;+ }++ // Check Docker availability+ this.dockerAvailable = await this.checkDockerAvailable();+ if (!this.dockerAvailable) {+ this.emit('warning', { message: 'Docker not available - container pool disabled' });+ return false;+ }++ // Pull image if needed+ await this.ensureImage();++ // Create minimum containers+ await this.scaleToMinimum();++ // Start health check timer+ this.startHealthChecks();++ // Start idle check timer+ this.startIdleChecks();++ this.initialized = true;+ this.emit('initialized', { containers: this.containers.size });++ return true;+ }++ /**+ * Execute a worker in a container+ */+ async execute(options: ContainerExecutionOptions): Promise<HeadlessExecutionResult> {+ if (!this.initialized) {+ await this.initialize();+ }++ if (!this.dockerAvailable) {+ return this.createErrorResult(options.workerType, 'Docker not available');+ }++ // Try to get a ready container+ const container = this.getReadyContainer();++ if (container) {+ return this.executeInContainer(container, options);+ }++ // No ready containers - check if we can create more+ if (this.containers.size < this.config.maxContainers) {+ const newContainer = await this.createContainer();+ if (newContainer) {+ return this.executeInContainer(newContainer, options);+ }+ }++ // Queue the task+ return new Promise((resolve, reject) => {+ this.taskQueue.push({+ options,+ resolve,+ reject,+ queuedAt: new Date(),+ });+ this.emit('taskQueued', {+ workerType: options.workerType,+ queuePosition: this.taskQueue.length,+ });+ });+ }++ /**+ * Scale pool for batch execution+ */+ async scaleForBatch(workerCount: number): Promise<void> {+ const targetSize = Math.min(workerCount, this.config.maxContainers);+ const currentSize = this.containers.size;++ if (targetSize > currentSize) {+ const toCreate = targetSize - currentSize;+ const createPromises: Promise<ContainerInfo | null>[] = [];++ for (let i = 0; i < toCreate; i++) {+ createPromises.push(this.createContainer());+ }++ await Promise.all(createPromises);+ this.emit('scaled', { from: currentSize, to: this.containers.size });+ }+ }++ /**+ * Get pool status+ */+ getStatus(): ContainerPoolStatus {+ const containers = Array.from(this.containers.values());+ return {+ totalContainers: containers.length,+ readyContainers: containers.filter(c => c.state === 'ready').length,+ busyContainers: containers.filter(c => c.state === 'busy').length,+ unhealthyContainers: containers.filter(c => c.state === 'unhealthy').length,+ queuedTasks: this.taskQueue.length,+ containers,+ dockerAvailable: this.dockerAvailable ?? false,+ lastHealthCheck: undefined, // Will be set by health check+ };+ }++ /**+ * Shutdown the pool+ */+ async shutdown(): Promise<void> {+ // Stop timers+ if (this.healthCheckTimer) {+ clearInterval(this.healthCheckTimer);+ }+ if (this.idleCheckTimer) {+ clearInterval(this.idleCheckTimer);+ }++ // Reject queued tasks+ for (const task of this.taskQueue) {+ task.reject(new Error('Pool shutting down'));+ }+ this.taskQueue = [];++ // Terminate all containers+ const terminatePromises: Promise<void>[] = [];+ for (const [id] of this.containers) {+ terminatePromises.push(this.terminateContainer(id));+ }+ await Promise.all(terminatePromises);++ this.initialized = false;+ this.emit('shutdown', {});+ }++ // ============================================+ // Private Methods - Container Lifecycle+ // ============================================++ /**+ * Check if Docker is available+ */+ private async checkDockerAvailable(): Promise<boolean> {+ try {+ execSync('docker --version', { encoding: 'utf-8', stdio: 'pipe' });+ execSync('docker info', { encoding: 'utf-8', stdio: 'pipe', timeout: 5000 });+ return true;+ } catch {+ return false;+ }+ }++ /**+ * Ensure the container image exists+ */+ private async ensureImage(): Promise<void> {+ try {+ execSync(`docker image inspect ${this.config.image}`, { stdio: 'pipe' });+ } catch {+ // Image not found, try to pull+ this.emit('imagePull', { image: this.config.image });+ try {+ execSync(`docker pull ${this.config.image}`, { stdio: 'pipe', timeout: 300000 });+ } catch (error) {+ this.emit('warning', { message: `Failed to pull image: ${error}` });+ // Continue anyway - might work with local image+ }+ }+ }++ /**+ * Create a new container+ */+ private async createContainer(): Promise<ContainerInfo | null> {+ const id = `cf-worker-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;+ const name = `claude-flow-worker-${id}`;++ const containerInfo: ContainerInfo = {+ id,+ name,+ state: 'creating',+ createdAt: new Date(),+ executionCount: 0,+ healthCheckFailures: 0,+ };++ this.containers.set(id, containerInfo);+ this.emit('containerCreating', { id, name });++ try {+ // Build docker run command+ const args = [+ 'run', '-d',+ '--name', name,+ '--cpus', this.config.resources.cpus,+ '--memory', this.config.resources.memory,+ '-v', `${this.projectRoot}:${this.config.workspacePath}:ro`,+ '-v', `${join(this.projectRoot, this.config.statePath)}:/root/.claude-flow`,+ '-w', this.config.workspacePath,+ ];++ // Add environment variables+ const env = {+ ...this.config.env,+ ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || '',+ CLAUDE_CODE_HEADLESS: 'true',+ CLAUDE_CODE_SANDBOX_MODE: this.config.defaultSandbox,+ };++ for (const [key, value] of Object.entries(env)) {+ if (value) {+ args.push('-e', `${key}=${value}`);+ }+ }++ // Add network if specified+ if (this.config.network) {+ args.push('--network', this.config.network);+ }++ // Add image and entrypoint to keep container running+ args.push(this.config.image, 'tail', '-f', '/dev/null');++ // Create the container+ const output = execSync(`docker ${args.join(' ')}`, {+ encoding: 'utf-8',+ stdio: 'pipe',+ }).trim();++ containerInfo.state = 'ready';+ this.emit('containerCreated', { id, name, containerId: output });++ return containerInfo;+ } catch (error) {+ this.containers.delete(id);+ this.emit('containerError', { id, error: String(error) });+ return null;+ }+ }++ /**+ * Terminate a container+ */+ private async terminateContainer(id: string): Promise<void> {+ const container = this.containers.get(id);+ if (!container) return;++ container.state = 'terminated';++ try {+ execSync(`docker rm -f ${container.name}`, { stdio: 'pipe' });+ } catch {+ // Ignore removal errors+ }++ this.containers.delete(id);+ this.emit('containerTerminated', { id, name: container.name });+ }++ /**+ * Get a ready container+ */+ private getReadyContainer(): ContainerInfo | null {+ for (const container of this.containers.values()) {+ if (container.state === 'ready') {+ return container;+ }+ }+ return null;+ }++ /**+ * Scale to minimum containers+ */+ private async scaleToMinimum(): Promise<void> {+ const current = this.containers.size;+ const needed = this.config.minContainers - current;++ if (needed > 0) {+ const createPromises: Promise<ContainerInfo | null>[] = [];+ for (let i = 0; i < needed; i++) {+ createPromises.push(this.createContainer());+ }+ await Promise.all(createPromises);+ }+ }++ // ============================================+ // Private Methods - Execution+ // ============================================++ /**+ * Execute worker in a specific container+ */+ private async executeInContainer(+ container: ContainerInfo,+ options: ContainerExecutionOptions+ ): Promise<HeadlessExecutionResult> {+ const startTime = Date.now();+ const executionId = `${options.workerType}_${startTime}_${Math.random().toString(36).slice(2, 8)}`;++ container.state = 'busy';+ container.workerType = options.workerType;+ container.lastUsedAt = new Date();++ this.emit('executionStart', { executionId, containerId: container.id, workerType: options.workerType });++ try {+ // Build the command to run inside container+ const command = this.buildWorkerCommand(options);++ // Execute in container with timeout+ const timeoutMs = options.timeoutMs || 300000;+ const output = await this.execInContainer(container.name, command, timeoutMs);++ container.state = 'ready';+ container.executionCount++;++ const result: HeadlessExecutionResult = {+ success: true,+ output: output,+ parsedOutput: this.tryParseJson(output),+ durationMs: Date.now() - startTime,+ model: options.model || 'sonnet',+ sandboxMode: options.sandbox || this.config.defaultSandbox,+ workerType: options.workerType,+ timestamp: new Date(),+ executionId,+ };++ this.emit('executionComplete', result);++ // Process queue+ this.processQueue();++ return result;+ } catch (error) {+ container.state = 'ready';++ const result = this.createErrorResult(+ options.workerType,+ error instanceof Error ? error.message : String(error)+ );+ result.executionId = executionId;+ result.durationMs = Date.now() - startTime;++ this.emit('executionError', result);++ // Process queue+ this.processQueue();++ return result;+ }+ }++ /**+ * Execute command in container+ */+ private async execInContainer(+ containerName: string,+ command: string[],+ timeoutMs: number+ ): Promise<string> {+ return new Promise((resolve, reject) => {+ const args = ['exec', containerName, ...command];++ const child = spawn('docker', args, {+ stdio: ['pipe', 'pipe', 'pipe'],+ });++ let stdout = '';+ let stderr = '';+ let timedOut = false;++ const timeout = setTimeout(() => {+ timedOut = true;+ child.kill('SIGTERM');+ setTimeout(() => {+ if (!child.killed) {+ child.kill('SIGKILL');+ }+ }, 5000);+ }, timeoutMs);++ child.stdout?.on('data', (data: Buffer) => {+ stdout += data.toString();+ });++ child.stderr?.on('data', (data: Buffer) => {+ stderr += data.toString();+ });++ child.on('close', (code) => {+ clearTimeout(timeout);++ if (timedOut) {+ reject(new Error(`Execution timed out after ${timeoutMs}ms`));+ return;+ }++ if (code !== 0) {+ reject(new Error(stderr || `Process exited with code ${code}`));+ return;+ }++ resolve(stdout);+ });++ child.on('error', (error) => {+ clearTimeout(timeout);+ reject(error);+ });+ });+ }++ /**+ * Build worker command for container execution+ */+ private buildWorkerCommand(options: ContainerExecutionOptions): string[] {+ // Use npx to run claude-flow daemon trigger+ return [+ 'npx', 'claude-flow@v3alpha',+ 'daemon', 'trigger',+ '-w', options.workerType,+ '--headless',+ ];+ }++ /**+ * Process queued tasks+ */+ private processQueue(): void {+ while (this.taskQueue.length > 0) {+ const container = this.getReadyContainer();+ if (!container) break;++ const task = this.taskQueue.shift();+ if (task) {+ this.executeInContainer(container, task.options)+ .then(task.resolve)+ .catch(task.reject);+ }+ }+ }++ // ============================================+ // Private Methods - Health & Maintenance+ // ============================================++ /**+ * Start health check timer+ */+ private startHealthChecks(): void {+ this.healthCheckTimer = setInterval(async () => {+ await this.runHealthChecks();+ }, this.config.healthCheckIntervalMs);+ }++ /**+ * Run health checks on all containers+ */+ private async runHealthChecks(): Promise<void> {+ for (const [id, container] of this.containers) {+ if (container.state === 'terminated') continue;++ try {+ // Check if container is running+ const output = execSync(`docker inspect -f '{{.State.Running}}' ${container.name}`, {+ encoding: 'utf-8',+ stdio: 'pipe',+ }).trim();++ if (output !== 'true') {+ container.healthCheckFailures++;+ if (container.healthCheckFailures >= 3) {+ container.state = 'unhealthy';+ this.emit('containerUnhealthy', { id, name: container.name });++ // Remove and replace+ await this.terminateContainer(id);+ if (this.containers.size < this.config.minContainers) {+ await this.createContainer();+ }+ }+ } else {+ container.healthCheckFailures = 0;+ }+ } catch {+ container.healthCheckFailures++;+ }+ }++ this.emit('healthCheckComplete', { containers: this.containers.size });+ }++ /**+ * Start idle check timer+ */+ private startIdleChecks(): void {+ this.idleCheckTimer = setInterval(async () => {+ await this.runIdleChecks();+ }, 60000); // Check every minute+ }++ /**+ * Terminate idle containers above minimum+ */+ private async runIdleChecks(): Promise<void> {+ const now = Date.now();+ const readyContainers = Array.from(this.containers.values())+ .filter(c => c.state === 'ready')+ .sort((a, b) => (a.lastUsedAt?.getTime() || 0) - (b.lastUsedAt?.getTime() || 0));++ // Keep minimum containers+ const toTerminate = readyContainers.slice(this.config.minContainers);++ for (const container of toTerminate) {+ const lastUsed = container.lastUsedAt?.getTime() || container.createdAt.getTime();+ if (now - lastUsed > this.config.idleTimeoutMs) {+ await this.terminateContainer(container.id);+ this.emit('containerIdleTerminated', { id: container.id, name: container.name });+ }+ }+ }++ // ============================================+ // Private Methods - Utilities+ // ============================================++ /**+ * Try to parse JSON from output+ */+ private tryParseJson(output: string): unknown {+ try {+ const jsonMatch = output.match(/\{[\s\S]*\}/);+ if (jsonMatch) {+ return JSON.parse(jsonMatch[0]);+ }+ return JSON.parse(output.trim());+ } catch {+ return undefined;+ }+ }++ /**+ * Create an error result+ */+ private createErrorResult(workerType: HeadlessWorkerType, error: string): HeadlessExecutionResult {+ return {+ success: false,+ output: '',+ durationMs: 0,+ model: 'unknown',+ sandboxMode: this.config.defaultSandbox,+ workerType,+ timestamp: new Date(),+ executionId: `error_${Date.now()}`,+ error,+ };+ }+}++// Export default+export default ContainerWorkerPool;