Skip to content
Shell
7.2 KB
Raw Blame
/**
 * Multi-Agent Parallel Scanning
 * Run multiple scanners concurrently for faster research
 */

import { findGoldmineThreads, fetchThread, searchSubreddit } from '../reddit/index.js';
import { analyzeThread, quickAnalyze } from './analyzer.js';
import { getRedditRateLimiter, retryWithBackoff } from '../utils/rateLimiter.js';
import type { GoldmineScore, MarketInsight } from '../types.js';

export interface ScanTask {
  id: string;
  subreddit: string;
  query?: string;
  maxThreads?: number;
  useClaudeAnalysis?: boolean;
}

export interface ScanResult {
  taskId: string;
  subreddit: string;
  query?: string;
  threads: GoldmineScore[];
  insights: MarketInsight[];
  errors: string[];
  duration: number;
}

/**
 * Worker function to scan a single subreddit
 */
async function scanSubredditWorker(task: ScanTask): Promise<ScanResult> {
  const startTime = Date.now();
  const rateLimiter = getRedditRateLimiter();
  const result: ScanResult = {
    taskId: task.id,
    subreddit: task.subreddit,
    query: task.query,
    threads: [],
    insights: [],
    errors: [],
    duration: 0
  };

  try {
    // Find goldmine threads with rate limiting
    await rateLimiter.acquireWithBackoff();

    const threads = task.query
      ? await retryWithBackoff(() => searchSubreddit(task.subreddit, task.query!, { limit: task.maxThreads || 20 }))
      : await retryWithBackoff(() => findGoldmineThreads(task.subreddit));

    rateLimiter.reportSuccess();
    result.threads = threads;

    // Analyze top threads
    const threadsToAnalyze = threads
      .filter(t => t.goldmineScore >= 15)
      .slice(0, 5);

    for (const thread of threadsToAnalyze) {
      try {
        await rateLimiter.acquireWithBackoff();
        const threadData = await retryWithBackoff(() => fetchThread(thread.url));
        rateLimiter.reportSuccess();

        let insight: MarketInsight | null = null;

        if (task.useClaudeAnalysis && process.env.ANTHROPIC_API_KEY) {
          insight = await analyzeThread(threadData);
        } else {
          // Quick analysis fallback
          const quick = quickAnalyze(threadData);
          insight = {
            id: crypto.randomUUID(),
            threadUrl: thread.url,
            subreddit: thread.subreddit,
            extractedAt: new Date().toISOString(),
            painPoints: [],
            opportunities: [],
            competitorMentions: quick.competitorMentions || [],
            buyingSignals: quick.buyingSignals || [],
            keyQuotes: [],
            summary: `Quick analysis of ${thread.title.slice(0, 50)}...`
          };
        }

        if (insight) {
          result.insights.push(insight);
        }
      } catch (error) {
        result.errors.push(`Failed to analyze ${thread.url}: ${error}`);
        rateLimiter.reportError();
      }
    }
  } catch (error) {
    result.errors.push(`Failed to scan r/${task.subreddit}: ${error}`);
    rateLimiter.reportError();
  }

  result.duration = Date.now() - startTime;
  return result;
}

/**
 * Multi-Agent Scanner
 * Coordinates parallel scanning of multiple subreddits
 */
export class MultiAgentScanner {
  private maxConcurrency: number;
  private verbose: boolean;

  constructor(options: { maxConcurrency?: number; verbose?: boolean } = {}) {
    this.maxConcurrency = options.maxConcurrency || 3;
    this.verbose = options.verbose ?? false;
  }

  /**
   * Run tasks in parallel with concurrency limit
   */
  async runParallel(tasks: ScanTask[]): Promise<ScanResult[]> {
    const results: ScanResult[] = [];
    const queue = [...tasks];
    const running: Promise<void>[] = [];

    const runTask = async (task: ScanTask): Promise<void> => {
      if (this.verbose) {
        console.log(`[Agent] Starting scan: r/${task.subreddit}${task.query ? ` "${task.query}"` : ''}`);
      }

      const result = await scanSubredditWorker(task);
      results.push(result);

      if (this.verbose) {
        console.log(`[Agent] Completed: r/${task.subreddit} - ${result.threads.length} threads, ${result.insights.length} insights (${result.duration}ms)`);
        if (result.errors.length > 0) {
          console.log(`[Agent] Errors: ${result.errors.length}`);
        }
      }
    };

    // Process queue with concurrency limit
    while (queue.length > 0 || running.length > 0) {
      // Start new tasks up to concurrency limit
      while (running.length < this.maxConcurrency && queue.length > 0) {
        const task = queue.shift()!;
        const promise = runTask(task).then(() => {
          const index = running.indexOf(promise);
          if (index > -1) running.splice(index, 1);
        });
        running.push(promise);
      }

      // Wait for at least one to complete
      if (running.length > 0) {
        await Promise.race(running);
      }
    }

    return results;
  }

  /**
   * Scan multiple subreddits
   */
  async scanSubreddits(
    subreddits: string[],
    options: { queries?: string[]; useClaudeAnalysis?: boolean } = {}
  ): Promise<ScanResult[]> {
    const tasks: ScanTask[] = [];

    for (const subreddit of subreddits) {
      if (options.queries && options.queries.length > 0) {
        // Create task for each query
        for (const query of options.queries) {
          tasks.push({
            id: `${subreddit}-${query}`,
            subreddit,
            query,
            useClaudeAnalysis: options.useClaudeAnalysis
          });
        }
      } else {
        // Single task without query
        tasks.push({
          id: subreddit,
          subreddit,
          useClaudeAnalysis: options.useClaudeAnalysis
        });
      }
    }

    return this.runParallel(tasks);
  }

  /**
   * Get aggregated stats from results
   */
  getAggregatedStats(results: ScanResult[]): {
    totalThreads: number;
    totalInsights: number;
    totalErrors: number;
    totalDuration: number;
    bySubreddit: Record<string, { threads: number; insights: number }>;
    topThreads: GoldmineScore[];
  } {
    const stats = {
      totalThreads: 0,
      totalInsights: 0,
      totalErrors: 0,
      totalDuration: 0,
      bySubreddit: {} as Record<string, { threads: number; insights: number }>,
      topThreads: [] as GoldmineScore[]
    };

    for (const result of results) {
      stats.totalThreads += result.threads.length;
      stats.totalInsights += result.insights.length;
      stats.totalErrors += result.errors.length;
      stats.totalDuration = Math.max(stats.totalDuration, result.duration);

      stats.bySubreddit[result.subreddit] = stats.bySubreddit[result.subreddit] || { threads: 0, insights: 0 };
      stats.bySubreddit[result.subreddit].threads += result.threads.length;
      stats.bySubreddit[result.subreddit].insights += result.insights.length;

      stats.topThreads.push(...result.threads);
    }

    // Sort and dedupe top threads
    const seenUrls = new Set<string>();
    stats.topThreads = stats.topThreads
      .filter(t => {
        if (seenUrls.has(t.url)) return false;
        seenUrls.add(t.url);
        return true;
      })
      .sort((a, b) => b.goldmineScore - a.goldmineScore)
      .slice(0, 50);

    return stats;
  }
}

/**
 * Create multi-agent scanner
 */
export function createMultiAgentScanner(options?: { maxConcurrency?: number; verbose?: boolean }): MultiAgentScanner {
  return new MultiAgentScanner(options);
}