Shell
/**
* Multi-Agent Parallel Scanning
* Run multiple scanners concurrently for faster research
*/
import { findGoldmineThreads, fetchThread, searchSubreddit } from '../reddit/index.js';
import { analyzeThread, quickAnalyze } from './analyzer.js';
import { getRedditRateLimiter, retryWithBackoff } from '../utils/rateLimiter.js';
import type { GoldmineScore, MarketInsight } from '../types.js';
export interface ScanTask {
id: string;
subreddit: string;
query?: string;
maxThreads?: number;
useClaudeAnalysis?: boolean;
}
export interface ScanResult {
taskId: string;
subreddit: string;
query?: string;
threads: GoldmineScore[];
insights: MarketInsight[];
errors: string[];
duration: number;
}
/**
* Worker function to scan a single subreddit
*/
async function scanSubredditWorker(task: ScanTask): Promise<ScanResult> {
const startTime = Date.now();
const rateLimiter = getRedditRateLimiter();
const result: ScanResult = {
taskId: task.id,
subreddit: task.subreddit,
query: task.query,
threads: [],
insights: [],
errors: [],
duration: 0
};
try {
// Find goldmine threads with rate limiting
await rateLimiter.acquireWithBackoff();
const threads = task.query
? await retryWithBackoff(() => searchSubreddit(task.subreddit, task.query!, { limit: task.maxThreads || 20 }))
: await retryWithBackoff(() => findGoldmineThreads(task.subreddit));
rateLimiter.reportSuccess();
result.threads = threads;
// Analyze top threads
const threadsToAnalyze = threads
.filter(t => t.goldmineScore >= 15)
.slice(0, 5);
for (const thread of threadsToAnalyze) {
try {
await rateLimiter.acquireWithBackoff();
const threadData = await retryWithBackoff(() => fetchThread(thread.url));
rateLimiter.reportSuccess();
let insight: MarketInsight | null = null;
if (task.useClaudeAnalysis && process.env.ANTHROPIC_API_KEY) {
insight = await analyzeThread(threadData);
} else {
// Quick analysis fallback
const quick = quickAnalyze(threadData);
insight = {
id: crypto.randomUUID(),
threadUrl: thread.url,
subreddit: thread.subreddit,
extractedAt: new Date().toISOString(),
painPoints: [],
opportunities: [],
competitorMentions: quick.competitorMentions || [],
buyingSignals: quick.buyingSignals || [],
keyQuotes: [],
summary: `Quick analysis of ${thread.title.slice(0, 50)}...`
};
}
if (insight) {
result.insights.push(insight);
}
} catch (error) {
result.errors.push(`Failed to analyze ${thread.url}: ${error}`);
rateLimiter.reportError();
}
}
} catch (error) {
result.errors.push(`Failed to scan r/${task.subreddit}: ${error}`);
rateLimiter.reportError();
}
result.duration = Date.now() - startTime;
return result;
}
/**
* Multi-Agent Scanner
* Coordinates parallel scanning of multiple subreddits
*/
export class MultiAgentScanner {
private maxConcurrency: number;
private verbose: boolean;
constructor(options: { maxConcurrency?: number; verbose?: boolean } = {}) {
this.maxConcurrency = options.maxConcurrency || 3;
this.verbose = options.verbose ?? false;
}
/**
* Run tasks in parallel with concurrency limit
*/
async runParallel(tasks: ScanTask[]): Promise<ScanResult[]> {
const results: ScanResult[] = [];
const queue = [...tasks];
const running: Promise<void>[] = [];
const runTask = async (task: ScanTask): Promise<void> => {
if (this.verbose) {
console.log(`[Agent] Starting scan: r/${task.subreddit}${task.query ? ` "${task.query}"` : ''}`);
}
const result = await scanSubredditWorker(task);
results.push(result);
if (this.verbose) {
console.log(`[Agent] Completed: r/${task.subreddit} - ${result.threads.length} threads, ${result.insights.length} insights (${result.duration}ms)`);
if (result.errors.length > 0) {
console.log(`[Agent] Errors: ${result.errors.length}`);
}
}
};
// Process queue with concurrency limit
while (queue.length > 0 || running.length > 0) {
// Start new tasks up to concurrency limit
while (running.length < this.maxConcurrency && queue.length > 0) {
const task = queue.shift()!;
const promise = runTask(task).then(() => {
const index = running.indexOf(promise);
if (index > -1) running.splice(index, 1);
});
running.push(promise);
}
// Wait for at least one to complete
if (running.length > 0) {
await Promise.race(running);
}
}
return results;
}
/**
* Scan multiple subreddits
*/
async scanSubreddits(
subreddits: string[],
options: { queries?: string[]; useClaudeAnalysis?: boolean } = {}
): Promise<ScanResult[]> {
const tasks: ScanTask[] = [];
for (const subreddit of subreddits) {
if (options.queries && options.queries.length > 0) {
// Create task for each query
for (const query of options.queries) {
tasks.push({
id: `${subreddit}-${query}`,
subreddit,
query,
useClaudeAnalysis: options.useClaudeAnalysis
});
}
} else {
// Single task without query
tasks.push({
id: subreddit,
subreddit,
useClaudeAnalysis: options.useClaudeAnalysis
});
}
}
return this.runParallel(tasks);
}
/**
* Get aggregated stats from results
*/
getAggregatedStats(results: ScanResult[]): {
totalThreads: number;
totalInsights: number;
totalErrors: number;
totalDuration: number;
bySubreddit: Record<string, { threads: number; insights: number }>;
topThreads: GoldmineScore[];
} {
const stats = {
totalThreads: 0,
totalInsights: 0,
totalErrors: 0,
totalDuration: 0,
bySubreddit: {} as Record<string, { threads: number; insights: number }>,
topThreads: [] as GoldmineScore[]
};
for (const result of results) {
stats.totalThreads += result.threads.length;
stats.totalInsights += result.insights.length;
stats.totalErrors += result.errors.length;
stats.totalDuration = Math.max(stats.totalDuration, result.duration);
stats.bySubreddit[result.subreddit] = stats.bySubreddit[result.subreddit] || { threads: 0, insights: 0 };
stats.bySubreddit[result.subreddit].threads += result.threads.length;
stats.bySubreddit[result.subreddit].insights += result.insights.length;
stats.topThreads.push(...result.threads);
}
// Sort and dedupe top threads
const seenUrls = new Set<string>();
stats.topThreads = stats.topThreads
.filter(t => {
if (seenUrls.has(t.url)) return false;
seenUrls.add(t.url);
return true;
})
.sort((a, b) => b.goldmineScore - a.goldmineScore)
.slice(0, 50);
return stats;
}
}
/**
* Create multi-agent scanner
*/
export function createMultiAgentScanner(options?: { maxConcurrency?: number; verbose?: boolean }): MultiAgentScanner {
return new MultiAgentScanner(options);
}