Skip to content
Shell
9.6 KB
Raw Blame
/**
 * Reddit Research Agent Runner
 * Orchestrates the scanning, analysis, and storage pipeline
 */

import cron from 'node-cron';
import { findGoldmineThreads, fetchThread, searchSubreddit } from '../reddit/index.js';
import { analyzeThread, quickAnalyze } from './analyzer.js';
import {
  initDatabase,
  saveThreads,
  getUnanalyzedThreads,
  markThreadAnalyzed,
  saveInsight,
  startAgentRun,
  completeAgentRun,
  failAgentRun,
  getTrackedSources,
  addTrackedSource,
  updateSourceScanned,
  getStats,
} from '../db/index.js';
import { AlertManager, createAlertManager } from '../alerts/index.js';
import { NotionSync, createNotionSync } from '../sync/index.js';
import { getRedditRateLimiter, retryWithBackoff } from '../utils/rateLimiter.js';
import type { ScanConfig, GoldmineScore, MarketInsight } from '../types.js';

export interface AgentOptions {
  dbPath?: string;
  maxThreadsPerScan?: number;
  minGoldmineScore?: number;
  useClaudeAnalysis?: boolean;
  verbose?: boolean;
  enableAlerts?: boolean;
  enableNotionSync?: boolean;
  alertThreshold?: number; // Goldmine score threshold for alerts
}

const DEFAULT_OPTIONS: Required<AgentOptions> = {
  dbPath: './reddit-research.db',
  maxThreadsPerScan: 20,
  minGoldmineScore: 15,
  useClaudeAnalysis: true,
  verbose: true,
  enableAlerts: true,
  enableNotionSync: true,
  alertThreshold: 25, // Only alert on high-value threads
};

/**
 * Log with timestamp if verbose
 */
function log(message: string, verbose: boolean): void {
  if (verbose) {
    console.log(`[${new Date().toISOString()}] ${message}`);
  }
}

/**
 * Run a single scan cycle
 */
export async function runScanCycle(options: AgentOptions = {}): Promise<{
  threadsScanned: number;
  insightsFound: number;
  costUsd: number;
}> {
  const opts = { ...DEFAULT_OPTIONS, ...options };
  const startTime = Date.now();
  const rateLimiter = getRedditRateLimiter();

  // Initialize services
  initDatabase(opts.dbPath);
  const runId = startAgentRun();

  // Initialize optional services
  const alertManager = opts.enableAlerts ? createAlertManager({ verbose: opts.verbose }) : null;
  const notionSync = opts.enableNotionSync ? createNotionSync({ verbose: opts.verbose }) : null;

  log('Starting scan cycle...', opts.verbose);
  if (alertManager?.isConfigured()) {
    log('Alerts enabled', opts.verbose);
  }
  if (notionSync?.isConfigured()) {
    log('Notion sync enabled', opts.verbose);
  }

  let threadsScanned = 0;
  let insightsFound = 0;
  let costUsd = 0;
  const highValueThreads: GoldmineScore[] = [];

  try {
    // Get tracked sources
    let sources = getTrackedSources();

    // Add defaults if no sources configured
    if (sources.length === 0) {
      log('No tracked sources. Adding defaults...', opts.verbose);
      addTrackedSource('SaaS');
      addTrackedSource('smallbusiness');
      addTrackedSource('entrepreneur');
      sources = getTrackedSources();
    }

    // Scan each source with rate limiting
    for (const source of sources) {
      log(`Scanning r/${source.subreddit}${source.query ? ` for "${source.query}"` : ''}...`, opts.verbose);

      try {
        // Apply rate limiting
        await rateLimiter.acquireWithBackoff();

        const threads = await retryWithBackoff(async () => {
          return source.query
            ? await searchSubreddit(source.subreddit, source.query!, { limit: opts.maxThreadsPerScan })
            : await findGoldmineThreads(source.subreddit);
        }, {
          onRetry: (error, attempt) => {
            log(`Retry ${attempt} for ${source.subreddit}: ${error.message}`, opts.verbose);
            rateLimiter.reportError();
          }
        });

        rateLimiter.reportSuccess();

        // Filter by goldmine score
        const goldmineThreads = threads.filter(t => t.goldmineScore >= opts.minGoldmineScore);
        log(`Found ${goldmineThreads.length} goldmine threads (score >= ${opts.minGoldmineScore})`, opts.verbose);

        // Save to database
        saveThreads(goldmineThreads);
        updateSourceScanned(source.subreddit, source.query || undefined);

        threadsScanned += goldmineThreads.length;

        // Track high-value threads for alerts
        const highValue = goldmineThreads.filter(t => t.goldmineScore >= opts.alertThreshold);
        highValueThreads.push(...highValue);

        // Alert on high-value threads immediately
        if (alertManager?.isConfigured()) {
          for (const thread of highValue) {
            await alertManager.alertThread(thread);
          }
        }

        // Sync to Notion
        if (notionSync?.isConfigured() && goldmineThreads.length > 0) {
          await notionSync.syncThreads(goldmineThreads);
        }
      } catch (error) {
        log(`Error scanning ${source.subreddit}: ${error}`, opts.verbose);
        rateLimiter.reportError();
      }
    }

    // Analyze unanalyzed threads
    const unanalyzed = getUnanalyzedThreads(5); // Analyze top 5 at a time
    log(`Analyzing ${unanalyzed.length} threads...`, opts.verbose);

    for (const { id, url, goldmineScore } of unanalyzed) {
      try {
        // Rate limit thread fetching
        await rateLimiter.acquireWithBackoff();

        log(`Fetching thread: ${url}`, opts.verbose);
        const threadData = await retryWithBackoff(() => fetchThread(url), {
          onRetry: (error, attempt) => {
            log(`Retry ${attempt} fetching ${url}: ${error.message}`, opts.verbose);
            rateLimiter.reportError();
          }
        });

        rateLimiter.reportSuccess();

        let insights: Partial<MarketInsight>;

        if (opts.useClaudeAnalysis) {
          log('Analyzing with Claude...', opts.verbose);
          insights = await analyzeThread(threadData);
          costUsd += 0.01; // Rough estimate per analysis
        } else {
          log('Using quick pattern analysis...', opts.verbose);
          insights = quickAnalyze(threadData);
        }

        // Save insights to database
        for (const painPoint of insights.painPoints || []) {
          saveInsight(id, 'pain_point', painPoint.description, {
            frequency: painPoint.frequency,
            sentiment: painPoint.sentiment,
            quotes: painPoint.quotes,
          }, 'medium');
          insightsFound++;
        }

        for (const opportunity of insights.opportunities || []) {
          saveInsight(id, 'opportunity', opportunity.description, {
            type: opportunity.type,
            targetAudience: opportunity.targetAudience,
          }, opportunity.confidence);
          insightsFound++;
        }

        for (const competitor of insights.competitorMentions || []) {
          saveInsight(id, 'competitor', competitor.name, {
            sentiment: competitor.sentiment,
            context: competitor.context,
            frequency: competitor.frequency,
          });
          insightsFound++;
        }

        for (const signal of insights.buyingSignals || []) {
          saveInsight(id, 'buying_signal', signal.quote, {
            author: signal.author,
            signalType: signal.signalType,
            score: signal.score,
          });
          insightsFound++;
        }

        for (const quote of insights.keyQuotes || []) {
          saveInsight(id, 'quote', quote.text, {
            author: quote.author,
            score: quote.score,
            insightType: quote.insightType,
          });
          insightsFound++;
        }

        if (insights.summary) {
          saveInsight(id, 'summary', insights.summary);
        }

        markThreadAnalyzed(id);
        log(`Thread analyzed: ${insightsFound} insights found`, opts.verbose);

      } catch (error) {
        log(`Error analyzing thread ${url}: ${error}`, opts.verbose);
      }
    }

    // Complete the run
    completeAgentRun(runId, { threadsScanned, insightsFound, costUsd });
    const duration = Date.now() - startTime;
    log(`Scan cycle complete. Threads: ${threadsScanned}, Insights: ${insightsFound}, Duration: ${Math.round(duration / 1000)}s`, opts.verbose);

    // Send scan summary alert
    if (alertManager?.isConfigured()) {
      await alertManager.alertScanSummary({
        threadsScanned,
        insightsFound,
        topThreads: highValueThreads.slice(0, 5),
        duration,
      });
    }

    return { threadsScanned, insightsFound, costUsd };

  } catch (error) {
    failAgentRun(runId, String(error));
    throw error;
  }
}

/**
 * Start scheduled agent
 */
export function startScheduledAgent(
  schedule: string = '0 9 * * *', // Daily at 9 AM
  options: AgentOptions = {}
): void {
  const opts = { ...DEFAULT_OPTIONS, ...options };

  log(`Starting scheduled agent with cron: ${schedule}`, opts.verbose);

  // Initialize database
  initDatabase(opts.dbPath);

  // Run immediately on start
  runScanCycle(options).catch(console.error);

  // Schedule future runs
  cron.schedule(schedule, async () => {
    log('Scheduled scan triggered', opts.verbose);
    try {
      await runScanCycle(options);
    } catch (error) {
      console.error('Scheduled scan failed:', error);
    }
  });

  log('Agent started. Press Ctrl+C to stop.', opts.verbose);
}

/**
 * Print agent stats
 */
export function printStats(): void {
  const stats = getStats();

  console.log('\n📊 Reddit Research Agent Stats\n');
  console.log(`Threads Scanned: ${stats.totalThreads}`);
  console.log(`Threads Analyzed: ${stats.analyzedThreads}`);
  console.log(`Total Insights: ${stats.totalInsights}`);
  console.log('\nInsights by Type:');
  for (const [type, count] of Object.entries(stats.insightsByType)) {
    console.log(`  ${type}: ${count}`);
  }
  console.log('\nTop Subreddits:');
  for (const { subreddit, count } of stats.topSubreddits) {
    console.log(`  r/${subreddit}: ${count} threads`);
  }
}