Shell
/**
* Reddit Research Agent Runner
* Orchestrates the scanning, analysis, and storage pipeline
*/
import cron from 'node-cron';
import { findGoldmineThreads, fetchThread, searchSubreddit } from '../reddit/index.js';
import { analyzeThread, quickAnalyze } from './analyzer.js';
import {
initDatabase,
saveThreads,
getUnanalyzedThreads,
markThreadAnalyzed,
saveInsight,
startAgentRun,
completeAgentRun,
failAgentRun,
getTrackedSources,
addTrackedSource,
updateSourceScanned,
getStats,
} from '../db/index.js';
import { AlertManager, createAlertManager } from '../alerts/index.js';
import { NotionSync, createNotionSync } from '../sync/index.js';
import { getRedditRateLimiter, retryWithBackoff } from '../utils/rateLimiter.js';
import type { ScanConfig, GoldmineScore, MarketInsight } from '../types.js';
export interface AgentOptions {
dbPath?: string;
maxThreadsPerScan?: number;
minGoldmineScore?: number;
useClaudeAnalysis?: boolean;
verbose?: boolean;
enableAlerts?: boolean;
enableNotionSync?: boolean;
alertThreshold?: number; // Goldmine score threshold for alerts
}
const DEFAULT_OPTIONS: Required<AgentOptions> = {
dbPath: './reddit-research.db',
maxThreadsPerScan: 20,
minGoldmineScore: 15,
useClaudeAnalysis: true,
verbose: true,
enableAlerts: true,
enableNotionSync: true,
alertThreshold: 25, // Only alert on high-value threads
};
/**
* Log with timestamp if verbose
*/
function log(message: string, verbose: boolean): void {
if (verbose) {
console.log(`[${new Date().toISOString()}] ${message}`);
}
}
/**
* Run a single scan cycle
*/
export async function runScanCycle(options: AgentOptions = {}): Promise<{
threadsScanned: number;
insightsFound: number;
costUsd: number;
}> {
const opts = { ...DEFAULT_OPTIONS, ...options };
const startTime = Date.now();
const rateLimiter = getRedditRateLimiter();
// Initialize services
initDatabase(opts.dbPath);
const runId = startAgentRun();
// Initialize optional services
const alertManager = opts.enableAlerts ? createAlertManager({ verbose: opts.verbose }) : null;
const notionSync = opts.enableNotionSync ? createNotionSync({ verbose: opts.verbose }) : null;
log('Starting scan cycle...', opts.verbose);
if (alertManager?.isConfigured()) {
log('Alerts enabled', opts.verbose);
}
if (notionSync?.isConfigured()) {
log('Notion sync enabled', opts.verbose);
}
let threadsScanned = 0;
let insightsFound = 0;
let costUsd = 0;
const highValueThreads: GoldmineScore[] = [];
try {
// Get tracked sources
let sources = getTrackedSources();
// Add defaults if no sources configured
if (sources.length === 0) {
log('No tracked sources. Adding defaults...', opts.verbose);
addTrackedSource('SaaS');
addTrackedSource('smallbusiness');
addTrackedSource('entrepreneur');
sources = getTrackedSources();
}
// Scan each source with rate limiting
for (const source of sources) {
log(`Scanning r/${source.subreddit}${source.query ? ` for "${source.query}"` : ''}...`, opts.verbose);
try {
// Apply rate limiting
await rateLimiter.acquireWithBackoff();
const threads = await retryWithBackoff(async () => {
return source.query
? await searchSubreddit(source.subreddit, source.query!, { limit: opts.maxThreadsPerScan })
: await findGoldmineThreads(source.subreddit);
}, {
onRetry: (error, attempt) => {
log(`Retry ${attempt} for ${source.subreddit}: ${error.message}`, opts.verbose);
rateLimiter.reportError();
}
});
rateLimiter.reportSuccess();
// Filter by goldmine score
const goldmineThreads = threads.filter(t => t.goldmineScore >= opts.minGoldmineScore);
log(`Found ${goldmineThreads.length} goldmine threads (score >= ${opts.minGoldmineScore})`, opts.verbose);
// Save to database
saveThreads(goldmineThreads);
updateSourceScanned(source.subreddit, source.query || undefined);
threadsScanned += goldmineThreads.length;
// Track high-value threads for alerts
const highValue = goldmineThreads.filter(t => t.goldmineScore >= opts.alertThreshold);
highValueThreads.push(...highValue);
// Alert on high-value threads immediately
if (alertManager?.isConfigured()) {
for (const thread of highValue) {
await alertManager.alertThread(thread);
}
}
// Sync to Notion
if (notionSync?.isConfigured() && goldmineThreads.length > 0) {
await notionSync.syncThreads(goldmineThreads);
}
} catch (error) {
log(`Error scanning ${source.subreddit}: ${error}`, opts.verbose);
rateLimiter.reportError();
}
}
// Analyze unanalyzed threads
const unanalyzed = getUnanalyzedThreads(5); // Analyze top 5 at a time
log(`Analyzing ${unanalyzed.length} threads...`, opts.verbose);
for (const { id, url, goldmineScore } of unanalyzed) {
try {
// Rate limit thread fetching
await rateLimiter.acquireWithBackoff();
log(`Fetching thread: ${url}`, opts.verbose);
const threadData = await retryWithBackoff(() => fetchThread(url), {
onRetry: (error, attempt) => {
log(`Retry ${attempt} fetching ${url}: ${error.message}`, opts.verbose);
rateLimiter.reportError();
}
});
rateLimiter.reportSuccess();
let insights: Partial<MarketInsight>;
if (opts.useClaudeAnalysis) {
log('Analyzing with Claude...', opts.verbose);
insights = await analyzeThread(threadData);
costUsd += 0.01; // Rough estimate per analysis
} else {
log('Using quick pattern analysis...', opts.verbose);
insights = quickAnalyze(threadData);
}
// Save insights to database
for (const painPoint of insights.painPoints || []) {
saveInsight(id, 'pain_point', painPoint.description, {
frequency: painPoint.frequency,
sentiment: painPoint.sentiment,
quotes: painPoint.quotes,
}, 'medium');
insightsFound++;
}
for (const opportunity of insights.opportunities || []) {
saveInsight(id, 'opportunity', opportunity.description, {
type: opportunity.type,
targetAudience: opportunity.targetAudience,
}, opportunity.confidence);
insightsFound++;
}
for (const competitor of insights.competitorMentions || []) {
saveInsight(id, 'competitor', competitor.name, {
sentiment: competitor.sentiment,
context: competitor.context,
frequency: competitor.frequency,
});
insightsFound++;
}
for (const signal of insights.buyingSignals || []) {
saveInsight(id, 'buying_signal', signal.quote, {
author: signal.author,
signalType: signal.signalType,
score: signal.score,
});
insightsFound++;
}
for (const quote of insights.keyQuotes || []) {
saveInsight(id, 'quote', quote.text, {
author: quote.author,
score: quote.score,
insightType: quote.insightType,
});
insightsFound++;
}
if (insights.summary) {
saveInsight(id, 'summary', insights.summary);
}
markThreadAnalyzed(id);
log(`Thread analyzed: ${insightsFound} insights found`, opts.verbose);
} catch (error) {
log(`Error analyzing thread ${url}: ${error}`, opts.verbose);
}
}
// Complete the run
completeAgentRun(runId, { threadsScanned, insightsFound, costUsd });
const duration = Date.now() - startTime;
log(`Scan cycle complete. Threads: ${threadsScanned}, Insights: ${insightsFound}, Duration: ${Math.round(duration / 1000)}s`, opts.verbose);
// Send scan summary alert
if (alertManager?.isConfigured()) {
await alertManager.alertScanSummary({
threadsScanned,
insightsFound,
topThreads: highValueThreads.slice(0, 5),
duration,
});
}
return { threadsScanned, insightsFound, costUsd };
} catch (error) {
failAgentRun(runId, String(error));
throw error;
}
}
/**
* Start scheduled agent
*/
export function startScheduledAgent(
schedule: string = '0 9 * * *', // Daily at 9 AM
options: AgentOptions = {}
): void {
const opts = { ...DEFAULT_OPTIONS, ...options };
log(`Starting scheduled agent with cron: ${schedule}`, opts.verbose);
// Initialize database
initDatabase(opts.dbPath);
// Run immediately on start
runScanCycle(options).catch(console.error);
// Schedule future runs
cron.schedule(schedule, async () => {
log('Scheduled scan triggered', opts.verbose);
try {
await runScanCycle(options);
} catch (error) {
console.error('Scheduled scan failed:', error);
}
});
log('Agent started. Press Ctrl+C to stop.', opts.verbose);
}
/**
* Print agent stats
*/
export function printStats(): void {
const stats = getStats();
console.log('\n📊 Reddit Research Agent Stats\n');
console.log(`Threads Scanned: ${stats.totalThreads}`);
console.log(`Threads Analyzed: ${stats.analyzedThreads}`);
console.log(`Total Insights: ${stats.totalInsights}`);
console.log('\nInsights by Type:');
for (const [type, count] of Object.entries(stats.insightsByType)) {
console.log(` ${type}: ${count}`);
}
console.log('\nTop Subreddits:');
for (const { subreddit, count } of stats.topSubreddits) {
console.log(` r/${subreddit}: ${count} threads`);
}
}