Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(cloro): add webhook callback mode for scraper results

Gökhan Geyik053470b502e6
diff --git a/server/.env.example b/server/.env.example
index 6b7260867..d1558ac2e 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -31,6 +31,12 @@ DATAFORSEO_PASSWORD=
# Cloro (platform web scrapers)
CLORO_API_KEY=
+# Cloro webhook callback URL (publicly reachable, e.g. https://api.example.com/cloro/callback).
+# When set, tracking jobs submit tasks with this webhook and DO NOT poll inline —
+# Cloro pushes results to /cloro/callback as they complete (survives server restarts
+# and scales without blocking workers). Leave empty to fall back to inline polling.
+CLORO_WEBHOOK_URL=
+
# Platform provider for web scrapers (default: cloro)
PLATFORM_PROVIDER=cloro
diff --git a/server/src/lib/cloro-result-handler.js b/server/src/lib/cloro-result-handler.js
new file mode 100644
index 000000000..abd2df869
--- /dev/null
+++ b/server/src/lib/cloro-result-handler.js
@@ -0,0 +1,74 @@
+/**
+ * Centralized handler for a Cloro scraper result.
+ *
+ * Used by both:
+ * - The polling fallback path in tracking-worker.js
+ * - The /cloro/callback webhook endpoint
+ *
+ * Given a parsed Cloro AI response and the prompt/brand context, this:
+ * 1. Counts brand mentions in the response text
+ * 2. Runs sentiment analysis (only if brand was mentioned)
+ * 3. Computes visibility metrics + competitor mentions via parseResponse
+ * 4. Inserts a `prompt_results` row
+ */
+
+import supabaseAdmin from '../config/supabase.js';
+import { analyzeSentimentAI } from './ai-tracker.js';
+import { parseResponse, countBrandMentions } from './response-parser.js';
+
+/**
+ * @param {object} args
+ * @param {{ text: string, citations: Array, model: string }} args.aiResponse
+ * Already parsed via parseScraperResponse (cloro-scraper.js)
+ * @param {string} args.scraperId Cloro scraper id (e.g. 'chatgpt-web')
+ * @param {string} args.promptId
+ * @param {string} args.brandId
+ * @param {string|null} args.region
+ * @param {{ brandName: string, domains: string[] }} args.brandInfo
+ * @param {Array<{ id: string, name: string, domain: string }>} args.competitors
+ * @returns {Promise<{ inserted: boolean }>}
+ */
+export async function handleScraperResult({
+ aiResponse,
+ scraperId,
+ promptId,
+ brandId,
+ region,
+ brandInfo,
+ competitors,
+}) {
+ const mentionCount = countBrandMentions(aiResponse.text, brandInfo);
+ const sentimentResult =
+ mentionCount > 0
+ ? await analyzeSentimentAI(aiResponse.text, brandInfo.brandName)
+ : { sentiment: 'neutral', confidence: 0, reason: 'Brand not mentioned' };
+
+ const metrics = parseResponse(
+ aiResponse,
+ brandInfo,
+ sentimentResult.sentiment,
+ competitors,
+ );
+
+ const { error } = await supabaseAdmin.from('prompt_results').insert({
+ prompt_id: promptId,
+ brand_id: brandId,
+ platform: scraperId,
+ response: aiResponse.text,
+ citations: aiResponse.citations,
+ mention_count: metrics.mentionCount,
+ citation_count: metrics.citationCount,
+ sentiment: sentimentResult.sentiment,
+ visibility_score: metrics.visibilityScore,
+ model_used: aiResponse.model,
+ region: region ?? null,
+ competitor_mentions: metrics.competitorMentions,
+ });
+
+ if (error) {
+ console.error('[cloro-result] Failed to insert result:', error.message);
+ throw error;
+ }
+
+ return { inserted: true };
+}
diff --git a/server/src/lib/cloro-scraper.js b/server/src/lib/cloro-scraper.js
index 1d016278d..9befaa28e 100644
--- a/server/src/lib/cloro-scraper.js
+++ b/server/src/lib/cloro-scraper.js
@@ -65,7 +65,7 @@ function buildRequestBody(promptText, scraperId, region) {
};
}
-function parseResponse(result, scraperId) {
+export function parseScraperResponse(result, scraperId) {
if (scraperId === 'google-aio') {
const aio = result.aioverview;
if (!aio) {
@@ -113,9 +113,10 @@ function parseResponse(result, scraperId) {
* @param {string} promptText
* @param {string} scraperId
* @param {string} [region]
+ * @param {{ webhookUrl?: string }} [opts]
* @returns {Promise<{ taskId: string, scraperId: string }>}
*/
-export async function submitScraperTask(promptText, scraperId, region) {
+export async function submitScraperTask(promptText, scraperId, region, opts = {}) {
const taskType = SCRAPER_TASK_TYPES[scraperId];
if (!taskType) throw new Error(`Unknown scraper: ${scraperId}`);
@@ -125,13 +126,18 @@ export async function submitScraperTask(promptText, scraperId, region) {
`[cloro] Submitting ${taskType} task | scraper=${scraperId} region=${region || 'US'} prompt="${promptText.slice(0, 60)}..."`,
);
+ const requestBody = { taskType, payload };
+ if (opts.webhookUrl) {
+ requestBody.webhook = { url: opts.webhookUrl };
+ }
+
const res = await fetch(`${CLORO_API}/v1/async/task`, {
method: 'POST',
headers: {
Authorization: `Bearer ${getApiKey()}`,
'Content-Type': 'application/json',
},
- body: JSON.stringify({ taskType, payload }),
+ body: JSON.stringify(requestBody),
});
if (!res.ok) {
@@ -209,7 +215,7 @@ export async function pollScraperResult(taskId, scraperId, opts = {}) {
throw new Error(
`Cloro task ${taskId} completed but returned no response`,
);
- return parseResponse(result, scraperId);
+ return parseScraperResponse(result, scraperId);
}
if (status === 'FAILED') {
diff --git a/server/src/server.js b/server/src/server.js
index d677381f7..a9abedb38 100644
--- a/server/src/server.js
+++ b/server/src/server.js
@@ -13,6 +13,8 @@ import routes from './routes/index.js';
import trafficRoutes from './routes/traffic.js';
import { createJob, cleanupStaleJobs, cleanupOldJobs } from './lib/job-manager.js';
import { runTrackingJob } from './lib/job-runner.js';
+import { parseScraperResponse } from './lib/cloro-scraper.js';
+import { handleScraperResult } from './lib/cloro-result-handler.js';
import supabaseAdmin from './config/supabase.js';
import { getPlan, hasFeature, isCloud } from './config/plans.js';
@@ -181,6 +183,118 @@ app.post('/api/internal/trigger-tracking', async (req, res) => {
}
});
+// --- Cloro webhook callback (public, no auth — security via task_id entropy + DB lookup) ---
+app.post('/cloro/callback', async (req, res) => {
+ // Always ack quickly so Cloro doesn't retry on slow processing
+ res.status(200).send();
+
+ try {
+ const { task, response } = req.body || {};
+ const taskId = task?.id;
+ const status = task?.status;
+
+ if (!taskId) {
+ console.warn('[cloro/callback] Missing task.id in payload');
+ return;
+ }
+
+ const { data: pending } = await supabaseAdmin
+ .from('cloro_pending_tasks')
+ .select('*')
+ .eq('task_id', taskId)
+ .maybeSingle();
+
+ if (!pending) {
+ console.log(`[cloro/callback] No pending task for ${taskId} (already processed?)`);
+ return;
+ }
+
+ if (status === 'FAILED') {
+ console.error(`[cloro/callback] Task ${taskId} (${pending.scraper_id}) FAILED`);
+ await supabaseAdmin
+ .from('cloro_pending_tasks')
+ .delete()
+ .eq('task_id', taskId);
+ return;
+ }
+
+ if (status !== 'COMPLETED') {
+ console.log(`[cloro/callback] Task ${taskId} status=${status}, ignoring`);
+ return;
+ }
+
+ if (!response) {
+ console.error(`[cloro/callback] Task ${taskId} COMPLETED but missing response`);
+ await supabaseAdmin
+ .from('cloro_pending_tasks')
+ .delete()
+ .eq('task_id', taskId);
+ return;
+ }
+
+ // Fetch context for result handler
+ const [
+ { data: brand },
+ { data: domains },
+ { data: competitorRows },
+ ] = await Promise.all([
+ supabaseAdmin
+ .from('brands')
+ .select('id, name')
+ .eq('id', pending.brand_id)
+ .single(),
+ supabaseAdmin
+ .from('brand_domains')
+ .select('domain')
+ .eq('brand_id', pending.brand_id),
+ supabaseAdmin
+ .from('competitors')
+ .select('id, name, domain')
+ .eq('brand_id', pending.brand_id),
+ ]);
+
+ if (!brand) {
+ console.error(`[cloro/callback] Brand ${pending.brand_id} for task ${taskId} not found — dropping`);
+ await supabaseAdmin
+ .from('cloro_pending_tasks')
+ .delete()
+ .eq('task_id', taskId);
+ return;
+ }
+
+ const brandInfo = {
+ brandName: brand.name,
+ domains: (domains || []).map((d) => d.domain),
+ };
+ const competitors = (competitorRows || []).map((c) => ({
+ id: c.id,
+ name: c.name,
+ domain: c.domain || '',
+ }));
+
+ const aiResponse = parseScraperResponse(response, pending.scraper_id);
+
+ await handleScraperResult({
+ aiResponse,
+ scraperId: pending.scraper_id,
+ promptId: pending.prompt_id,
+ brandId: pending.brand_id,
+ region: pending.region,
+ brandInfo,
+ competitors,
+ });
+
+ await supabaseAdmin
+ .from('cloro_pending_tasks')
+ .delete()
+ .eq('task_id', taskId);
+
+ console.log(`[cloro/callback] Task ${taskId} (${pending.scraper_id}) processed and inserted`);
+ } catch (err) {
+ console.error('[cloro/callback] Error processing webhook:', err?.message || err);
+ }
+});
+
// --- Authenticated API routes ---
app.use('/api', Middleware.decodeToken.bind(Middleware), routes);
diff --git a/server/src/workers/tracking-worker.js b/server/src/workers/tracking-worker.js
index 0b6d730e4..c48e83ae8 100644
--- a/server/src/workers/tracking-worker.js
+++ b/server/src/workers/tracking-worker.js
@@ -122,8 +122,12 @@ export async function processTrackingJob({ brandId, promptId, promptIds, job })
}
}
+ const webhookUrl = process.env.CLORO_WEBHOOK_URL;
+
if (scraperTasks.length > 0) {
- console.log(`[tracking] Submitting ${scraperTasks.length} scraper tasks to Cloro...`);
+ console.log(
+ `[tracking] Submitting ${scraperTasks.length} scraper tasks to Cloro (mode=${webhookUrl ? 'webhook' : 'polling'})...`,
+ );
if (job) {
job.progress({
@@ -138,7 +142,7 @@ export async function processTrackingJob({ brandId, promptId, promptIds, job })
// Submit all tasks concurrently
const submissions = await Promise.allSettled(
scraperTasks.map((t) =>
- submitScraperTask(t.prompt.text, t.scraperId, t.region)
+ submitScraperTask(t.prompt.text, t.scraperId, t.region, { webhookUrl })
.then((res) => ({ ...res, meta: t }))
),
);
@@ -158,56 +162,97 @@ export async function processTrackingJob({ brandId, promptId, promptIds, job })
}
}
- console.log(`[tracking] ${submitted.length}/${scraperTasks.length} tasks submitted, polling for results...`);
-
- // Poll + parse + insert each task independently (all in parallel)
- await Promise.allSettled(
- submitted.map(async ({ taskId, scraperId, meta }) => {
- try {
- console.log(`[tracking] Polling task=${taskId} scraper=${scraperId}...`);
- const aiResponse = await pollScraperResult(taskId, scraperId);
- console.log(`[tracking] Task=${taskId} scraper=${scraperId} completed, inserting result...`);
-
- const mentionCount = countBrandMentions(aiResponse.text, brandInfo);
- const sentimentResult = mentionCount > 0
- ? await analyzeSentimentAI(aiResponse.text, brandInfo.brandName)
- : { sentiment: 'neutral', confidence: 0, reason: 'Brand not mentioned' };
- const metrics = parseResponse(aiResponse, brandInfo, sentimentResult.sentiment, competitors);
-
- await insertResult({
- prompt_id: meta.prompt.id,
- brand_id: brandId,
- platform: meta.scraperId,
- response: aiResponse.text,
- citations: aiResponse.citations,
- mention_count: metrics.mentionCount,
- citation_count: metrics.citationCount,
- sentiment: metrics.sentiment,
- visibility_score: metrics.visibilityScore,
- model_used: aiResponse.model,
- region: meta.region,
- competitor_mentions: metrics.competitorMentions,
- });
-
- console.log(`[tracking] Task=${taskId} scraper=${scraperId} result saved.`);
- } catch (err) {
+ if (webhookUrl) {
+ // Webhook mode: persist (taskId → prompt) mapping; the /cloro/callback
+ // endpoint will pick up results asynchronously when Cloro pushes them.
+ if (submitted.length > 0) {
+ const pendingRows = submitted.map(({ taskId, scraperId, meta }) => ({
+ task_id: taskId,
+ prompt_id: meta.prompt.id,
+ brand_id: brandId,
+ scraper_id: scraperId,
+ region: meta.region,
+ }));
+
+ const { error: pendingErr } = await supabaseAdmin
+ .from('cloro_pending_tasks')
+ .insert(pendingRows);
+
+ if (pendingErr) {
console.error(
- `[tracking] Task=${taskId} scraper=${scraperId} failed: ${err.message}`,
+ '[tracking] Failed to record pending Cloro tasks — webhook results will be dropped:',
+ pendingErr.message,
+ );
+ } else {
+ console.log(
+ `[tracking] ${submitted.length} pending tasks recorded. Webhook will deliver results.`,
);
}
+ }
- completedTasks++;
- if (job) {
- job.progress({
- current: completedTasks,
- total: totalTasks,
- promptText: meta.prompt.text.slice(0, 80),
- model: scraperId,
- platform: 'cloro',
- });
- }
- }),
- );
+ // Mark these as "completed" for progress purposes (job is done submitting).
+ completedTasks += submitted.length;
+ if (job && submitted.length > 0) {
+ job.progress({
+ current: completedTasks,
+ total: totalTasks,
+ promptText: 'All platform tasks queued — results streaming via webhook',
+ model: null,
+ platform: 'cloro',
+ });
+ }
+ } else {
+ console.log(`[tracking] ${submitted.length}/${scraperTasks.length} tasks submitted, polling for results...`);
+
+ // Polling fallback: wait for each task inline (legacy behavior)
+ await Promise.allSettled(
+ submitted.map(async ({ taskId, scraperId, meta }) => {
+ try {
+ console.log(`[tracking] Polling task=${taskId} scraper=${scraperId}...`);
+ const aiResponse = await pollScraperResult(taskId, scraperId);
+ console.log(`[tracking] Task=${taskId} scraper=${scraperId} completed, inserting result...`);
+
+ const mentionCount = countBrandMentions(aiResponse.text, brandInfo);
+ const sentimentResult = mentionCount > 0
+ ? await analyzeSentimentAI(aiResponse.text, brandInfo.brandName)
+ : { sentiment: 'neutral', confidence: 0, reason: 'Brand not mentioned' };
+ const metrics = parseResponse(aiResponse, brandInfo, sentimentResult.sentiment, competitors);
+
+ await insertResult({
+ prompt_id: meta.prompt.id,
+ brand_id: brandId,
+ platform: meta.scraperId,
+ response: aiResponse.text,
+ citations: aiResponse.citations,
+ mention_count: metrics.mentionCount,
+ citation_count: metrics.citationCount,
+ sentiment: metrics.sentiment,
+ visibility_score: metrics.visibilityScore,
+ model_used: aiResponse.model,
+ region: meta.region,
+ competitor_mentions: metrics.competitorMentions,
+ });
+
+ console.log(`[tracking] Task=${taskId} scraper=${scraperId} result saved.`);
+ } catch (err) {
+ console.error(
+ `[tracking] Task=${taskId} scraper=${scraperId} failed: ${err.message}`,
+ );
+ }
+
+ completedTasks++;
+ if (job) {
+ job.progress({
+ current: completedTasks,
+ total: totalTasks,
+ promptText: meta.prompt.text.slice(0, 80),
+ model: scraperId,
+ platform: 'cloro',
+ });
+ }
+ }),
+ );
+ }
}
// 7. Phase 2: Run AI model tasks concurrently