Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(tracking): capture & store observed query fan-out (Cloro searchQueries) (#341)
Gökhan Geyik0cb11b4b7ae8
diff --git a/server/src/lib/cloro-result-handler.js b/server/src/lib/cloro-result-handler.jsindex 5eceedb43..cc77b3191 100644--- a/server/src/lib/cloro-result-handler.js+++ b/server/src/lib/cloro-result-handler.js@@ -21,7 +21,7 @@ import { logger } from './logger.js';/*** @param {object} args- * @param {{ text: string, citations: Array, model: string, shopping_cards: Array }} args.aiResponse+ * @param {{ text: string, citations: Array, model: string, shopping_cards: Array, search_queries?: Array }} args.aiResponse* Already parsed via parseScraperResponse (cloro-scraper.js)* @param {string} args.scraperId Cloro scraper id (e.g. 'chatgpt-web')* @param {string} args.promptId@@ -65,6 +65,7 @@ export async function handleScraperResult({competitor_mentions: metrics.competitorMentions,shopping_cards: aiResponse.shopping_cards ?? [],inline_products: aiResponse.inline_products ?? [],+ search_queries: Array.isArray(aiResponse.search_queries) ? aiResponse.search_queries : [],}).select('id').single();diff --git a/server/src/lib/cloro-scraper.js b/server/src/lib/cloro-scraper.jsindex b4f4d0016..deac0cc5f 100644--- a/server/src/lib/cloro-scraper.js+++ b/server/src/lib/cloro-scraper.js@@ -76,11 +76,49 @@ function buildRequestBody(promptText, scraperId, region) {html: false,markdown: true,rawResponse: false,- searchQueries: false,+ // Opt in to the observed query fan-out (#332). Copilot/Perplexity+ // populate it; ChatGPT surfaces the key but returns it empty in+ // practice — enabling it is free and harmless.+ searchQueries: true,},};}+/**+ * Normalize the observed query fan-out from a raw Cloro response into a rich,+ * engine-labelled array: `[{ query, engine?, source_platform }]` (#332).+ *+ * Two provider shapes:+ * - Perplexity → `search_model_queries: [{ query, engine, limit }]` (keep `engine`)+ * - Copilot / ChatGPT / Grok / Gemini → `searchQueries: string[]`+ * Anything else (or missing) → `[]`. This is OBSERVED data only — we never+ * synthesize sub-queries.+ */+function normalizeSearchQueries(result, scraperId) {+ // Trim on write so the same sub-query never fragments into whitespace+ // variants — the #333 aggregation groups by query string. `engine` is only+ // kept when it's a non-empty string (never a stray non-string value).+ if (Array.isArray(result.search_model_queries)) {+ return result.search_model_queries+ .filter((q) => typeof q?.query === 'string' && q.query.trim() !== '')+ .map((q) => ({+ query: q.query.trim(),+ ...(typeof q.engine === 'string' && q.engine.trim() !== ''+ ? { engine: q.engine.trim() }+ : {}),+ source_platform: scraperId,+ }));+ }++ if (Array.isArray(result.searchQueries)) {+ return result.searchQueries+ .filter((q) => typeof q === 'string' && q.trim() !== '')+ .map((q) => ({ query: q.trim(), source_platform: scraperId }));+ }++ return [];+}+export function parseScraperResponse(result, scraperId) {if (scraperId === 'chatgpt-shopping') {// ChatGPT Shopping returns a normal ChatGPT response shape *plus*@@ -162,6 +200,8 @@ export function parseScraperResponse(result, scraperId) {// .shopping_cards column raw, in their own provider shape — downstream// branches on `platform` to interpret. Other providers have neither key.shopping_cards: result.shopping_cards ?? result.shoppingCards ?? [],+ // Observed query fan-out (#332) — rich, engine-labelled; [] when absent.+ search_queries: normalizeSearchQueries(result, scraperId),};}diff --git a/server/src/lib/cloro-scraper.test.js b/server/src/lib/cloro-scraper.test.jsindex 292deb86c..42a38ba63 100644--- a/server/src/lib/cloro-scraper.test.js+++ b/server/src/lib/cloro-scraper.test.js@@ -169,4 +169,97 @@ describe('cloro-scraper – parseScraperResponse', () => {expect(parsed.shopping_cards).toEqual([]);});});++ describe('search_queries (observed query fan-out, #332)', () => {+ it('maps Copilot string[] searchQueries to rich items tagged with the source platform', () => {+ const result = {+ markdown: 'text',+ sources: [],+ searchQueries: ['best running shoes 2026', 'running shoes flat feet'],+ };+ const parsed = parseScraperResponse(result, 'copilot-web');+ expect(parsed.search_queries).toEqual([+ { query: 'best running shoes 2026', source_platform: 'copilot-web' },+ { query: 'running shoes flat feet', source_platform: 'copilot-web' },+ ]);+ });++ it('preserves the per-item engine label from Perplexity search_model_queries', () => {+ const result = {+ markdown: 'text',+ sources: [],+ search_model_queries: [+ { query: 'best laptops 2026', engine: 'web', limit: 10 },+ { query: 'laptop reviews', engine: 'web' },+ ],+ };+ const parsed = parseScraperResponse(result, 'perplexity-web');+ expect(parsed.search_queries).toEqual([+ { query: 'best laptops 2026', engine: 'web', source_platform: 'perplexity-web' },+ { query: 'laptop reviews', engine: 'web', source_platform: 'perplexity-web' },+ ]);+ });++ it('drops blank / non-string fan-out entries', () => {+ const result = {+ markdown: 'text',+ sources: [],+ searchQueries: ['ok query', ' ', '', 42, null],+ };+ const parsed = parseScraperResponse(result, 'copilot-web');+ expect(parsed.search_queries).toEqual([+ { query: 'ok query', source_platform: 'copilot-web' },+ ]);+ });++ it('trims surrounding whitespace so a sub-query has one canonical form', () => {+ const copilot = parseScraperResponse(+ { markdown: 't', sources: [], searchQueries: [' spaced query '] },+ 'copilot-web',+ );+ expect(copilot.search_queries).toEqual([+ { query: 'spaced query', source_platform: 'copilot-web' },+ ]);++ const perplexity = parseScraperResponse(+ {+ markdown: 't',+ sources: [],+ search_model_queries: [{ query: ' spaced ', engine: ' web ' }],+ },+ 'perplexity-web',+ );+ expect(perplexity.search_queries).toEqual([+ { query: 'spaced', engine: 'web', source_platform: 'perplexity-web' },+ ]);+ });++ it('omits a non-string / empty Perplexity engine label', () => {+ const parsed = parseScraperResponse(+ {+ markdown: 't',+ sources: [],+ search_model_queries: [+ { query: 'no engine', engine: 42 },+ { query: 'blank engine', engine: ' ' },+ ],+ },+ 'perplexity-web',+ );+ expect(parsed.search_queries).toEqual([+ { query: 'no engine', source_platform: 'perplexity-web' },+ { query: 'blank engine', source_platform: 'perplexity-web' },+ ]);+ });++ it('defaults to an empty array when the engine returns none (ChatGPT in practice)', () => {+ expect(+ parseScraperResponse({ markdown: 't', sources: [], searchQueries: [] }, 'chatgpt-web')+ .search_queries,+ ).toEqual([]);+ expect(+ parseScraperResponse({ markdown: 't', sources: [] }, 'gemini-web').search_queries,+ ).toEqual([]);+ });+ });});diff --git a/server/src/workers/tracking-worker.js b/server/src/workers/tracking-worker.jsindex 3a8ddaa8f..a071bc634 100644--- a/server/src/workers/tracking-worker.js+++ b/server/src/workers/tracking-worker.js@@ -309,6 +309,9 @@ export async function processTrackingJob({ brandId, promptId, promptIds, job })model_used: aiResponse.model,region: meta.region,competitor_mentions: metrics.competitorMentions,+ search_queries: Array.isArray(aiResponse.search_queries)+ ? aiResponse.search_queries+ : [],});logger.debug({ taskId, scraperId }, 'scraper task result saved');diff --git a/supabase/migrations/00021_prompt_results_search_queries.sql b/supabase/migrations/00021_prompt_results_search_queries.sqlnew file mode 100644index 000000000..24c5e3df7--- /dev/null+++ b/supabase/migrations/00021_prompt_results_search_queries.sql@@ -0,0 +1,21 @@+-- 00021_prompt_results_search_queries.sql+-- Capture the observed query fan-out on each tracked answer.+--+-- Answer engines (Copilot, Perplexity, and — in principle — ChatGPT) run their+-- own sub-queries to build an answer. Cloro already returns these in the poll /+-- webhook response; we simply had nowhere to store them. This column holds that+-- OBSERVED fan-out (straight from the engine, not an LLM prediction) as a rich,+-- normalized array so the UI can keep the per-item engine label:+--+-- [{ "query": "best running shoes 2026", "engine": "web", "source_platform": "perplexity-web" }]+--+-- (`engine` is present only where the provider labels it — Perplexity today.)+--+-- Existing rows default to an empty array, so nothing changes for historical+-- data; new Copilot/Perplexity results populate it as they land.++ALTER TABLE prompt_results+ ADD COLUMN IF NOT EXISTS search_queries jsonb NOT NULL DEFAULT '[]'::jsonb;++COMMENT ON COLUMN prompt_results.search_queries IS+ 'Observed query fan-out from the answer engine: [{ query, engine?, source_platform }]. Copilot is the primary source; Perplexity secondary (web queries, carries engine); ChatGPT usually empty. Defaults to [].';diff --git a/supabase/schema.sql b/supabase/schema.sqlindex 38fdb6204..89588bfdd 100644--- a/supabase/schema.sql+++ b/supabase/schema.sql@@ -2932,3 +2932,28 @@ ALTER TABLE brandsCOMMENT ON COLUMN brands.is_active IS'When false, the brand is paused: the daily tracking cron and on-demand tracking skip it. Historical data stays viewable. Defaults to true.';+-- ─────────────────────────────────────────────────────────────────────────+-- migrations/00021_prompt_results_search_queries.sql+-- ─────────────────────────────────────────────────────────────────────────+-- 00021_prompt_results_search_queries.sql+-- Capture the observed query fan-out on each tracked answer.+--+-- Answer engines (Copilot, Perplexity, and — in principle — ChatGPT) run their+-- own sub-queries to build an answer. Cloro already returns these in the poll /+-- webhook response; we simply had nowhere to store them. This column holds that+-- OBSERVED fan-out (straight from the engine, not an LLM prediction) as a rich,+-- normalized array so the UI can keep the per-item engine label:+--+-- [{ "query": "best running shoes 2026", "engine": "web", "source_platform": "perplexity-web" }]+--+-- (`engine` is present only where the provider labels it — Perplexity today.)+--+-- Existing rows default to an empty array, so nothing changes for historical+-- data; new Copilot/Perplexity results populate it as they land.++ALTER TABLE prompt_results+ ADD COLUMN IF NOT EXISTS search_queries jsonb NOT NULL DEFAULT '[]'::jsonb;++COMMENT ON COLUMN prompt_results.search_queries IS+ 'Observed query fan-out from the answer engine: [{ query, engine?, source_platform }]. Copilot is the primary source; Perplexity secondary (web queries, carries engine); ChatGPT usually empty. Defaults to [].';+