Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(volumes): ask DataForSEO once per brand instead of once per prompt (#749)

Gökhan Geyik080f47f6992b
diff --git a/server/src/lib/volume-analysis.js b/server/src/lib/volume-analysis.js
index cd88e2b61..1a38a9003 100644
--- a/server/src/lib/volume-analysis.js
+++ b/server/src/lib/volume-analysis.js
@@ -3,6 +3,7 @@ import { regionToLocationCode, languageToCode } from '../lib/dataforseo-codes.js
import supabaseAdmin from '../config/supabase.js';
import { AI_VOLUME_MULTIPLIER, extractIntentKeywords } from '../lib/intent-extraction.js';
import { getSearchVolumes } from './dataforseo.js';
+import { withRetry } from './retry.js';
import logger from './logger.js';
export function mapVolumeRow(saved) {
@@ -23,20 +24,68 @@ export function mapVolumeRow(saved) {
};
}
-export async function fetchAndSaveVolumes(promptId, keywords, intent, locationCode, languageCode) {
- const volumes = await getSearchVolumes(keywords, {
- locationCode: locationCode || undefined,
- languageCode: languageCode || undefined,
- });
+/**
+ * DataForSEO bills per request, not per keyword: one request carries up to
+ * MAX_KEYWORDS_PER_REQUEST keywords, and a request for five costs exactly what
+ * a request for a thousand costs ($0.09 on the live endpoint). Asking once per
+ * prompt therefore multiplied the bill by the number of prompts — a 100-prompt
+ * brand spent $9.00 doing the work of a single $0.09 request.
+ */
+const MAX_KEYWORDS_PER_REQUEST = 1000;
+
+/**
+ * Look up every distinct keyword in as few requests as possible.
+ *
+ * A chunk is retried before it is given up on. It now carries up to two
+ * hundred prompts' worth of work, so losing one to a provider blip costs far
+ * more than it did when a failed request set back a single prompt.
+ */
+async function fetchVolumesForKeywords(keywords, { locationCode, languageCode }) {
+ const distinct = [...new Set((keywords || []).map((k) => String(k).trim()).filter(Boolean))];
+ const volumes = {};
+
+ for (let i = 0; i < distinct.length; i += MAX_KEYWORDS_PER_REQUEST) {
+ const chunk = distinct.slice(i, i + MAX_KEYWORDS_PER_REQUEST);
+ const part = await withRetry(
+ () =>
+ getSearchVolumes(chunk, {
+ locationCode: locationCode || undefined,
+ languageCode: languageCode || undefined,
+ }),
+ { attempts: 3, label: 'dataforseo search_volume' },
+ );
+ Object.assign(volumes, part);
+ }
+
+ return { volumes, requests: Math.ceil(distinct.length / MAX_KEYWORDS_PER_REQUEST) };
+}
+
+/**
+ * Reduce one prompt's keywords against a volume map that may cover the whole
+ * brand. google_volumes stays a { keyword: number } map for the UI, and
+ * competition is a volume-weighted average of the keyword indices.
+ *
+ * Lookups are case-folded because the response echoes keywords in its own
+ * normalised form; the stored key stays the returned one, so what lands in the
+ * column is unchanged from when each prompt had the response to itself.
+ */
+function summarizeVolumes(keywords, volumes) {
+ const byKeyword = new Map();
+ for (const [keyword, data] of Object.entries(volumes)) {
+ byKeyword.set(keyword.trim().toLowerCase(), [keyword, data]);
+ }
- // google_volumes stays a { keyword: number } map for the UI. Competition is
- // aggregated per prompt as a volume-weighted average of the keyword indices.
const googleVolumes = {};
let totalGoogleVolume = 0;
let competitionWeightedSum = 0;
let competitionWeight = 0;
- for (const [keyword, data] of Object.entries(volumes)) {
- googleVolumes[keyword] = data.volume;
+
+ for (const keyword of keywords || []) {
+ const hit = byKeyword.get(String(keyword).trim().toLowerCase());
+ if (!hit) continue;
+
+ const [returnedKeyword, data] = hit;
+ googleVolumes[returnedKeyword] = data.volume;
totalGoogleVolume += data.volume;
if (data.competitionIndex !== null && data.competitionIndex !== undefined) {
competitionWeightedSum += data.competitionIndex * data.volume;
@@ -55,8 +104,16 @@ export async function fetchAndSaveVolumes(promptId, keywords, intent, locationCo
? 'MEDIUM'
: 'HIGH';
- const estAiVolume = Math.round(totalGoogleVolume * AI_VOLUME_MULTIPLIER);
+ return {
+ googleVolumes,
+ totalGoogleVolume,
+ competitionIndex,
+ competition,
+ estAiVolume: Math.round(totalGoogleVolume * AI_VOLUME_MULTIPLIER),
+ };
+}
+async function saveVolumeRow(promptId, intent, keywords, summary, locationCode, languageCode) {
const { data: saved, error: dbError } = await supabaseAdmin
.from('prompt_volumes')
.upsert(
@@ -64,12 +121,12 @@ export async function fetchAndSaveVolumes(promptId, keywords, intent, locationCo
prompt_id: promptId,
intent,
keywords,
- google_volumes: googleVolumes,
- total_google_volume: totalGoogleVolume,
+ google_volumes: summary.googleVolumes,
+ total_google_volume: summary.totalGoogleVolume,
ai_volume_multiplier: AI_VOLUME_MULTIPLIER,
- est_ai_volume: estAiVolume,
- competition_index: competitionIndex,
- competition,
+ est_ai_volume: summary.estAiVolume,
+ competition_index: summary.competitionIndex,
+ competition: summary.competition,
location_code: locationCode || null,
language_code: languageCode || null,
fetched_at: new Date().toISOString(),
@@ -83,6 +140,82 @@ export async function fetchAndSaveVolumes(promptId, keywords, intent, locationCo
return saved;
}
+/**
+ * Single-prompt path (POST /api/volumes/analyze). One prompt is one request
+ * either way, so there is nothing to batch here.
+ */
+export async function fetchAndSaveVolumes(promptId, keywords, intent, locationCode, languageCode) {
+ const { volumes } = await fetchVolumesForKeywords(keywords, { locationCode, languageCode });
+ return saveVolumeRow(
+ promptId,
+ intent,
+ keywords,
+ summarizeVolumes(keywords, volumes),
+ locationCode,
+ languageCode,
+ );
+}
+
+/**
+ * Look the whole set up at once, then write each prompt's slice of the answer.
+ *
+ * A failed lookup fails every prompt it covered — that is the cost of asking
+ * once instead of N times, and why the request retries first. A failed write
+ * is still per-prompt.
+ */
+async function fetchAndStore(prepared, { locationCode, languageCode, brandId }) {
+ const results = [];
+ if (prepared.length === 0) return results;
+
+ let volumes;
+ let requests;
+ const lookupStart = Date.now();
+ try {
+ ({ volumes, requests } = await fetchVolumesForKeywords(
+ prepared.flatMap((p) => p.keywords || []),
+ { locationCode, languageCode },
+ ));
+ } catch (err) {
+ logger.error({ err, brandId, prompts: prepared.length }, 'volume lookup failed for brand');
+ return prepared.map(({ promptId }) => ({ promptId, error: err.message }));
+ }
+
+ const lookupMs = Date.now() - lookupStart;
+ const writeStart = Date.now();
+
+ for (const { promptId, intent, keywords } of prepared) {
+ try {
+ const saved = await saveVolumeRow(
+ promptId,
+ intent,
+ keywords,
+ summarizeVolumes(keywords, volumes),
+ locationCode,
+ languageCode,
+ );
+ results.push(mapVolumeRow(saved));
+ } catch (err) {
+ logger.error({ err, promptId }, 'volume save failed for prompt');
+ results.push({ promptId, error: err.message });
+ }
+ }
+
+ // Split by phase so the next run answers "how long does this take now?"
+ // without anyone having to guess which side the time went to.
+ logger.info(
+ {
+ brandId,
+ prompts: prepared.length,
+ dataforseoRequests: requests,
+ lookupMs,
+ writeMs: Date.now() - writeStart,
+ },
+ 'brand volume lookup complete',
+ );
+
+ return results;
+}
+
/**
* PostgREST caps an un-paginated select at 1000 rows, and an `.in()` filter
* travels in the query string rather than the body — a brand near the
@@ -136,6 +269,14 @@ function activePromptsQuery(setIds) {
.order('id', { ascending: true });
}
+function existingVolumesQuery(setIds) {
+ return supabaseAdmin
+ .from('prompt_volumes')
+ .select('prompt_id, intent, keywords, prompts!inner(prompt_set_id)')
+ .in('prompts.prompt_set_id', setIds)
+ .order('prompt_id', { ascending: true });
+}
+
/**
* How much of a brand's active prompt set already carries volumes. This is the
* pair the Prompts page polls while an analysis runs, so it stays two exact
@@ -197,13 +338,7 @@ export async function analyzeBrandVolumes(
const existingPromptIds = new Set();
if (!force || onlyMissing) {
- const existingRows = await fetchAllPages(() =>
- supabaseAdmin
- .from('prompt_volumes')
- .select('prompt_id, intent, keywords, prompts!inner(prompt_set_id)')
- .in('prompts.prompt_set_id', setIds)
- .order('prompt_id', { ascending: true }),
- );
+ const existingRows = await fetchAllPages(() => existingVolumesQuery(setIds));
for (const row of existingRows) {
existingPromptIds.add(row.prompt_id);
@@ -225,39 +360,74 @@ export async function analyzeBrandVolumes(
const aiModel = resolveModel();
const results = [];
+ const prepared = [];
+ const keywordStart = Date.now();
+ // Keywords first, one prompt at a time — a prompt with saved keywords costs
+ // nothing here, the rest cost one LLM call each. This is now the only
+ // per-prompt work in the run.
for (const { id: promptId, text: promptText } of promptsToAnalyze) {
try {
- let intent;
- let keywords;
-
const cached = existingMap[promptId];
if (cached) {
- intent = cached.intent;
- keywords = cached.keywords;
+ prepared.push({ promptId, intent: cached.intent, keywords: cached.keywords });
} else {
- const intentResult = await extractIntentKeywords(promptText, aiModel);
- intent = intentResult.intent;
- keywords = intentResult.keywords;
+ const { intent, keywords } = await extractIntentKeywords(promptText, aiModel);
+ prepared.push({ promptId, intent, keywords });
}
-
- const saved = await fetchAndSaveVolumes(
- promptId,
- keywords,
- intent,
- resolvedLocationCode,
- resolvedLanguageCode,
- );
-
- results.push(mapVolumeRow(saved));
} catch (err) {
- logger.error({ err, promptId }, 'volume analysis failed for prompt');
-
- results.push({
- promptId,
- error: err.message,
- });
+ logger.error({ err, promptId }, 'keyword extraction failed for prompt');
+ results.push({ promptId, error: err.message });
}
}
+
+ logger.info(
+ { brandId, prompts: prepared.length, keywordMs: Date.now() - keywordStart },
+ 'brand keyword extraction complete',
+ );
+
+ results.push(
+ ...(await fetchAndStore(prepared, {
+ locationCode: resolvedLocationCode,
+ languageCode: resolvedLanguageCode,
+ brandId,
+ })),
+ );
+
return results;
}
+
+/**
+ * Re-read Google volumes for every active prompt that already has keywords.
+ * No LLM is involved, so the whole refresh is one DataForSEO request per 1000
+ * keywords — it used to be one per prompt.
+ */
+export async function refreshBrandVolumes(brandId, { locationCode, languageCode } = {}) {
+ const { data: brand, error: brandError } = await supabaseAdmin
+ .from('brands')
+ .select('region, language')
+ .eq('id', brandId)
+ .maybeSingle();
+
+ if (brandError) throw new Error(`Failed to fetch brand: ${brandError.message}`);
+ if (!brand) throw new Error(`Brand ${brandId} not found`);
+
+ const resolvedLocationCode =
+ locationCode ?? (brand.region ? regionToLocationCode(brand.region) : undefined);
+ const resolvedLanguageCode =
+ languageCode ?? (brand.language ? languageToCode(brand.language) : undefined);
+
+ const setIds = await fetchPromptSetIds(brandId);
+ if (setIds.length === 0) return [];
+
+ const rows = await fetchAllPages(() => existingVolumesQuery(setIds));
+ const prepared = rows
+ .filter((row) => row.keywords?.length)
+ .map((row) => ({ promptId: row.prompt_id, intent: row.intent, keywords: row.keywords }));
+
+ return fetchAndStore(prepared, {
+ locationCode: resolvedLocationCode,
+ languageCode: resolvedLanguageCode,
+ brandId,
+ });
+}
diff --git a/server/src/lib/volume-analysis.test.js b/server/src/lib/volume-analysis.test.js
index b7a320cf0..eb164951d 100644
--- a/server/src/lib/volume-analysis.test.js
+++ b/server/src/lib/volume-analysis.test.js
@@ -211,16 +211,17 @@ describe('analyzeBrandVolumes', () => {
expect(extractIntentKeywords).toHaveBeenCalledTimes(3);
});
- // One prompt's upstream failure is recorded against that prompt; the rest of
- // the brand still gets analyzed.
- it('records a per-prompt failure without dropping the run', async () => {
+ // Keyword extraction is still per prompt, so its failures still are: one
+ // prompt whose LLM call fails does not take the rest of the brand with it.
+ it('records a keyword-extraction failure against that prompt alone', async () => {
const { analyzeBrandVolumes } = await load(seed({ activeCount: 3 }));
- getSearchVolumes.mockRejectedValueOnce(new Error('DataForSEO API error (429)'));
+ extractIntentKeywords.mockRejectedValueOnce(new Error('model overloaded'));
const results = await analyzeBrandVolumes('b1', { onlyMissing: true });
expect(results).toHaveLength(3);
expect(results.filter((r) => r.error)).toHaveLength(1);
+ expect(results.filter((r) => !r.error)).toHaveLength(2);
});
it('returns nothing for a brand with no prompt sets', async () => {
@@ -231,6 +232,107 @@ describe('analyzeBrandVolumes', () => {
expect(await analyzeBrandVolumes('b1', { onlyMissing: true })).toEqual([]);
});
+ /**
+ * DataForSEO bills per request and takes up to 1000 keywords in one. Asking
+ * per prompt cost $0.09 each — $9.00 for a 100-prompt brand doing work a
+ * single request covers. These pin the request count, which is the bill.
+ */
+ it('looks up a 100-prompt brand in one request, not a hundred', async () => {
+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 100 }));
+ extractIntentKeywords.mockImplementation((text) =>
+ Promise.resolve({ intent: 'informational', keywords: [`${text} a`, `${text} b`] }),
+ );
+
+ await analyzeBrandVolumes('b1', { onlyMissing: true });
+
+ expect(getSearchVolumes).toHaveBeenCalledTimes(1);
+ expect(getSearchVolumes.mock.calls[0][0]).toHaveLength(200);
+ });
+
+ it('splits into one request per 1000 keywords and no more', async () => {
+ // 600 prompts x 5 distinct keywords = 3000 keywords = 3 requests.
+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 600 }));
+ extractIntentKeywords.mockImplementation((text) =>
+ Promise.resolve({
+ intent: 'informational',
+ keywords: Array.from({ length: 5 }, (_, i) => `${text} kw${i}`),
+ }),
+ );
+
+ await analyzeBrandVolumes('b1', { onlyMissing: true });
+
+ expect(getSearchVolumes).toHaveBeenCalledTimes(3);
+ for (const [chunk] of getSearchVolumes.mock.calls) {
+ expect(chunk.length).toBeLessThanOrEqual(1000);
+ }
+ });
+
+ it('sends a keyword shared by several prompts only once', async () => {
+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 10 }));
+ extractIntentKeywords.mockResolvedValue({ intent: 'informational', keywords: ['same', 'kw'] });
+
+ await analyzeBrandVolumes('b1', { onlyMissing: true });
+
+ expect(getSearchVolumes).toHaveBeenCalledTimes(1);
+ expect(getSearchVolumes.mock.calls[0][0]).toEqual(['same', 'kw']);
+ });
+
+ // Each prompt must read only its own keywords out of a map that now holds
+ // the whole brand's, or every prompt would report the brand's total volume.
+ it('gives each prompt only its own keywords out of the shared answer', async () => {
+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 2 }));
+ extractIntentKeywords
+ .mockResolvedValueOnce({ intent: 'i', keywords: ['alpha'] })
+ .mockResolvedValueOnce({ intent: 'i', keywords: ['beta'] });
+ getSearchVolumes.mockResolvedValue({
+ alpha: { volume: 100, competitionIndex: 10, competition: 'LOW' },
+ beta: { volume: 7, competitionIndex: 90, competition: 'HIGH' },
+ });
+
+ const results = await analyzeBrandVolumes('b1', { onlyMissing: true });
+
+ expect(results.map((r) => r.totalGoogleVolume).sort((a, b) => a - b)).toEqual([7, 100]);
+ expect(results.map((r) => r.googleVolumes)).toEqual([{ alpha: 100 }, { beta: 7 }]);
+ });
+
+ // The response echoes keywords in its own normalised form, so a prompt whose
+ // keyword differs only by case must still find its volume.
+ it('matches keywords back case-insensitively', async () => {
+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 1 }));
+ extractIntentKeywords.mockResolvedValue({ intent: 'i', keywords: ['Best CRM Software'] });
+ getSearchVolumes.mockResolvedValue({
+ 'best crm software': { volume: 500, competitionIndex: 40, competition: 'MEDIUM' },
+ });
+
+ const [row] = await analyzeBrandVolumes('b1', { onlyMissing: true });
+
+ expect(row.totalGoogleVolume).toBe(500);
+ });
+
+ // One request now stands in for up to two hundred prompts, so a blip that
+ // used to cost one prompt would cost all of them. It is retried first.
+ it('retries a failed request before giving up on the batch', async () => {
+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 5 }));
+ getSearchVolumes
+ .mockRejectedValueOnce(new Error('DataForSEO API error (429)'))
+ .mockResolvedValue({ a: { volume: 10, competitionIndex: 50, competition: 'MEDIUM' } });
+
+ const results = await analyzeBrandVolumes('b1', { onlyMissing: true });
+
+ expect(getSearchVolumes).toHaveBeenCalledTimes(2);
+ expect(results.filter((r) => r.error)).toHaveLength(0);
+ });
+
+ it('fails every prompt the request covered when it cannot be retried through', async () => {
+ const { analyzeBrandVolumes } = await load(seed({ activeCount: 5 }));
+ getSearchVolumes.mockRejectedValue(new Error('DataForSEO down'));
+
+ const results = await analyzeBrandVolumes('b1', { onlyMissing: true });
+
+ expect(results).toHaveLength(5);
+ expect(results.every((r) => r.error)).toBe(true);
+ });
+
// An `.in()` filter travels in the query string, so a list of prompt ids is a
// request-line overflow waiting for a big enough brand — the failure #741 was
// opened for. Scoping through prompt_sets keeps the filter one id wide.
@@ -246,6 +348,49 @@ describe('analyzeBrandVolumes', () => {
});
});
+/**
+ * Refreshing re-reads Google volumes for prompts that already have keywords.
+ * It never calls the LLM, so after batching the whole refresh is one request
+ * per 1000 keywords — it used to be one request per prompt.
+ */
+describe('refreshBrandVolumes', () => {
+ it('refreshes a brand in one request and never calls the LLM', async () => {
+ const withVolumes = Array.from({ length: 40 }, (_, i) => `p${i}`);
+ const { refreshBrandVolumes } = await load(seed({ activeCount: 40, withVolumes }));
+
+ const results = await refreshBrandVolumes('b1');
+
+ expect(results).toHaveLength(40);
+ expect(getSearchVolumes).toHaveBeenCalledTimes(1);
+ expect(extractIntentKeywords).not.toHaveBeenCalled();
+ });
+
+ it('reuses the saved keywords rather than inventing new ones', async () => {
+ const { refreshBrandVolumes } = await load(seed({ activeCount: 2, withVolumes: ['p0', 'p1'] }));
+
+ await refreshBrandVolumes('b1');
+
+ expect(getSearchVolumes.mock.calls[0][0]).toEqual(['saved keyword']);
+ });
+
+ it('skips rows that carry no keywords to look up', async () => {
+ const db = seed({ activeCount: 2, withVolumes: ['p0', 'p1'] });
+ db.prompt_volumes[1].keywords = [];
+ const { refreshBrandVolumes } = await load(db);
+
+ expect(await refreshBrandVolumes('b1')).toHaveLength(1);
+ });
+
+ it('returns nothing for a brand with no prompt sets', async () => {
+ const db = seed({ activeCount: 2, withVolumes: ['p0'] });
+ db.prompt_sets = [];
+ const { refreshBrandVolumes } = await load(db);
+
+ expect(await refreshBrandVolumes('b1')).toEqual([]);
+ expect(getSearchVolumes).not.toHaveBeenCalled();
+ });
+});
+
/**
* The pair the Prompts page polls while a run is in flight. It has to agree
* with what analyzeBrandVolumes actually does, or the progress readout stalls
diff --git a/server/src/routes/volumes.js b/server/src/routes/volumes.js
index 1b4ed0a4e..03491b8d3 100644
--- a/server/src/routes/volumes.js
+++ b/server/src/routes/volumes.js
@@ -14,6 +14,7 @@ import {
mapVolumeRow,
fetchAndSaveVolumes,
analyzeBrandVolumes,
+ refreshBrandVolumes,
getVolumeProgress,
} from '../lib/volume-analysis.js';
import logger from '../lib/logger.js';
@@ -276,75 +277,9 @@ router.post('/refresh', requireFeature('prompt_volumes'), async (req, res) => {
await assertBrandAccess(brandId, req.user.id);
- let resolvedLocationCode = locationCode;
- let resolvedLanguageCode = languageCode;
- if (resolvedLocationCode == null || resolvedLanguageCode == null) {
- const { data: brand } = await supabaseAdmin
- .from('brands')
- .select('region, language')
- .eq('id', brandId)
- .maybeSingle();
- if (brand) {
- if (resolvedLocationCode == null) {
- resolvedLocationCode = regionToLocationCode(brand.region);
- }
- if (resolvedLanguageCode == null) {
- resolvedLanguageCode = languageToCode(brand.language);
- }
- }
- }
-
- const { data: promptSets } = await supabaseAdmin
- .from('prompt_sets')
- .select('id')
- .eq('brand_id', brandId);
-
- if (!promptSets?.length) {
- return res.json({ results: [], refreshed: 0, remaining });
- }
-
- const { data: prompts } = await supabaseAdmin
- .from('prompts')
- .select('id')
- .in(
- 'prompt_set_id',
- promptSets.map((ps) => ps.id),
- );
-
- if (!prompts?.length) {
- return res.json({ results: [], refreshed: 0, remaining });
- }
-
- const { data: existingVolumes } = await supabaseAdmin
- .from('prompt_volumes')
- .select('prompt_id, intent, keywords')
- .in(
- 'prompt_id',
- prompts.map((p) => p.id),
- );
-
- if (!existingVolumes?.length) {
- return res.json({ results: [], refreshed: 0, remaining });
- }
-
- const results = [];
-
- for (const row of existingVolumes) {
- try {
- const saved = await fetchAndSaveVolumes(
- row.prompt_id,
- row.keywords,
- row.intent,
- resolvedLocationCode,
- resolvedLanguageCode,
- );
- results.push(mapVolumeRow(saved));
- } catch (err) {
- results.push({ promptId: row.prompt_id, error: err.message });
- }
- }
-
+ const results = await refreshBrandVolumes(brandId, { locationCode, languageCode });
const refreshed = results.filter((r) => !r.error).length;
+
if (refreshed > 0 && orgId) {
await supabaseAdmin.from('volume_usage').insert({
organization_id: orgId,
@@ -366,7 +301,7 @@ router.post('/refresh', requireFeature('prompt_volumes'), async (req, res) => {
message: error.message,
});
}
- req.log.error({ err: error }, 'volume refresh error');
+ logger.error({ err: error }, 'volume refresh error');
return res.status(error.status || 500).json({
error: 'Failed to refresh volumes',
details: error.message,