Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(fan-out): classify fan-out query intents in batches with the decision model (#1406)

Gökhan Geyik81f4cf7880a5
diff --git a/server/.env.example b/server/.env.example
index f8edbf98e..8a7bfef9d 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -32,6 +32,12 @@ GOOGLE_GENERATIVE_AI_API_KEY=
# cost. It falls back to openai/gpt-5-mini if the decision model fails.
# DECIDER_MODEL=perplexity/pplx-decider-v1.1-27b
#
+# Query Fan-out intents. Set FANOUT_INTENTS_DECIDER=1 (with OPENROUTER_API_KEY)
+# to classify new fan-out queries with the decision model, 128 per request,
+# instead of a chat-model call per query. Compare the two first:
+# node src/scripts/compare-fanout-intents.js
+# FANOUT_INTENTS_DECIDER=1
+#
# Setting SENTIMENT_MODEL (provider/model) picks a chat model instead. Any
# configured provider works; without OpenRouter it defaults to
# openai/gpt-5-mini — the previous hard-coded value.
diff --git a/server/src/lib/intent-extraction.js b/server/src/lib/intent-extraction.js
index 567f0ca0c..b3b57971f 100644
--- a/server/src/lib/intent-extraction.js
+++ b/server/src/lib/intent-extraction.js
@@ -1,5 +1,6 @@
import { generateObject } from 'ai';
import { z } from 'zod';
+import { logger } from './logger.js';
/**
* The share of a prompt's Google volume estimated to get an AI answer: the
@@ -86,16 +87,26 @@ export const intentOnlySchema = z.object({
intent: intentKeywordSchema.shape.intent,
});
+/**
+ * What each intent label means: the chat model's list and a decision model's
+ * criteria (#988), so the two classify by the same words.
+ */
+export const INTENT_CRITERIA = {
+ comparison: 'weighing multiple options against each other',
+ 'how-to': 'accomplishing a task or process',
+ 'what-is': 'a definition or understanding a concept',
+ 'best-top': 'finding the best / top options in a category',
+ 'vs-review': 'a head-to-head or a review of a specific thing',
+ recommendation: 'asking for a suggestion or what to pick',
+ 'problem-solving': 'fixing an issue or troubleshooting',
+ other: 'none of the above fits',
+};
+
export const INTENT_ONLY_SYSTEM_PROMPT = `You classify the primary search intent behind a short query into exactly one of these labels:
-- comparison: weighing multiple options against each other
-- how-to: accomplishing a task or process
-- what-is: a definition or understanding a concept
-- best-top: finding the best / top options in a category
-- vs-review: a head-to-head or a review of a specific thing
-- recommendation: asking for a suggestion or what to pick
-- problem-solving: fixing an issue or troubleshooting
-- other: none of the above fits
+${Object.entries(INTENT_CRITERIA)
+ .map(([label, meaning]) => `- ${label}: ${meaning}`)
+ .join('\n')}
Return only the single best-fitting label.`;
@@ -117,3 +128,90 @@ export async function classifyPromptIntent(text, aiModel) {
});
return object.intent;
}
+
+/** The most questions a decision model answers in one request. */
+export const DECIDER_BATCH_SIZE = 128;
+
+/** Chat-model calls in flight at once, as the route has always made them. */
+const CHAT_CONCURRENCY = 6;
+
+/**
+ * Classifies queries' intents with a decision model (#988): one `choice`
+ * question per query, up to DECIDER_BATCH_SIZE per request, where the chat
+ * model takes a call per query. A batch that fails, or an answer outside the
+ * labels, leaves those queries out: nothing is guessed, so nothing wrong is
+ * cached.
+ *
+ * @param {string[]} queries - normalized queries, each once
+ * @param {(params: { state: object, questions: object }) => Promise<Record<string, object>>} decideFn - decider.js's decide
+ * @returns {Promise<Record<string, string>>} the intent of each query answered
+ */
+export async function classifyIntentsByDecider(queries, decideFn) {
+ const batches = [];
+ for (let i = 0; i < queries.length; i += DECIDER_BATCH_SIZE) {
+ batches.push(queries.slice(i, i + DECIDER_BATCH_SIZE));
+ }
+ const intents = {};
+ const settled = await Promise.allSettled(
+ batches.map(async (batch) => {
+ const named = Object.fromEntries(batch.map((query, i) => [`q${i}`, query]));
+ const answers = await decideFn({
+ state: { task: 'Search intent of each query', queries: named },
+ questions: Object.fromEntries(
+ Object.entries(named).map(([name, query]) => [
+ name,
+ {
+ type: 'choice',
+ instructions: `The primary search intent behind query ${name}: "${query}".`,
+ criteria: INTENT_CRITERIA,
+ },
+ ]),
+ ),
+ });
+ for (const [name, query] of Object.entries(named)) {
+ const choice = answers?.[name]?.choice;
+ if (Object.hasOwn(INTENT_CRITERIA, choice)) intents[query] = choice;
+ }
+ }),
+ );
+ const failed = settled.filter((s) => s.status === 'rejected');
+ if (failed.length > 0) {
+ logger.warn(
+ { err: failed[0].reason, batches: failed.length },
+ '[fanout-intents] decision model batches failed; those queries stay unclassified',
+ );
+ }
+ return intents;
+}
+
+/**
+ * Classifies the queries the cache doesn't hold yet. With
+ * FANOUT_INTENTS_DECIDER set and OpenRouter configured, a decision model
+ * answers them in batches; otherwise the chat model takes each, six at a time,
+ * as before. A query that fails stays out, and is asked again next time.
+ *
+ * @param {string[]} queries - normalized queries, each once
+ * @param {object} deps
+ * @param {() => import('ai').LanguageModel} deps.chatModel - resolves the chat model, only when it's used
+ * @param {(params: object) => Promise<Record<string, object>>} deps.decideFn
+ * @param {() => boolean} deps.deciderConfigured
+ * @returns {Promise<Record<string, string>>} the intent of each query classified
+ */
+export async function classifyQueryIntents(queries, { chatModel, decideFn, deciderConfigured }) {
+ if (queries.length === 0) return {};
+ if (process.env.FANOUT_INTENTS_DECIDER && deciderConfigured()) {
+ return classifyIntentsByDecider(queries, decideFn);
+ }
+ const aiModel = chatModel();
+ const intents = {};
+ for (let i = 0; i < queries.length; i += CHAT_CONCURRENCY) {
+ const chunk = queries.slice(i, i + CHAT_CONCURRENCY);
+ const settled = await Promise.allSettled(
+ chunk.map((query) => classifyPromptIntent(query, aiModel)),
+ );
+ settled.forEach((s, j) => {
+ if (s.status === 'fulfilled') intents[chunk[j]] = s.value;
+ });
+ }
+ return intents;
+}
diff --git a/server/src/lib/intent-extraction.test.js b/server/src/lib/intent-extraction.test.js
index c37be7f4d..fd620c1fc 100644
--- a/server/src/lib/intent-extraction.test.js
+++ b/server/src/lib/intent-extraction.test.js
@@ -1,4 +1,8 @@
-import { afterEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const generateObject = vi.fn();
+vi.mock('ai', () => ({ generateObject: (...args) => generateObject(...args) }));
+vi.mock('./logger.js', () => ({ logger: { warn: vi.fn(), error: vi.fn(), info: vi.fn() } }));
/**
* The AI adoption rate behind every est_ai_volume: 45% of a prompt's Google
@@ -25,3 +29,122 @@ describe('AI_VOLUME_MULTIPLIER', () => {
expect(await multiplier()).toBe(0.2);
});
});
+
+const {
+ DECIDER_BATCH_SIZE,
+ INTENT_CRITERIA,
+ INTENT_ONLY_SYSTEM_PROMPT,
+ classifyIntentsByDecider,
+ classifyQueryIntents,
+ intentOnlySchema,
+} = await import('./intent-extraction.js');
+
+/** A decision model that answers every question with `pick(query)`. */
+const answering = (pick) =>
+ vi.fn(async ({ state, questions }) =>
+ Object.fromEntries(
+ Object.keys(questions).map((name) => [name, { choice: pick(state.queries[name]) }]),
+ ),
+ );
+
+const queriesOf = (n) => Array.from({ length: n }, (_, i) => `query ${i}`);
+
+describe('intent labels', () => {
+ it('gives the chat model and the decision model the same labels', () => {
+ expect(Object.keys(INTENT_CRITERIA)).toEqual(intentOnlySchema.shape.intent.options);
+ for (const [label, meaning] of Object.entries(INTENT_CRITERIA)) {
+ expect(INTENT_ONLY_SYSTEM_PROMPT).toContain(`- ${label}: ${meaning}\n`.trimEnd());
+ }
+ });
+});
+
+describe('classifyIntentsByDecider', () => {
+ it('asks 200 queries in two requests, one choice question per query', async () => {
+ const decide = answering((query) => (query.endsWith('7') ? 'how-to' : 'what-is'));
+ const intents = await classifyIntentsByDecider(queriesOf(200), decide);
+
+ expect(decide).toHaveBeenCalledTimes(2);
+ const sizes = decide.mock.calls.map(([{ questions }]) => Object.keys(questions).length);
+ expect(sizes).toEqual([DECIDER_BATCH_SIZE, 200 - DECIDER_BATCH_SIZE]);
+ const [{ questions }] = decide.mock.calls[0];
+ expect(questions.q0).toMatchObject({ type: 'choice', criteria: INTENT_CRITERIA });
+ expect(questions.q0.instructions).toContain('"query 0"');
+
+ expect(Object.keys(intents)).toHaveLength(200);
+ expect(intents['query 17']).toBe('how-to');
+ expect(intents['query 150']).toBe('what-is');
+ });
+
+ it('leaves out an answer outside the labels, and a batch that fails', async () => {
+ let call = 0;
+ const decide = vi.fn(async ({ questions }) => {
+ call += 1;
+ if (call === 2) throw new Error('Decisions API 503');
+ return {
+ ...Object.fromEntries(Object.keys(questions).map((name) => [name, { choice: 'other' }])),
+ q1: { choice: 'shopping' },
+ q2: {},
+ };
+ });
+ const intents = await classifyIntentsByDecider(queriesOf(130), decide);
+
+ expect(decide).toHaveBeenCalledTimes(2);
+ expect(Object.keys(intents)).toHaveLength(DECIDER_BATCH_SIZE - 2);
+ expect(intents['query 0']).toBe('other');
+ expect(intents).not.toHaveProperty('query 1');
+ expect(intents).not.toHaveProperty('query 2');
+ expect(intents).not.toHaveProperty('query 128');
+ });
+});
+
+describe('classifyQueryIntents', () => {
+ const chatModel = vi.fn(() => ({ id: 'chat' }));
+
+ beforeEach(() => {
+ generateObject.mockReset();
+ generateObject.mockImplementation(async () => ({ object: { intent: 'comparison' } }));
+ chatModel.mockClear();
+ });
+
+ it('uses the decision model when FANOUT_INTENTS_DECIDER is set and OpenRouter is configured', async () => {
+ vi.stubEnv('FANOUT_INTENTS_DECIDER', '1');
+ const decide = answering(() => 'best-top');
+ const intents = await classifyQueryIntents(['best crm'], {
+ chatModel,
+ decideFn: decide,
+ deciderConfigured: () => true,
+ });
+ expect(intents).toEqual({ 'best crm': 'best-top' });
+ expect(generateObject).not.toHaveBeenCalled();
+ expect(chatModel).not.toHaveBeenCalled();
+ });
+
+ it('keeps the chat model, a call per query, when the switch is off or OpenRouter is not configured', async () => {
+ const decide = answering(() => 'best-top');
+ for (const [flag, configured] of [
+ ['', true],
+ ['1', false],
+ ]) {
+ vi.stubEnv('FANOUT_INTENTS_DECIDER', flag);
+ generateObject.mockClear();
+ const intents = await classifyQueryIntents(['crm vs erp', 'what is a crm'], {
+ chatModel,
+ decideFn: decide,
+ deciderConfigured: () => configured,
+ });
+ expect(intents).toEqual({ 'crm vs erp': 'comparison', 'what is a crm': 'comparison' });
+ expect(generateObject).toHaveBeenCalledTimes(2);
+ }
+ expect(decide).not.toHaveBeenCalled();
+ });
+
+ it('leaves out a query whose chat call fails, and makes no call for no queries', async () => {
+ generateObject
+ .mockRejectedValueOnce(new Error('timeout'))
+ .mockResolvedValueOnce({ object: { intent: 'how-to' } });
+ const deps = { chatModel, decideFn: vi.fn(), deciderConfigured: () => false };
+ expect(await classifyQueryIntents(['a', 'b'], deps)).toEqual({ b: 'how-to' });
+ expect(await classifyQueryIntents([], deps)).toEqual({});
+ expect(chatModel).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/server/src/routes/prompts.js b/server/src/routes/prompts.js
index c739116be..5a9db0488 100644
--- a/server/src/routes/prompts.js
+++ b/server/src/routes/prompts.js
@@ -3,7 +3,8 @@ import { NoObjectGeneratedError, generateObject } from 'ai';
import { z } from 'zod';
import supabaseAdmin from '../config/supabase.js';
import { resolveModel } from '../lib/ai-provider.js';
-import { classifyPromptIntent } from '../lib/intent-extraction.js';
+import { decide, isDeciderConfigured } from '../lib/decider.js';
+import { classifyQueryIntents } from '../lib/intent-extraction.js';
import { getLanguageName } from '../lib/languages.js';
import {
PlanLimitError,
@@ -857,7 +858,6 @@ Return the exact topic names as provided above.`,
// globally per normalized query string in `fanout_query_intents`; only
// cache-misses hit the LLM. Body: { queries: string[] }.
const FANOUT_INTENT_MAX_QUERIES = 200;
-const FANOUT_INTENT_CONCURRENCY = 6;
// Cache-lookup batch size — keeps each `.in('query', ...)` URL well under
// PostgREST / proxy length limits even with long sub-queries.
const FANOUT_INTENT_LOOKUP_CHUNK = 40;
@@ -892,28 +892,20 @@ router.post('/fanout-intents', async (req, res) => {
for (const row of cached ?? []) intents[row.query] = row.intent;
}
- // 2. Classify the misses (bounded concurrency), then persist them.
+ // 2. Classify the misses (a decision model in batches, or the chat model
+ // a query at a time; #988), then persist them.
const misses = queries.filter((q) => !(q in intents));
- if (misses.length > 0) {
- const aiModel = resolveModel();
- const fresh = [];
- for (let i = 0; i < misses.length; i += FANOUT_INTENT_CONCURRENCY) {
- const chunk = misses.slice(i, i + FANOUT_INTENT_CONCURRENCY);
- const settled = await Promise.allSettled(
- chunk.map((q) => classifyPromptIntent(q, aiModel).then((intent) => ({ q, intent }))),
- );
- for (const s of settled) {
- if (s.status === 'fulfilled') {
- intents[s.value.q] = s.value.intent;
- fresh.push({ query: s.value.q, intent: s.value.intent });
- }
- }
- }
- if (fresh.length > 0) {
- await supabaseAdmin
- .from('fanout_query_intents')
- .upsert(fresh, { onConflict: 'query', ignoreDuplicates: true });
- }
+ const classified = await classifyQueryIntents(misses, {
+ chatModel: () => resolveModel(),
+ decideFn: decide,
+ deciderConfigured: isDeciderConfigured,
+ });
+ const fresh = Object.entries(classified).map(([query, intent]) => ({ query, intent }));
+ if (fresh.length > 0) {
+ Object.assign(intents, classified);
+ await supabaseAdmin
+ .from('fanout_query_intents')
+ .upsert(fresh, { onConflict: 'query', ignoreDuplicates: true });
}
return res.json({ intents });
diff --git a/server/src/scripts/compare-fanout-intents.js b/server/src/scripts/compare-fanout-intents.js
new file mode 100644
index 000000000..a500cb244
--- /dev/null
+++ b/server/src/scripts/compare-fanout-intents.js
@@ -0,0 +1,68 @@
+/**
+ * Compare the decision model's fan-out intents with the cached ones (#988),
+ * before FANOUT_INTENTS_DECIDER switches the Query Fan-out tab over to it.
+ *
+ * Run from server/, with OPENROUTER_API_KEY and the Supabase keys in .env:
+ * node src/scripts/compare-fanout-intents.js [count]
+ * count defaults to 256: the newest cached queries, two decision requests
+ *
+ * Reads `fanout_query_intents` (labelled by the chat model) and asks the
+ * decision model the same queries. Prints how often the two agree, overall
+ * and per label, and the most common disagreements. Writes nothing, and never
+ * prints a key.
+ */
+import 'dotenv/config';
+import supabaseAdmin from '../config/supabase.js';
+import { decide, isDeciderConfigured } from '../lib/decider.js';
+import { classifyIntentsByDecider } from '../lib/intent-extraction.js';
+
+const count = Math.max(1, Math.min(1024, Number.parseInt(process.argv[2] ?? '256', 10) || 256));
+
+if (!isDeciderConfigured()) {
+ console.error('OPENROUTER_API_KEY is not set (server/.env).');
+ process.exit(1);
+}
+
+const { data: rows, error } = await supabaseAdmin
+ .from('fanout_query_intents')
+ .select('query, intent')
+ .order('created_at', { ascending: false })
+ .limit(count);
+if (error) {
+ console.error('Reading fanout_query_intents failed:', error.message);
+ process.exit(1);
+}
+
+const cached = Object.fromEntries(rows.map((row) => [row.query, row.intent]));
+const started = Date.now();
+const decided = await classifyIntentsByDecider(Object.keys(cached), decide);
+const seconds = ((Date.now() - started) / 1000).toFixed(1);
+
+const answered = Object.keys(decided);
+const agree = answered.filter((query) => decided[query] === cached[query]);
+const pct = (part, whole) => (whole ? `${((part / whole) * 100).toFixed(1)}%` : '—');
+
+console.log(`Queries compared: ${rows.length}, answered by the decision model: ${answered.length}`);
+console.log(
+ `Agreement: ${agree.length} of ${answered.length} (${pct(agree.length, answered.length)})`,
+);
+console.log(`Decision requests: ${Math.ceil(rows.length / 128)}, ${seconds} s`);
+
+console.log('\nPer cached label (agree / total):');
+const labels = [...new Set(answered.map((query) => cached[query]))].sort();
+for (const label of labels) {
+ const ofLabel = answered.filter((query) => cached[query] === label);
+ const same = ofLabel.filter((query) => decided[query] === label).length;
+ console.log(` ${label.padEnd(16)} ${same} / ${ofLabel.length} (${pct(same, ofLabel.length)})`);
+}
+
+const pairs = new Map();
+for (const query of answered) {
+ if (decided[query] === cached[query]) continue;
+ const key = `${cached[query]} → ${decided[query]}`;
+ pairs.set(key, [...(pairs.get(key) ?? []), query]);
+}
+console.log('\nMost common disagreements (cached → decision model):');
+for (const [pair, queries] of [...pairs].sort((a, b) => b[1].length - a[1].length).slice(0, 8)) {
+ console.log(` ${pair}: ${queries.length}, e.g. "${queries[0]}"`);
+}