Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(suggestions): actually search the web for competitor and topic ideas (#884)
Gökhan Geyikcb1420e8f544
diff --git a/server/src/lib/ai-provider.js b/server/src/lib/ai-provider.jsindex b8a2547b2..35ec30ce2 100644--- a/server/src/lib/ai-provider.js+++ b/server/src/lib/ai-provider.js@@ -112,3 +112,31 @@ export function getGoogleProvider() {}return googleProvider;}++/**+ * The provider-hosted web search tool for a "provider/model" string, as a+ * `tools` object for generateText, or null when that provider has none we can+ * use (OpenRouter does not relay them).+ *+ * The suggestion routes used to ask for search with+ * `resolveModel(model, { useSearchGrounding: true })`. That was an+ * @ai-sdk/google v1 model setting; from v2 the provider takes only a model id+ * and drops the second argument, so "Search the web" prompts were answered+ * from the model's memory alone. Search is a tool now, attached per call.+ *+ * @param {string} modelString+ * @returns {Record<string, unknown> | null}+ */+export function webSearchTools(modelString) {+ const [providerName] = (modelString || '').split('/');+ if (providerName === 'google' && googleProvider) {+ return { google_search: googleProvider.tools.googleSearch({}) };+ }+ if (providerName === 'openai' && openaiProvider) {+ return { web_search: openaiProvider.tools.webSearch({}) };+ }+ if (providerName === 'anthropic' && providers.anthropic) {+ return { web_search: providers.anthropic.tools.webSearch_20250305({ maxUses: 5 }) };+ }+ return null;+}diff --git a/server/src/lib/ai-provider.test.js b/server/src/lib/ai-provider.test.jsindex 71f36ebb7..c03713d7c 100644--- a/server/src/lib/ai-provider.test.js+++ b/server/src/lib/ai-provider.test.js@@ -72,4 +72,27 @@ describe('ai-provider registry', () => {expect(() => resolveModel('mistral/large')).toThrow(/not configured.*openrouter/s);});++ it('attaches each first-party provider its own hosted web search tool', async () => {+ const { webSearchTools } = await loadWith({+ GOOGLE_GENERATIVE_AI_API_KEY: 'g-test',+ OPENAI_API_KEY: 'sk-test',+ ANTHROPIC_API_KEY: 'sk-ant-test',+ });++ expect(webSearchTools('google/gemini-3-flash-preview').google_search.id).toBe(+ 'google.google_search',+ );+ expect(webSearchTools('openai/gpt-5-mini').web_search.id).toBe('openai.web_search');+ expect(webSearchTools('anthropic/claude-haiku-4-5').web_search.id).toBe(+ 'anthropic.web_search_20250305',+ );+ });++ it('has no search tool for openrouter or an unconfigured provider', async () => {+ const { webSearchTools } = await loadWith({ OPENROUTER_API_KEY: 'sk-or-test' });++ expect(webSearchTools('openrouter/google/gemini-3-flash')).toBeNull();+ expect(webSearchTools('google/gemini-3-flash-preview')).toBeNull();+ });});diff --git a/server/src/lib/web-research.js b/server/src/lib/web-research.jsnew file mode 100644index 000000000..765c888bf--- /dev/null+++ b/server/src/lib/web-research.js@@ -0,0 +1,25 @@+import { generateText } from 'ai';+import { resolveModel, webSearchTools } from './ai-provider.js';++/**+ * The research phase of the suggestion routes: one generateText call with the+ * provider's web search attached when it has one.+ *+ * Returns the text plus how many sources the answer drew on, so callers can+ * log it — a research step that silently stops searching (as the old+ * `useSearchGrounding` flag did) shows up as `sources: 0` instead of as+ * suggestions that are quietly worse.+ *+ * @param {string} modelString - "provider/model"+ * @param {string} prompt+ * @returns {Promise<{ text: string, searched: boolean, sources: number }>}+ */+export async function webResearch(modelString, prompt) {+ const tools = webSearchTools(modelString);+ const result = await generateText({+ model: resolveModel(modelString),+ prompt,+ ...(tools ? { tools } : {}),+ });+ return { text: result.text, searched: Boolean(tools), sources: result.sources?.length ?? 0 };+}diff --git a/server/src/routes/competitors.js b/server/src/routes/competitors.jsindex 251cf2c86..2546544e6 100644--- a/server/src/routes/competitors.js+++ b/server/src/routes/competitors.js@@ -1,9 +1,10 @@import { Router } from 'express';-import { generateText, generateObject } from 'ai';+import { generateObject } from 'ai';import { z } from 'zod';import { resolveModel } from '../lib/ai-provider.js';import { getLanguageName } from '../lib/languages.js';import { withRetry } from '../lib/retry.js';+import { webResearch } from '../lib/web-research.js';const router = Router();@@ -19,18 +20,33 @@ const competitorSchema = z.object({.max(10),});+/** English country name for a region code ("TR" → "Turkey"), or null. */+function countryName(region) {+ const code = String(region || '')+ .split('-')[0]+ .toUpperCase();+ if (!/^[A-Z]{2}$/.test(code)) return null;+ try {+ return new Intl.DisplayNames(['en'], { type: 'region' }).of(code) || null;+ } catch {+ return null;+ }+}+/*** POST /api/competitors/suggest- * Body: { brandName, industry, description? }+ * Body: { brandName, industry, description?, website?, region?, language? }* Returns: { competitors: [{ name, domain }] }*- * Uses OpenAI Responses API with web search to find real competitors,- * then structures the results with generateObject.+ * Researches the brand with the model's web search, then structures the+ * results with generateObject. The website pins down which company the name+ * means, and the region which market its competitors come from.*/router.post('/suggest', async (req, res) => {try {- const { brandName, industry, description, language } = req.body;+ const { brandName, industry, description, website, region, language } = req.body;const langName = getLanguageName(language);+ const country = countryName(region);if (!brandName) {return res.status(400).json({ error: 'brandName is required' });@@ -40,28 +56,31 @@ router.post('/suggest', async (req, res) => {process.env.COMPETITOR_SUGGESTION_MODEL || 'google/gemini-3-flash-preview';// Phase 1: web search to find real competitors- const researchPrompt = `Search the web and find 5-10 direct competitors of "${brandName}".+ const market = country ? `in ${country}` : `in the ${langName}-speaking market`;+ const researchPrompt = `Search the web and find 5-10 direct competitors of "${brandName}"${website ? ` (${website})` : ''}.Industry: ${industry || 'Not specified'}Description: ${description || 'Not specified'}+Market: ${country || 'Not specified'}, language ${langName}-Find REAL companies that compete directly with "${brandName}" — selling similar products/services to similar audiences, especially in the ${langName}-speaking market. For each competitor, provide the company name and their actual website domain.`;+${website ? `First look up ${website} to understand exactly what "${brandName}" sells and to whom — the name alone may refer to other companies.\n\n` : ''}Then find REAL companies that compete directly with it — selling similar products/services to similar audiences ${market}. Prefer companies a customer ${market} would actually compare it with over global brands outside that market. For each competitor, provide the company name and their actual website domain.`;// #379 — retry the WHOLE two-phase flow: the schema's .min(3) means a// Phase-2 under-count throws NoObjectGeneratedError, and re-running the// research gives the structuring phase fresh material to work with.const { object } = await withRetry(async () => {- const { text: research } = await generateText({- model: resolveModel(competitorModel, { useSearchGrounding: true }),- prompt: researchPrompt,- });+ const research = await webResearch(competitorModel, researchPrompt);+ req.log.info(+ { model: competitorModel, searched: research.searched, sources: research.sources },+ 'competitor research',+ );// Phase 2: structure the research into the schemareturn generateObject({model: resolveModel(competitorModel),schema: competitorSchema,- system: `Extract competitor information from the research below. Only include companies with REAL, verified domains. Return the root domain (e.g. "monday.com"), not full URLs. Do NOT include "${brandName}" itself.`,- prompt: research,+ system: `Extract competitor information from the research below. Only include companies with REAL, verified domains. Return the root domain (e.g. "monday.com"), not full URLs. Do NOT include "${brandName}"${website ? ` (${website})` : ''} itself.`,+ prompt: research.text,});},{ attempts: 3, baseDelayMs: 500, label: 'competitor-suggest' },diff --git a/server/src/routes/topics.js b/server/src/routes/topics.jsindex 091615048..2a549be20 100644--- a/server/src/routes/topics.js+++ b/server/src/routes/topics.js@@ -1,11 +1,13 @@import { Router } from 'express';-import { generateText, generateObject } from 'ai';+import { generateObject } from 'ai';import { z } from 'zod';import supabaseAdmin from '../config/supabase.js';import { assertBrandAccess } from '../lib/access.js';import { requireFeature } from '../lib/plan-guard.js';import { resolveModel } from '../lib/ai-provider.js';import { withRetry } from '../lib/retry.js';+import { webResearch } from '../lib/web-research.js';+import logger from '../lib/logger.js';import { getLanguageName } from '../lib/languages.js';const router = Router();@@ -181,16 +183,17 @@ IMPORTANT: Generate all topic names in ${langName}.`;// phase re-runs the research too, not just the last call.const { object } = await withRetry(async () => {- const { text: research } = await generateText({- model: resolveModel(topicModel, { useSearchGrounding: true }),- prompt: researchPrompt,- });+ const research = await webResearch(topicModel, researchPrompt);+ logger.info(+ { model: topicModel, searched: research.searched, sources: research.sources },+ 'topic research',+ );return generateObject({model: resolveModel(topicModel),schema: topicSchema,system: `Extract AEO tracking topics from the research below. Each topic should be concise (3-8 words) and represent an area where AI assistants might mention or discuss "${brandName}". Do NOT include the brand name "${brandName}" in any topic — keep them generic. Include a mix of: competitive comparisons, product/service features, industry trends, use cases, and problem-solving topics. Each topic MUST focus on a single concept — never combine two ideas with "and" or "&". IMPORTANT: All topic names MUST be written in ${langName}.`,- prompt: research,+ prompt: research.text,});},{ attempts: 3, baseDelayMs: 500, label: 'topic-suggest' },diff --git a/web/src/app/[locale]/(dashboard)/dashboard/brands/new/page.tsx b/web/src/app/[locale]/(dashboard)/dashboard/brands/new/page.tsxindex 94c42ad1e..43c9643b2 100644--- a/web/src/app/[locale]/(dashboard)/dashboard/brands/new/page.tsx+++ b/web/src/app/[locale]/(dashboard)/dashboard/brands/new/page.tsx@@ -589,6 +589,7 @@ export default function NewBrandPage() {description: description.trim(),website: domain,language,+ region,});setSuggestedCompetitors((prev) => mergeCompetitorSuggestions(prev, competitors));} catch (err) {diff --git a/web/src/app/[locale]/(onboarding)/dashboard/onboarding/page.tsx b/web/src/app/[locale]/(onboarding)/dashboard/onboarding/page.tsxindex 4ee9c8a4d..cbfc93432 100644--- a/web/src/app/[locale]/(onboarding)/dashboard/onboarding/page.tsx+++ b/web/src/app/[locale]/(onboarding)/dashboard/onboarding/page.tsx@@ -801,6 +801,7 @@ export default function OnboardingPage() {description: description.trim(),website: domain,language,+ region,});setSuggestedCompetitors((prev) => mergeCompetitorSuggestions(prev, competitors));} catch (err) {diff --git a/web/src/lib/ai-brand-setup.ts b/web/src/lib/ai-brand-setup.tsindex 54a424438..a3b97f177 100644--- a/web/src/lib/ai-brand-setup.ts+++ b/web/src/lib/ai-brand-setup.ts@@ -134,7 +134,7 @@ export interface SuggestCompetitorsResult {* Throws if the request fails.*/export async function suggestCompetitors(- params: BrandSetupParams,+ params: BrandSetupParams & { region?: string },): Promise<SuggestCompetitorsResult> {const authorization = await getAuthHeader();const res = await fetch(`${API_BASE}/api/competitors/suggest`, {@@ -147,6 +147,10 @@ export async function suggestCompetitors(brandName: params.brandName,industry: '',description: params.description,+ // The site tells the model which company the name means, the region+ // which market its competitors should come from.+ website: params.website,+ region: params.region,language: params.language,}),});