Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(content): skip opportunities whose prompt and fan-out baskets duplicate an existing one (#1022)
Cihan63bae9f77409
diff --git a/server/.env.example b/server/.env.exampleindex b4ce348ee..6eaa4d18b 100644--- a/server/.env.example+++ b/server/.env.example@@ -49,6 +49,10 @@ COMPETITOR_SUGGESTION_MODEL=google/gemini-3-flash-preview# (improveMonthlyBudgetUsd). Falls back to# DEFAULT_SUGGESTION_MODEL, then google/gemini-3-flash-preview.# IMPROVE_SUGGESTION_MODEL=google/gemini-3-flash-preview+# How alike two content opportunities' prompt and fan-out baskets must be to+# count as one (0-1, default 0.6). A new suggestion above it is skipped and+# uses no quota (CONTENT-004). Raise it if distinct work is being dropped.+# OPPORTUNITY_DUPLICATE_THRESHOLD=0.6# Model used for the Site Audit LLM-judged signals (provider/model). Cheap by# default; point it at anthropic/claude-... or openai/... if you prefer.AUDIT_LLM_MODEL=google/gemini-3-flash-previewdiff --git a/server/src/lib/open-opportunities.js b/server/src/lib/open-opportunities.jsindex 130a61342..b68611a87 100644--- a/server/src/lib/open-opportunities.js+++ b/server/src/lib/open-opportunities.js@@ -25,3 +25,25 @@ export async function loadClusterOpportunities(brandId) {if (!data || data.length < PAGE) return rows;}}++/**+ * A brand's opportunities that can still block a duplicate (CONTENT-004):+ * every status but archived, with the source data that holds their prompt+ * and fan-out baskets. Older per-prompt rows (no cluster) are included, since+ * a new cluster can repeat them too.+ */+export async function loadOpportunityBaskets(brandId) {+ const rows = [];+ for (let from = 0; ; from += PAGE) {+ const { data, error } = await supabaseAdmin+ .from('content_opportunities')+ .select('id, status, updated_at, cluster_id, related_cluster_ids, source_data')+ .eq('brand_id', brandId)+ .neq('status', 'archived')+ .order('id')+ .range(from, from + PAGE - 1);+ if (error) throw new Error(error.message);+ rows.push(...(data || []));+ if (!data || data.length < PAGE) return rows;+ }+}diff --git a/server/src/lib/opportunity-dedup.js b/server/src/lib/opportunity-dedup.jsnew file mode 100644index 000000000..8b5962ed8--- /dev/null+++ b/server/src/lib/opportunity-dedup.js@@ -0,0 +1,113 @@+/**+ * Duplicate protection for content opportunities (CONTENT-004).+ *+ * Two opportunities are the same work when they rest on the same baskets:+ * the prompts they answer and the searches AI engines ran for those prompts+ * (query fan-out). Titles vary from run to run ("Comprehensive Guide to the+ * Best AI SEO Tools", "Definitive Listicle of…") while the baskets behind+ * them stay put, so the baskets decide, not the wording. The check runs+ * before the model is called, so a duplicate costs neither a model call nor+ * a slot of the monthly quota.+ *+ * Pure on purpose: the generator loads the baskets, this compares them, and+ * the tests pin known duplicate and distinct pairs.+ */++/** Weight of the prompt basket in the score; the fan-out basket gets the rest. */+export const PROMPT_WEIGHT = 0.6;++/**+ * Score from which two baskets count as the same work. Configurable on the+ * server (OPPORTUNITY_DUPLICATE_THRESHOLD) so it can be tuned without a+ * release; 0.6 keeps the known duplicate pairs in the tests and lets the+ * distinct ones through.+ */+export const DEFAULT_DUPLICATE_THRESHOLD = 0.6;++/** A dismissed opportunity blocks its basket for this long, then may return. */+export const DISMISSED_BLOCK_DAYS = 90;++export function duplicateThreshold(env = process.env) {+ const value = Number(env.OPPORTUNITY_DUPLICATE_THRESHOLD);+ return Number.isFinite(value) && value > 0 && value <= 1 ? value : DEFAULT_DUPLICATE_THRESHOLD;+}++/** Lowercase, no punctuation or extra spaces: "Best AI-SEO tools?" → "best ai seo tools". */+export function normalize(text) {+ return String(text ?? '')+ .toLowerCase()+ .normalize('NFKD')+ .replace(/[̀-ͯ]/g, '')+ .replace(/[^\p{L}\p{N}]+/gu, ' ')+ .trim();+}++function toSet(texts) {+ return new Set((texts || []).map(normalize).filter(Boolean));+}++/**+ * Dice overlap, 0–1: twice the shared items over both sizes. A basket that+ * grew by a merge still matches its original (3 of 3 inside 5 → 0.75), while+ * one shared prompt doesn't make a ten-prompt cluster a copy of an old+ * single-prompt opportunity (1 of 10 → 0.18).+ */+export function overlap(a, b) {+ if (a.size === 0 || b.size === 0) return 0;+ let shared = 0;+ for (const x of a) if (b.has(x)) shared += 1;+ return (2 * shared) / (a.size + b.size);+}++/** { prompts, queries } as normalized sets. */+export function basket({ prompts, queries }) {+ return { prompts: toSet(prompts), queries: toSet(queries) };+}++/**+ * An existing opportunity's baskets from its stored source_data. Rows from+ * the per-prompt generator have no prompt list, only the prompt they were+ * made for, which then is their prompt basket.+ */+export function storedBasket(sourceData) {+ const sd = sourceData || {};+ const prompts = Array.isArray(sd.prompts) && sd.prompts.length ? sd.prompts : [sd.promptText];+ const queries = Array.isArray(sd.queries?.top) ? sd.queries.top.map((q) => q.query) : [];+ return basket({ prompts, queries });+}++/**+ * 0–1: how much two baskets are the same work. The prompt basket leads; the+ * fan-out basket counts only when both sides recorded one, so a missing+ * basket neither helps nor hurts.+ */+export function basketSimilarity(a, b) {+ const prompts = overlap(a.prompts, b.prompts);+ if (a.queries.size === 0 || b.queries.size === 0) return prompts;+ return PROMPT_WEIGHT * prompts + (1 - PROMPT_WEIGHT) * overlap(a.queries, b.queries);+}++const DAY_MS = 86_400_000;++/** Whether an existing opportunity still blocks a new one with its basket. */+export function blocks(row, now = Date.now()) {+ if (row.status === 'archived') return false;+ if (row.status !== 'dismissed') return true;+ const at = Date.parse(row.updated_at);+ return Number.isFinite(at) && now - at < DISMISSED_BLOCK_DAYS * DAY_MS;+}++/**+ * The existing opportunity a candidate basket duplicates, as { row, score },+ * or null. `existing` rows carry { status, updated_at, basket }.+ */+export function findDuplicate(candidate, existing, { threshold, now = Date.now() } = {}) {+ const min = threshold ?? DEFAULT_DUPLICATE_THRESHOLD;+ let best = null;+ for (const row of existing) {+ if (!blocks(row, now)) continue;+ const score = basketSimilarity(candidate, row.basket);+ if (score >= min && (!best || score > best.score)) best = { row, score };+ }+ return best;+}diff --git a/server/src/lib/opportunity-dedup.test.js b/server/src/lib/opportunity-dedup.test.jsnew file mode 100644index 000000000..6bd6d5a66--- /dev/null+++ b/server/src/lib/opportunity-dedup.test.js@@ -0,0 +1,170 @@+import { describe, expect, it } from 'vitest';+import {+ DEFAULT_DUPLICATE_THRESHOLD,+ DISMISSED_BLOCK_DAYS,+ basket,+ basketSimilarity,+ duplicateThreshold,+ findDuplicate,+ normalize,+ overlap,+ storedBasket,+} from './opportunity-dedup.js';++// CONTENT-004: duplicates are decided by the prompt and fan-out baskets+// behind an opportunity, not by its title. The pairs below are the offline+// test set the threshold is tuned against.++const AI_SEO_PROMPTS = [+ 'What are the best AI SEO tools in 2026?',+ 'Which AI SEO tool should I use for content?',+ 'Best AI SEO software for small teams',+];+const AI_SEO_QUERIES = [+ 'best ai seo tools 2026',+ 'ai seo tools comparison',+ 'ai seo software pricing',+];++const stored = (sourceData, over = {}) => ({+ id: 'o1',+ status: 'new',+ updated_at: '2026-10-01T00:00:00.000Z',+ basket: storedBasket(sourceData),+ ...over,+});++describe('normalize and overlap', () => {+ it('ignores case, punctuation and accents', () => {+ expect(normalize(' Best AI-SEO tools?! ')).toBe('best ai seo tools');+ expect(normalize('Café crème')).toBe('cafe creme');+ });++ it('is the Dice overlap of two sets', () => {+ expect(overlap(new Set(['a', 'b', 'c']), new Set(['a', 'b', 'c', 'd', 'e']))).toBe(0.75);+ expect(overlap(new Set(['a']), new Set('abcdefghij'))).toBeCloseTo(0.18, 2);+ expect(overlap(new Set(), new Set(['a']))).toBe(0);+ });+});++describe('known duplicate pairs', () => {+ it('the same prompts and searches, worded differently, are one opportunity', () => {+ // "Comprehensive Guide to the Best AI SEO Tools" vs "Definitive Listicle…":+ // the titles differ, the baskets don't.+ const candidate = basket({ prompts: AI_SEO_PROMPTS, queries: AI_SEO_QUERIES });+ const existing = storedBasket({+ prompts: AI_SEO_PROMPTS.map((p) => p.toUpperCase()),+ queries: { top: AI_SEO_QUERIES.map((query) => ({ query, timesSearched: 3 })) },+ });+ expect(basketSimilarity(candidate, existing)).toBe(1);+ });++ it('a re-clustered basket that kept most of its prompts is still the same work', () => {+ const candidate = basket({+ prompts: [...AI_SEO_PROMPTS, 'Top AI tools for SEO audits'],+ queries: AI_SEO_QUERIES,+ });+ const existing = storedBasket({+ prompts: AI_SEO_PROMPTS,+ queries: { top: AI_SEO_QUERIES.slice(0, 2).map((query) => ({ query })) },+ });+ expect(basketSimilarity(candidate, existing)).toBeGreaterThanOrEqual(+ DEFAULT_DUPLICATE_THRESHOLD,+ );+ });++ it('half the prompts plus the same searches cross the line', () => {+ const candidate = basket({+ prompts: ['AEO vs SEO', 'Is AEO replacing SEO?'],+ queries: ['aeo vs seo', 'answer engine optimization vs seo', 'aeo seo difference'],+ });+ const existing = storedBasket({+ prompts: ['AEO vs SEO', 'How is AEO different from search marketing?'],+ queries: {+ top: [+ { query: 'aeo vs seo' },+ { query: 'answer engine optimization vs seo' },+ { query: 'aeo seo difference' },+ ],+ },+ });+ expect(basketSimilarity(candidate, existing)).toBeGreaterThanOrEqual(+ DEFAULT_DUPLICATE_THRESHOLD,+ );+ });++ it('an old single-prompt opportunity matches a small cluster built around its prompt', () => {+ const candidate = basket({ prompts: ['GEO best practices', 'GEO checklist'], queries: [] });+ const existing = storedBasket({ promptText: 'GEO best practices' });+ expect(basketSimilarity(candidate, existing)).toBeGreaterThanOrEqual(+ DEFAULT_DUPLICATE_THRESHOLD,+ );+ });+});++describe('known distinct pairs', () => {+ it('similar wording over different prompts is different work', () => {+ const candidate = basket({+ prompts: ['Best AI SEO tools for agencies', 'AI SEO reporting for clients'],+ queries: ['ai seo agency tools', 'ai seo client reporting'],+ });+ const existing = storedBasket({+ prompts: AI_SEO_PROMPTS,+ queries: { top: AI_SEO_QUERIES.map((query) => ({ query })) },+ });+ expect(basketSimilarity(candidate, existing)).toBeLessThan(DEFAULT_DUPLICATE_THRESHOLD);+ });++ it('shared searches alone, over different prompts, are not enough', () => {+ const candidate = basket({ prompts: ['How do I rank in ChatGPT?'], queries: AI_SEO_QUERIES });+ const existing = storedBasket({+ prompts: AI_SEO_PROMPTS,+ queries: { top: AI_SEO_QUERIES.map((query) => ({ query })) },+ });+ expect(basketSimilarity(candidate, existing)).toBeLessThan(DEFAULT_DUPLICATE_THRESHOLD);+ });++ it('one shared prompt in a large cluster is not a copy of an old single-prompt row', () => {+ const prompts = Array.from({ length: 10 }, (_, i) => `coffee prompt ${i}`);+ const candidate = basket({ prompts, queries: [] });+ const existing = storedBasket({ promptText: 'coffee prompt 0' });+ expect(basketSimilarity(candidate, existing)).toBeLessThan(DEFAULT_DUPLICATE_THRESHOLD);+ });+});++describe('findDuplicate', () => {+ const NOW = Date.parse('2026-10-13T00:00:00.000Z');+ const candidate = basket({ prompts: AI_SEO_PROMPTS, queries: AI_SEO_QUERIES });+ const same = { prompts: AI_SEO_PROMPTS };++ it('finds an open or sent opportunity on the same baskets', () => {+ expect(findDuplicate(candidate, [stored(same)], { now: NOW })?.row.id).toBe('o1');+ expect(findDuplicate(candidate, [stored(same, { status: 'sent' })], { now: NOW })?.row.id).toBe(+ 'o1',+ );+ });++ it(`keeps a dismissed basket out for ${DISMISSED_BLOCK_DAYS} days, then lets it back`, () => {+ const recent = stored(same, { status: 'dismissed', updated_at: '2026-09-01T00:00:00.000Z' });+ const old = stored(same, { status: 'dismissed', updated_at: '2026-06-01T00:00:00.000Z' });+ expect(findDuplicate(candidate, [recent], { now: NOW })).not.toBeNull();+ expect(findDuplicate(candidate, [old], { now: NOW })).toBeNull();+ });++ it('ignores archived rows and returns the closest match', () => {+ const archived = stored(same, { id: 'a', status: 'archived' });+ const partial = stored({ prompts: AI_SEO_PROMPTS.slice(0, 2) }, { id: 'p' });+ const exact = stored(same, { id: 'e' });+ expect(findDuplicate(candidate, [archived], { now: NOW })).toBeNull();+ expect(findDuplicate(candidate, [partial, exact], { now: NOW })?.row.id).toBe('e');+ });+});++describe('duplicateThreshold', () => {+ it('reads OPPORTUNITY_DUPLICATE_THRESHOLD, falling back on bad values', () => {+ expect(duplicateThreshold({ OPPORTUNITY_DUPLICATE_THRESHOLD: '0.75' })).toBe(0.75);+ expect(duplicateThreshold({ OPPORTUNITY_DUPLICATE_THRESHOLD: 'abc' })).toBe(0.6);+ expect(duplicateThreshold({ OPPORTUNITY_DUPLICATE_THRESHOLD: '2' })).toBe(0.6);+ expect(duplicateThreshold({})).toBe(0.6);+ });+});diff --git a/server/src/lib/opportunity-generator.js b/server/src/lib/opportunity-generator.jsindex 43e327865..a20eba204 100644--- a/server/src/lib/opportunity-generator.js+++ b/server/src/lib/opportunity-generator.js@@ -23,7 +23,8 @@ import {pickClusters,scoreComponents,} from './opportunity-limits.js';-import { loadClusterOpportunities } from './open-opportunities.js';+import { loadClusterOpportunities, loadOpportunityBaskets } from './open-opportunities.js';+import { basket, duplicateThreshold, findDuplicate, storedBasket } from './opportunity-dedup.js';import { loadClusters } from './prompt-clusters.js';import { reopenStrengthened } from './opportunity-reopen.js';import {@@ -148,6 +149,67 @@ function basketSummary(clusterIds, basketRows) {};}+/** Clusters per basket request; a cluster can hold dozens of fan-out queries. */+const BASKET_CHUNK = 10;++/**+ * Drops the open clusters that duplicate an existing opportunity (CONTENT-004)+ * by adding them to `covered`, and folds each into its match when the match+ * is still open and has room, counting it on the match as `duplicatesMerged`+ * for the detail panel. A dismissed match only keeps the cluster out; after+ * DISMISSED_BLOCK_DAYS it may be suggested again. Returns how many were+ * skipped.+ */+async function skipDuplicates(brandId, open, { clusterById, basketRows, covered }) {+ if (!open.length) return 0;+ const existing = (await loadOpportunityBaskets(brandId)).map((row) => ({+ ...row,+ basket: storedBasket(row.source_data),+ }));+ if (!existing.length) return 0;++ const threshold = duplicateThreshold();+ let skipped = 0;+ for (const cand of open) {+ const c = clusterById.get(cand.id);+ const candidate = basket({+ prompts: c.promptTexts,+ queries: basketSummary([c.id], basketRows).top.map((q) => q.query),+ });+ const match = findDuplicate(candidate, existing, { threshold });+ if (!match) continue;+ covered.add(cand.id);+ skipped += 1;++ const row = match.row;+ const related = row.related_cluster_ids || [];+ if (+ row.status === 'dismissed' ||+ !row.cluster_id ||+ related.length >= MAX_MERGED_CLUSTERS ||+ related.includes(cand.id)+ ) {+ continue;+ }+ row.related_cluster_ids = [...related, cand.id];+ row.source_data = {+ ...(row.source_data || {}),+ duplicatesMerged: (row.source_data?.duplicatesMerged || 0) + 1,+ };+ const { error } = await supabaseAdmin+ .from('content_opportunities')+ .update({+ related_cluster_ids: row.related_cluster_ids,+ source_data: row.source_data,+ updated_at: new Date().toISOString(),+ })+ .eq('id', row.id);+ if (error) throw new Error(error.message);+ }+ if (skipped) logger.info({ brandId, skipped }, '[opportunities] duplicates skipped');+ return skipped;+}+async function loadPromptTexts(promptIds) {const { data, error } = await selectInChunks(promptIds, (chunk) =>supabaseAdmin.from('prompts').select('id, text').in('id', chunk),@@ -244,13 +306,39 @@ export async function generateContentOpportunities(brandId, { model, onProgress}const candidates = clusters.map((c) => ({ id: c.id, topicId: c.topic_id, ...measure([c.id]) }));- const wanted = pickClusters(candidates, coveredClusterIds(existing));+ const covered = coveredClusterIds(existing);++ // The fan-out baskets of every cluster still open for an opportunity: the+ // duplicate check compares them, and the prompt shows the picked ones'.+ const open = candidates.filter((c) => c.metrics.tested > 0 && !covered.has(c.id));+ const { data: basketRows, error: basketErr } = await selectInChunks(+ open.map((c) => c.id),+ (chunk) =>+ supabaseAdmin+ .from('prompt_cluster_queries')+ .select('cluster_id, query, times_searched, included')+ .in('cluster_id', chunk),+ BASKET_CHUNK,+ );+ if (basketErr) throw new Error(basketErr.message);++ // CONTENT-004: a cluster whose baskets match an existing opportunity is+ // that opportunity again. It is never written, so it costs neither a model+ // call nor a slot of the quota; an open match takes it in as a related+ // cluster, so it is covered from now on.+ const duplicates = await skipDuplicates(brandId, open, {+ clusterById,+ basketRows: basketRows || [],+ covered,+ });++ const wanted = pickClusters(candidates, covered);if (!wanted.length) {logger.info(- { brandId, clusters: clusters.length, opportunities: existing.length, reopened },+ { brandId, clusters: clusters.length, opportunities: existing.length, reopened, duplicates },'[opportunities] every cluster with answers already has an opportunity',);- return { generated: 0, merged: 0, reopened };+ return { generated: 0, merged: 0, reopened, duplicates };}// Each new opportunity takes one of the organization's monthly slots@@ -267,18 +355,6 @@ export async function generateContentOpportunities(brandId, { model, onProgresslogger.error({ err, brandId }, '[opportunities] handing back slots failed'),);- const { data: basketRows, error: basketErr } = await supabaseAdmin- .from('prompt_cluster_queries')- .select('cluster_id, query, times_searched, included')- .in(- 'cluster_id',- picked.map((c) => c.id),- );- if (basketErr) {- await releaseAll();- throw new Error(basketErr.message);- }-const shown = existing.filter((o) =>@@ -445,8 +521,8 @@ Write every title and description in ${getLanguageName(brand.language)}.`;await releaseOpportunitySlots(slots.ids.slice(created.length));}logger.info(- { brandId, generated: inserts.length, merged, reopened, clusters: clusters.length },+ { brandId, generated: inserts.length, merged, reopened, duplicates, clusters: clusters.length },'[opportunities] generated',);- return { generated: inserts.length, merged, reopened };+ return { generated: inserts.length, merged, reopened, duplicates };}diff --git a/web/messages/en.json b/web/messages/en.jsonindex ca2cb6c86..9c4196925 100644--- a/web/messages/en.json+++ b/web/messages/en.json@@ -1926,6 +1926,8 @@"sendToActionCenterError": "Failed to send to the Action Center","markReviewed": "Mark as reviewed","reviewedError": "Failed to mark as reviewed",+ "duplicatesMerged": "{count, plural, one {1 similar suggestion was merged into this one.} other {# similar suggestions were merged into this one.}}",+ "duplicatesMergedHint": "They rested on the same prompts and AI searches, so they weren't created and didn't use your quota.","newSignal": "New signal detected","newSignalScoreRose": "Completed earlier, and its opportunity has grown since: the score rose from {from} to {to}.","newSignalGainLost": "Completed earlier, but the gain did not hold: visibility on these prompts was {after}% after the work and is {now}% now."diff --git a/web/src/components/content/opportunity-detail.tsx b/web/src/components/content/opportunity-detail.tsxindex 5f1b17c2c..0b361338b 100644--- a/web/src/components/content/opportunity-detail.tsx+++ b/web/src/components/content/opportunity-detail.tsx@@ -26,6 +26,7 @@ import {Check,Copy,ExternalLink,+ Info,} from 'lucide-react';import { cn } from '@/lib/utils';import { Link } from '@/i18n/navigation';@@ -452,6 +453,18 @@ export function OpportunityDetail({</div>)}+ {(sd.duplicatesMerged ?? 0) > 0 && (+ <div className="flex items-start gap-2 rounded-md border bg-muted/40 px-4 py-3 text-xs">+ <Info className="mt-0.5 h-3.5 w-3.5 shrink-0 text-muted-foreground" />+ <p>+ <span className="font-medium">+ {t('duplicatesMerged', { count: sd.duplicatesMerged ?? 0 })}+ </span>{' '}+ <span className="text-muted-foreground">{t('duplicatesMergedHint')}</span>+ </p>+ </div>+ )}+{opportunity.clusterId && (<div className="grid grid-cols-1 gap-4 @5xl:grid-cols-3"><div className="space-y-4 @5xl:col-span-2">diff --git a/web/src/types/index.ts b/web/src/types/index.tsindex dddf5270d..cd88e2ccb 100644--- a/web/src/types/index.ts+++ b/web/src/types/index.ts@@ -227,6 +227,11 @@ export interface ContentOpportunitySourceData {actions?: Record<string, string>;/** Actions from before the opportunity was re-opened. */pastActions?: string[];+ /**+ * Duplicates folded into this opportunity instead of being created+ * (CONTENT-004): clusters whose prompt and fan-out baskets matched its own.+ */+ duplicatesMerged?: number;/** Why a finished opportunity went back to New (#857, Phase 4). */reopened?: {at: string;