Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(citations): write citations as rows alongside each answer (#737)
Gökhan Geyik7d583abf5f06
diff --git a/server/src/lib/citation-rows.js b/server/src/lib/citation-rows.jsnew file mode 100644index 000000000..9d8cddda2--- /dev/null+++ b/server/src/lib/citation-rows.js@@ -0,0 +1,171 @@+/**+ * Citations as rows (#732).+ *+ * `prompt_results.citations` keeps the provider's array as jsonb, which is+ * fine to store and expensive to read: every page load expands it again, and+ * pulling the host out of each URL is where the time goes. This writes the+ * same citations out once, at the moment the answer is stored, so reading them+ * later is an indexed scan over narrow rows instead.+ *+ * Exported so the backfill script runs the exact same code path over historical+ * answers rather than a second implementation that could disagree with this one.+ */++import supabaseAdmin from '../config/supabase.js';+import logger from './logger.js';++/**+ * A btree entry cannot exceed roughly 2,704 bytes, and the unique index on+ * `url` is what makes the dictionary a dictionary. The longest URL seen in+ * production is 1,333 characters; truncating well above that keeps one absurd+ * link from failing the insert and taking the citation with it.+ */+export const MAX_URL_LENGTH = 2048;++/**+ * Host without `www.`, lowercased — a port of `extractHostname` from+ * `web/src/lib/citations/classify.ts`, fallback included. Kept deliberately+ * identical: the stored domain has to group the way the surface groups, or the+ * two disagree about what a domain is, and every figure shifts when the page+ * switches over to reading these rows.+ */+export function extractDomain(rawUrl) {+ if (!rawUrl || typeof rawUrl !== 'string') return null;+ try {+ const host = new URL(rawUrl.trim()).hostname.replace(/^www\./i, '').toLowerCase();+ return host || null;+ } catch {+ // Not every citation arrives as a well-formed URL — 252 of the 2,023,979+ // in production have no scheme at all. The page counts those today by+ // recovering a hostname-like token, so this does too; dropping them here+ // would quietly lower every figure the new path reports against the old.+ const match = rawUrl.match(/^(?:https?:\/\/)?(?:www\.)?([^/\s?#]+)/i);+ return match ? match[1].toLowerCase() : null;+ }+}++/**+ * Citation entries worth storing, in array order.+ *+ * Entries without a parsable URL are dropped rather than stored with a null+ * domain: they cannot be grouped, counted per domain, or clicked, so a row for+ * one would only be a row that every reader has to remember to exclude.+ * `position` is the index in the original array, so the identity of a citation+ * survives that filtering.+ */+export function normalizeCitations(rawCitations) {+ if (!Array.isArray(rawCitations)) return [];++ const out = [];+ rawCitations.forEach((cite, index) => {+ const rawUrl = typeof cite?.url === 'string' ? cite.url.trim() : '';+ if (!rawUrl) return;+ const domain = extractDomain(rawUrl);+ if (!domain) return;++ out.push({+ position: index,+ url: rawUrl.slice(0, MAX_URL_LENGTH),+ domain,+ title: typeof cite?.title === 'string' ? cite.title.trim() || null : null,+ });+ });+ return out;+}++/**+ * Resolve URLs to dictionary ids, inserting the ones not seen before.+ *+ * Two steps rather than one upsert-returning: `ignoreDuplicates` does not+ * return the rows it skipped, so an upsert alone would leave the already-known+ * URLs — the overwhelming majority — without ids. Insert only what is missing,+ * then read back the whole set.+ *+ * The re-read after insert is not redundant. Two runs can insert the same new+ * URL concurrently; the loser's insert is ignored rather than failing, and the+ * final select is what gives it the winner's id.+ */+async function resolveUrlIds(entries) {+ const byUrl = new Map();+ for (const entry of entries) {+ if (!byUrl.has(entry.url)) byUrl.set(entry.url, entry);+ }+ const urls = [...byUrl.keys()];+ if (urls.length === 0) return new Map();++ const { data: known, error: knownErr } = await supabaseAdmin+ .from('citation_urls')+ .select('id, url')+ .in('url', urls);+ if (knownErr) throw new Error(`citation url lookup: ${knownErr.message}`);++ const ids = new Map((known ?? []).map((row) => [row.url, row.id]));+ const missing = urls.filter((url) => !ids.has(url));++ if (missing.length > 0) {+ const { error: insertErr } = await supabaseAdmin.from('citation_urls').upsert(+ missing.map((url) => {+ const entry = byUrl.get(url);+ return { url, domain: entry.domain, title: entry.title };+ }),+ { onConflict: 'url', ignoreDuplicates: true },+ );+ if (insertErr) throw new Error(`citation url insert: ${insertErr.message}`);++ const { data: added, error: addedErr } = await supabaseAdmin+ .from('citation_urls')+ .select('id, url')+ .in('url', missing);+ if (addedErr) throw new Error(`citation url re-read: ${addedErr.message}`);+ for (const row of added ?? []) ids.set(row.url, row.id);+ }++ return ids;+}++/**+ * Write one answer's citations.+ *+ * Best effort by design: the caller has already stored the answer, and a+ * failure here must not cost that. The jsonb column still holds the same data,+ * so anything missed is recoverable by re-running the backfill over the+ * affected window.+ *+ * @returns {Promise<number>} rows written (0 when there is nothing to write, or on failure)+ */+export async function persistCitationRows({ promptResultId, brandId, createdAt, citations }) {+ const entries = normalizeCitations(citations);+ if (entries.length === 0) return 0;++ try {+ const urlIds = await resolveUrlIds(entries);++ const rows = entries+ .map((entry) => ({+ prompt_result_id: promptResultId,+ position: entry.position,+ url_id: urlIds.get(entry.url),+ brand_id: brandId,+ created_at: createdAt,+ }))+ .filter((row) => row.url_id !== undefined);++ if (rows.length === 0) return 0;++ // Idempotent on (prompt_result_id, position): a retried insert, a+ // re-delivered webhook, or a re-run of the backfill over the same answer+ // adds nothing and errors on nothing.+ const { error } = await supabaseAdmin+ .from('prompt_result_citations')+ .upsert(rows, { onConflict: 'prompt_result_id,position', ignoreDuplicates: true });+ if (error) throw new Error(error.message);++ return rows.length;+ } catch (err) {+ logger.error(+ { err, promptResultId, brandId, citationCount: entries.length },+ '[citations] failed to persist citation rows',+ );+ return 0;+ }+}diff --git a/server/src/lib/citation-rows.test.js b/server/src/lib/citation-rows.test.jsnew file mode 100644index 000000000..21986f8db--- /dev/null+++ b/server/src/lib/citation-rows.test.js@@ -0,0 +1,110 @@+import { describe, expect, it, vi } from 'vitest';++// The module reaches the supabase admin client at import time, whose config+// hard-exits when SUPABASE_* env is absent (as in CI).+vi.mock('../config/supabase.js', () => ({ default: {} }));++import { extractDomain, normalizeCitations, MAX_URL_LENGTH } from './citation-rows.js';++/**+ * What gets written per citation (#732).+ *+ * The stored domain has to group exactly the way the Citations page groups, or+ * the aggregate and the surface disagree about what a domain is — so these+ * pin the rule rather than the implementation.+ */+describe('extractDomain', () => {+ it('lowercases the host and drops www.', () => {+ expect(extractDomain('https://WWW.Example.COM/a/b?c=d')).toBe('example.com');+ });++ it('keeps subdomains other than www', () => {+ expect(extractDomain('https://docs.example.com/guide')).toBe('docs.example.com');+ expect(extractDomain('https://www2.example.com/')).toBe('www2.example.com');+ });++ it('ignores port, path, query and fragment', () => {+ expect(extractDomain('https://example.com:8443/x?y=1#z')).toBe('example.com');+ });++ // 252 of the 2,023,979 citations in production arrive without a scheme.+ // The page counts them today by recovering a hostname-like token, so a+ // writer that dropped them would quietly lower every figure once the page+ // reads these rows instead.+ it('recovers a host from a URL with no scheme', () => {+ expect(extractDomain('example.com/page')).toBe('example.com');+ expect(extractDomain('www.example.com/page')).toBe('example.com');+ expect(extractDomain('WWW.EXAMPLE.COM')).toBe('example.com');+ });++ it('returns null only when there is no host-like token at all', () => {+ for (const bad of ['', ' ', '/relative/path', null, undefined, 42, {}]) {+ expect(extractDomain(bad)).toBeNull();+ }+ });+});++describe('normalizeCitations', () => {+ it('keeps the array index as position', () => {+ const rows = normalizeCitations([+ { url: 'https://a.com/1' },+ { url: 'https://b.com/2' },+ { url: 'https://c.com/3' },+ ]);+ expect(rows.map((r) => r.position)).toEqual([0, 1, 2]);+ });++ // Dropping an entry must not renumber the ones after it: position is half+ // the primary key, so a shifted index would collide with a different+ // citation on a re-run.+ it('preserves the original index when an entry is dropped', () => {+ const rows = normalizeCitations([+ { url: 'https://a.com/1' },+ { url: '/relative/only' },+ { url: 'https://c.com/3' },+ ]);+ expect(rows.map((r) => r.position)).toEqual([0, 2]);+ expect(rows.map((r) => r.domain)).toEqual(['a.com', 'c.com']);+ });++ it('drops entries with no usable URL rather than storing a null domain', () => {+ const rows = normalizeCitations([+ { url: '' },+ { url: ' ' },+ { title: 'no url at all' },+ { url: '/relative/only' },+ null,+ undefined,+ ]);+ expect(rows).toEqual([]);+ });++ it('truncates a URL that would not fit a btree entry', () => {+ const long = `https://example.com/${'x'.repeat(5000)}`;+ const [row] = normalizeCitations([{ url: long }]);+ expect(row.url).toHaveLength(MAX_URL_LENGTH);+ expect(row.domain).toBe('example.com');+ });++ it('trims the title and stores null when it is empty', () => {+ expect(normalizeCitations([{ url: 'https://a.com', title: ' Hello ' }])[0].title).toBe(+ 'Hello',+ );+ expect(normalizeCitations([{ url: 'https://a.com', title: ' ' }])[0].title).toBeNull();+ expect(normalizeCitations([{ url: 'https://a.com' }])[0].title).toBeNull();+ });++ it('returns an empty list for anything that is not an array', () => {+ for (const bad of [null, undefined, {}, 'citations', 7]) {+ expect(normalizeCitations(bad)).toEqual([]);+ }+ });++ // The same page cited twice in one answer is two citations, not one: the+ // page's own count of how often it was cited depends on it.+ it('keeps repeated URLs as separate rows', () => {+ const rows = normalizeCitations([{ url: 'https://a.com/x' }, { url: 'https://a.com/x' }]);+ expect(rows).toHaveLength(2);+ expect(rows.map((r) => r.position)).toEqual([0, 1]);+ });+});diff --git a/server/src/lib/cloro-result-handler.js b/server/src/lib/cloro-result-handler.jsindex d6e891a70..8d8656393 100644--- a/server/src/lib/cloro-result-handler.js+++ b/server/src/lib/cloro-result-handler.js@@ -18,6 +18,7 @@ import { analyzeSentimentAI } from './ai-tracker.js';import { parseResponse, countBrandMentions } from './response-parser.js';import { normalizeShoppingCards, matchCardBrand } from './shopping-cards.js';import { updateTargetUrlStats } from './target-url-stats.js';+import { persistCitationRows } from './citation-rows.js';import { logger } from './logger.js';/**@@ -70,7 +71,7 @@ export async function handleScraperResult({inline_products: aiResponse.inline_products ?? [],search_queries: Array.isArray(aiResponse.search_queries) ? aiResponse.search_queries : [],})- .select('id')+ .select('id, created_at').single();if (error) {@@ -81,6 +82,17 @@ export async function handleScraperResult({// Best-effort: mark target URLs cited by this answer (00032).await updateTargetUrlStats(promptId, aiResponse.citations, new Date().toISOString());+ // Expand the citation array into rows (#732). Best-effort for the same+ // reason as the line above: the answer is already stored, and the jsonb+ // column still holds this data, so a failure here costs a backfill and+ // nothing else.+ await persistCitationRows({+ promptResultId: inserted.id,+ brandId,+ createdAt: inserted.created_at,+ citations: aiResponse.citations,+ });+// Best-effort normalization of any shopping cards on this result.// Failures here must not abort the result insert — the raw cards are// already persisted on prompt_results.shopping_cards and the backfilldiff --git a/server/src/scripts/backfill-citations.js b/server/src/scripts/backfill-citations.jsnew file mode 100644index 000000000..0e95f7d79--- /dev/null+++ b/server/src/scripts/backfill-citations.js@@ -0,0 +1,122 @@+/**+ * Backfill prompt_result_citations from the raw prompt_results.citations jsonb.+ *+ * Run: `node src/scripts/backfill-citations.js`+ *+ * Options via env:+ * - CITATIONS_BACKFILL_BATCH answers pulled per batch (default 200)+ * - CITATIONS_BACKFILL_DAYS only answers newer than this many days+ * (default: no limit — the whole history)+ * - CITATIONS_BACKFILL_PAUSE milliseconds to wait between batches+ * (default 100)+ *+ * Idempotent and restartable. `persistCitationRows` upserts on+ * `(prompt_result_id, position)` with `ignoreDuplicates`, so re-running skips+ * what is already there, and the walk is keyset-based on `(created_at, id)`+ * rather than OFFSET — a run that is interrupted can simply be started again.+ *+ * Deliberately unhurried. There are ~2M citations to write across ~177k+ * answers on a database that is already 1.1 GB on a 2 GB instance, and this+ * runs against the same instance serving the dashboard. Small batches with a+ * pause between them keep it from evicting the cache that live queries depend+ * on; finishing in twenty minutes instead of five is a good trade.+ */++import 'dotenv/config';+import supabaseAdmin from '../config/supabase.js';+import { persistCitationRows } from '../lib/citation-rows.js';++const BATCH = Number.parseInt(process.env.CITATIONS_BACKFILL_BATCH ?? '200', 10);+const PAUSE_MS = Number.parseInt(process.env.CITATIONS_BACKFILL_PAUSE ?? '100', 10);+const DAYS = process.env.CITATIONS_BACKFILL_DAYS+ ? Number.parseInt(process.env.CITATIONS_BACKFILL_DAYS, 10)+ : null;++const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));++/**+ * One page of answers, ordered by `(created_at, id)` so the cursor is total.+ *+ * Ordering on created_at alone would be ambiguous: a nightly run writes+ * thousands of answers inside the same second, and a cursor that cannot+ * separate them either repeats a page or skips one.+ */+async function fetchBatch(cursor) {+ let query = supabaseAdmin+ .from('prompt_results')+ .select('id, brand_id, created_at, citations')+ .not('citations', 'is', null)+ .order('created_at', { ascending: true })+ .order('id', { ascending: true })+ .limit(BATCH);++ if (DAYS !== null) {+ query = query.gte('created_at', new Date(Date.now() - DAYS * 86_400_000).toISOString());+ }+ if (cursor) {+ // `or` expresses the keyset condition: strictly later timestamp, or the+ // same timestamp and a later id.+ query = query.or(+ `created_at.gt.${cursor.createdAt},and(created_at.eq.${cursor.createdAt},id.gt.${cursor.id})`,+ );+ }++ const { data, error } = await query;+ if (error) throw new Error(`fetch batch: ${error.message}`);+ return data ?? [];+}++async function main() {+ const startedAt = Date.now();+ let cursor = null;+ let answers = 0;+ let citations = 0;+ let empty = 0;++ console.log(+ `Backfilling citations — batch ${BATCH}, pause ${PAUSE_MS}ms` ++ (DAYS !== null ? `, last ${DAYS} days` : ', full history'),+ );++ for (;;) {+ const batch = await fetchBatch(cursor);+ if (batch.length === 0) break;++ for (const row of batch) {+ const written = await persistCitationRows({+ promptResultId: row.id,+ brandId: row.brand_id,+ createdAt: row.created_at,+ citations: row.citations,+ });+ answers += 1;+ citations += written;+ if (written === 0) empty += 1;+ }++ const last = batch[batch.length - 1];+ cursor = { createdAt: last.created_at, id: last.id };++ const elapsed = Math.round((Date.now() - startedAt) / 1000);+ console.log(+ ` ${answers} answers · ${citations} citations · ${empty} with none · ${elapsed}s · at ${last.created_at}`,+ );++ if (batch.length < BATCH) break;+ if (PAUSE_MS > 0) await sleep(PAUSE_MS);+ }++ const elapsed = Math.round((Date.now() - startedAt) / 1000);+ console.log(+ `\nDone. ${answers} answers scanned, ${citations} citation rows written, ` ++ `${empty} answers had none, ${elapsed}s.`,+ );+}++main().catch((err) => {+ // Exit non-zero so a wrapper can tell a crash from a clean finish. The walk+ // is restartable, so the fix is to run it again rather than to resume from+ // some recorded position.+ console.error('Backfill failed:', err);+ process.exit(1);+});diff --git a/server/src/workers/tracking-worker.js b/server/src/workers/tracking-worker.jsindex f43802940..fb2a5d3b8 100644--- a/server/src/workers/tracking-worker.js+++ b/server/src/workers/tracking-worker.js@@ -11,6 +11,7 @@ import { hasFeature, getPlan, isCloud } from '../config/plans.js';import { applyPlanOverrides } from '../lib/plan-guard.js';import { generateContentOpportunities } from '../lib/opportunity-generator.js';import { updateTargetUrlStats } from '../lib/target-url-stats.js';+import { persistCitationRows } from '../lib/citation-rows.js';import logger from '../lib/logger.js';function resolveModelPlatform(model) {@@ -262,7 +263,14 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,let completedTasks = 0;async function insertResult(row) {- const { error } = await supabaseAdmin.from('prompt_results').insert(row);+ // `created_at` comes back from the insert rather than being stamped here:+ // the citation rows copy it, and a value computed in app code would drift+ // from the column default by the round trip.+ const { data: inserted, error } = await supabaseAdmin+ .from('prompt_results')+ .insert(row)+ .select('id, created_at')+ .single();if (error) {logger.error({ err: error, brandId }, 'failed to insert tracking result');throw error;@@ -270,6 +278,15 @@ export async function processTrackingJob({ brandId, promptId, promptIds, source,insertedCount++;// Best-effort: mark target URLs cited by this answer (00032).await updateTargetUrlStats(row.prompt_id, row.citations, new Date().toISOString());+ // Best-effort: expand the citation array into rows (#732). The jsonb+ // column still holds the same data, so anything missed here is recoverable+ // with the backfill rather than lost.+ await persistCitationRows({+ promptResultId: inserted.id,+ brandId: row.brand_id,+ createdAt: inserted.created_at,+ citations: row.citations,+ });}// 6. Phase 1: Collect & run all scraper (platform) tasks firstdiff --git a/supabase/migrations/00058_prompt_result_citations.sql b/supabase/migrations/00058_prompt_result_citations.sqlnew file mode 100644index 000000000..01def1762--- /dev/null+++ b/supabase/migrations/00058_prompt_result_citations.sql@@ -0,0 +1,129 @@+-- Citations as rows (#732), phase 1: the tables and the write path.+--+-- `prompt_results.citations` holds the provider's citation array as jsonb.+-- Reading it means expanding every answer on every page load: on the largest+-- brand, 51,679 answers become 679,304 citation entries, and pulling the host+-- out of each URL costs 4.4s of the 4.6s an aggregate over that window takes —+-- above the 8s statement timeout once anything else is asked for. The Citations+-- page works around it by paging the raw rows out to the app tier instead: 50+-- sequential requests carrying 65 MB of jsonb, capped at 50,000 rows, which+-- silently truncates that same brand at 51,679.+--+-- Expanding once at write time turns that into an ordinary indexed read.+--+-- Two tables rather than one, because the naive shape does not fit. Across the+-- database there are 2,023,979 citation entries but only 449,559 distinct URLs+-- — the expensive part (url ~92 bytes, title ~63) repeats 4.5 times on+-- average. One row per citation with the text inline measures 513 MB of heap+-- against 262 MB split this way, and the instance has 2 GB of RAM to hold a+-- database that is already 1.1 GB.+--+-- Phase 1 only fills the tables; nothing reads them yet. The Citations page+-- keeps using the jsonb until phase 2 lands the aggregate RPC, so the two can+-- be compared against each other before anything switches over.++-- ─── Dictionary ──────────────────────────────────────────────────────────────++create table if not exists public.citation_urls (+ id bigint generated always as identity primary key,+ -- Truncated to 2048 chars on write. The longest URL observed is 1,333, and+ -- a btree entry cannot exceed ~2,704 bytes — without a bound, one absurd URL+ -- would fail the unique index and take the whole result insert with it.+ url text not null,+ -- Host without `www.`, lowercased. Derived from the URL at write time so a+ -- domain rollup never has to parse 2M strings; the application's+ -- extractHostname() is the single definition of how.+ domain text not null,+ -- Whatever title the provider sent the first time this URL appeared.+ -- Providers word the same page differently between answers, and one stable+ -- label beats re-deciding per citation.+ title text,+ first_seen_at timestamptz not null default now()+);++create unique index if not exists citation_urls_url_key on public.citation_urls (url);+create index if not exists citation_urls_domain_idx on public.citation_urls (domain);++-- ─── Facts ───────────────────────────────────────────────────────────────────++create table if not exists public.prompt_result_citations (+ prompt_result_id uuid not null references public.prompt_results(id) on delete cascade,+ -- Index within the answer's citation array. With prompt_result_id this is+ -- the natural identity of a citation, which is what keeps the write path and+ -- the backfill idempotent when either re-runs over the same answer.+ position integer not null,+ url_id bigint not null references public.citation_urls(id),++ -- Denormalized from the parent answer. brand_id carries the row-level+ -- security rule, which cannot be expressed through a join, and created_at+ -- lets a brand's window be read without touching prompt_results — the table+ -- whose 1 GB of TOAST this change exists to stop reading.+ brand_id uuid not null references public.brands(id) on delete cascade,+ created_at timestamptz not null,++ primary key (prompt_result_id, position)+);++-- The Citations page always asks the same question first: this brand, this+-- window. Everything else narrows what comes back.+create index if not exists prompt_result_citations_brand_created_idx+ on public.prompt_result_citations (brand_id, created_at desc);++-- Reverse lookup: which answers cited this URL.+create index if not exists prompt_result_citations_url_idx+ on public.prompt_result_citations (url_id);++-- ─── Row level security ──────────────────────────────────────────────────────++alter table public.prompt_result_citations enable row level security;++-- Mirrors prompt_results and prompt_result_shopping_cards: org members read+-- their own org's rows, the service role writes. The worker goes through+-- supabaseAdmin and bypasses RLS, but the explicit policy keeps the table+-- reachable from an authenticated surface.+create policy "citations: org member select"+ on public.prompt_result_citations+ for select+ using (+ brand_id in (+ select b.id+ from public.brands b+ where b.organization_id in (+ select organization_id+ from public.profiles+ where id = auth.uid()+ )+ )+ );++create policy "Service role can insert citations"+ on public.prompt_result_citations+ for insert+ with check (true);++create policy "Service role can delete citations"+ on public.prompt_result_citations+ for delete+ using (true);++-- The dictionary is shared across every organization and has no tenant column,+-- so no row-level rule can express who may read a given URL — the answer lives+-- in the join, not in the row. RLS is therefore enabled with no select policy+-- at all: direct reads are denied to everyone but the service role, and the+-- phase 2 aggregate function will reach it as SECURITY DEFINER after checking+-- the caller owns the brand it was asked about. Adding a permissive policy+-- here would let any authenticated user enumerate every URL ever cited for+-- every customer.+alter table public.citation_urls enable row level security;++create policy "Service role can insert citation urls"+ on public.citation_urls+ for insert+ with check (true);++comment on table public.prompt_result_citations is+ 'One row per citation per answer (#732). Written alongside prompt_results by the tracking worker and the Cloro webhook handler, and backfilled by server/src/scripts/backfill-citations.js. Source of truth for the Citations page from phase 2 onward; prompt_results.citations remains the raw archival copy.';+comment on table public.citation_urls is+ 'Deduplicated URL dictionary for prompt_result_citations. Cross-tenant by design: the same page is cited for many brands, and storing the text once is what keeps the citation table a quarter of the size it would otherwise be.';+comment on column public.prompt_result_citations.position is+ 'Index within prompt_results.citations. With prompt_result_id it is the natural key that makes re-running the write path or the backfill idempotent.';diff --git a/supabase/schema.sql b/supabase/schema.sqlindex b1ad4caa9..e41134525 100644--- a/supabase/schema.sql+++ b/supabase/schema.sql@@ -5802,3 +5802,136 @@ $$;GRANT EXECUTE ON FUNCTION public.topics_overview_aggregates(uuid) TO authenticated;+-- ─────────────────────────────────────────────────────────────────────────+-- migrations/00058_prompt_result_citations.sql+-- ─────────────────────────────────────────────────────────────────────────+-- Citations as rows (#732), phase 1: the tables and the write path.+--+-- `prompt_results.citations` holds the provider's citation array as jsonb.+-- Reading it means expanding every answer on every page load: on the largest+-- brand, 51,679 answers become 679,304 citation entries, and pulling the host+-- out of each URL costs 4.4s of the 4.6s an aggregate over that window takes —+-- above the 8s statement timeout once anything else is asked for. The Citations+-- page works around it by paging the raw rows out to the app tier instead: 50+-- sequential requests carrying 65 MB of jsonb, capped at 50,000 rows, which+-- silently truncates that same brand at 51,679.+--+-- Expanding once at write time turns that into an ordinary indexed read.+--+-- Two tables rather than one, because the naive shape does not fit. Across the+-- database there are 2,023,979 citation entries but only 449,559 distinct URLs+-- — the expensive part (url ~92 bytes, title ~63) repeats 4.5 times on+-- average. One row per citation with the text inline measures 513 MB of heap+-- against 262 MB split this way, and the instance has 2 GB of RAM to hold a+-- database that is already 1.1 GB.+--+-- Phase 1 only fills the tables; nothing reads them yet. The Citations page+-- keeps using the jsonb until phase 2 lands the aggregate RPC, so the two can+-- be compared against each other before anything switches over.++-- ─── Dictionary ──────────────────────────────────────────────────────────────++create table if not exists public.citation_urls (+ id bigint generated always as identity primary key,+ -- Truncated to 2048 chars on write. The longest URL observed is 1,333, and+ -- a btree entry cannot exceed ~2,704 bytes — without a bound, one absurd URL+ -- would fail the unique index and take the whole result insert with it.+ url text not null,+ -- Host without `www.`, lowercased. Derived from the URL at write time so a+ -- domain rollup never has to parse 2M strings; the application's+ -- extractHostname() is the single definition of how.+ domain text not null,+ -- Whatever title the provider sent the first time this URL appeared.+ -- Providers word the same page differently between answers, and one stable+ -- label beats re-deciding per citation.+ title text,+ first_seen_at timestamptz not null default now()+);++create unique index if not exists citation_urls_url_key on public.citation_urls (url);+create index if not exists citation_urls_domain_idx on public.citation_urls (domain);++-- ─── Facts ───────────────────────────────────────────────────────────────────++create table if not exists public.prompt_result_citations (+ prompt_result_id uuid not null references public.prompt_results(id) on delete cascade,+ -- Index within the answer's citation array. With prompt_result_id this is+ -- the natural identity of a citation, which is what keeps the write path and+ -- the backfill idempotent when either re-runs over the same answer.+ position integer not null,+ url_id bigint not null references public.citation_urls(id),++ -- Denormalized from the parent answer. brand_id carries the row-level+ -- security rule, which cannot be expressed through a join, and created_at+ -- lets a brand's window be read without touching prompt_results — the table+ -- whose 1 GB of TOAST this change exists to stop reading.+ brand_id uuid not null references public.brands(id) on delete cascade,+ created_at timestamptz not null,++ primary key (prompt_result_id, position)+);++-- The Citations page always asks the same question first: this brand, this+-- window. Everything else narrows what comes back.+create index if not exists prompt_result_citations_brand_created_idx+ on public.prompt_result_citations (brand_id, created_at desc);++-- Reverse lookup: which answers cited this URL.+create index if not exists prompt_result_citations_url_idx+ on public.prompt_result_citations (url_id);++-- ─── Row level security ──────────────────────────────────────────────────────++alter table public.prompt_result_citations enable row level security;++-- Mirrors prompt_results and prompt_result_shopping_cards: org members read+-- their own org's rows, the service role writes. The worker goes through+-- supabaseAdmin and bypasses RLS, but the explicit policy keeps the table+-- reachable from an authenticated surface.+create policy "citations: org member select"+ on public.prompt_result_citations+ for select+ using (+ brand_id in (+ select b.id+ from public.brands b+ where b.organization_id in (+ select organization_id+ from public.profiles+ where id = auth.uid()+ )+ )+ );++create policy "Service role can insert citations"+ on public.prompt_result_citations+ for insert+ with check (true);++create policy "Service role can delete citations"+ on public.prompt_result_citations+ for delete+ using (true);++-- The dictionary is shared across every organization and has no tenant column,+-- so no row-level rule can express who may read a given URL — the answer lives+-- in the join, not in the row. RLS is therefore enabled with no select policy+-- at all: direct reads are denied to everyone but the service role, and the+-- phase 2 aggregate function will reach it as SECURITY DEFINER after checking+-- the caller owns the brand it was asked about. Adding a permissive policy+-- here would let any authenticated user enumerate every URL ever cited for+-- every customer.+alter table public.citation_urls enable row level security;++create policy "Service role can insert citation urls"+ on public.citation_urls+ for insert+ with check (true);++comment on table public.prompt_result_citations is+ 'One row per citation per answer (#732). Written alongside prompt_results by the tracking worker and the Cloro webhook handler, and backfilled by server/src/scripts/backfill-citations.js. Source of truth for the Citations page from phase 2 onward; prompt_results.citations remains the raw archival copy.';+comment on table public.citation_urls is+ 'Deduplicated URL dictionary for prompt_result_citations. Cross-tenant by design: the same page is cited for many brands, and storing the text once is what keeps the citation table a quarter of the size it would otherwise be.';+comment on column public.prompt_result_citations.position is+ 'Index within prompt_results.citations. With prompt_result_id it is the natural key that makes re-running the write path or the backfill idempotent.';+