Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
chore(server): clean ChatGPT's entity tags out of stored answers and recount them, with backfill-mention-counts.js --entity-markup (#1551)
Cihan49f0ab54ed6c
diff --git a/server/src/lib/mention-recount.js b/server/src/lib/mention-recount.jsnew file mode 100644index 000000000..7d9bc9d38--- /dev/null+++ b/server/src/lib/mention-recount.js@@ -0,0 +1,63 @@+/**+ * A stored answer's mentions, counted again by today's response parser: the+ * backfills that fix how answers were counted (backfill-mention-counts.js)+ * rewrite what it derives.+ *+ * - mention_count, visibility_score, mention_position, mentioned_entity_count+ * - mention_count, visibility_score and mention_position of each existing+ * competitor_mentions entry whose competitor still exists+ * - with `cleanMarkup`, the answer text itself without ChatGPT's+ * `<Entity …/>` tags (entity-markup.js), as new answers are saved+ *+ * Sentiment is not re-analysed: an answer that now counts as mentioning the+ * brand keeps the 'neutral' it was stored with, because re-running the model+ * over thousands of answers is a separate decision. One that no longer+ * mentions it is 'neutral', as an answer is saved (cloro-result-handler.js):+ * a brand named only in an entity tag's disambig had its sentiment read.+ */+import { stripEntityMarkup } from './entity-markup.js';+import { parseResponse } from './response-parser.js';++/** The row's new values, or null when nothing changes. */+export function recountRow(row, brandInfo, competitors, { cleanMarkup = false } = {}) {+ const response = cleanMarkup ? stripEntityMarkup(row.response ?? '') : (row.response ?? '');+ const answer = { text: response, citations: Array.isArray(row.citations) ? row.citations : [] };+ let sentiment = row.sentiment ?? 'neutral';+ let metrics = parseResponse(answer, brandInfo, sentiment, competitors);+ if (metrics.mentionCount === 0 && sentiment !== 'neutral') {+ sentiment = 'neutral';+ metrics = parseResponse(answer, brandInfo, sentiment, competitors);+ }+ const fresh = new Map(metrics.competitorMentions.map((m) => [m.competitor_id, m]));+ const entries = Array.isArray(row.competitor_mentions) ? row.competitor_mentions : [];+ const competitorMentions = entries.map((entry) => {+ const m = fresh.get(entry.competitor_id);+ if (!m) return entry;+ return {+ ...entry,+ mention_count: m.mention_count,+ visibility_score: m.visibility_score,+ mention_position: m.mention_position,+ };+ });++ const next = {+ mention_count: metrics.mentionCount,+ visibility_score: metrics.visibilityScore,+ mention_position: metrics.mentionPosition,+ mentioned_entity_count: metrics.mentionedEntityCount,+ competitor_mentions: competitorMentions,+ };+ // Only a text the tags came out of, and a sentiment that moved, are written back.+ if (cleanMarkup && response !== (row.response ?? '')) next.response = response;+ if (sentiment !== (row.sentiment ?? 'neutral')) next.sentiment = sentiment;+ const unchanged =+ next.response === undefined &&+ next.sentiment === undefined &&+ next.mention_count === row.mention_count &&+ next.visibility_score === row.visibility_score &&+ next.mention_position === row.mention_position &&+ next.mentioned_entity_count === row.mentioned_entity_count &&+ JSON.stringify(competitorMentions) === JSON.stringify(entries);+ return unchanged ? null : next;+}diff --git a/server/src/lib/mention-recount.test.js b/server/src/lib/mention-recount.test.jsnew file mode 100644index 000000000..a609177e4--- /dev/null+++ b/server/src/lib/mention-recount.test.js@@ -0,0 +1,156 @@+import { describe, expect, it } from 'vitest';+import { recountRow } from './mention-recount.js';++const brandInfo = { brandName: 'Acme', domains: ['acme.example'] };+const competitors = [+ { id: 'c1', name: 'Rival', domain: 'rival.example' },+ { id: 'c2', name: 'Other', domain: '' },+];++/** A stored answer as the backfill reads it, its counts as they were saved. */+const stored = (response, over = {}) => ({+ response,+ citations: [],+ sentiment: 'neutral',+ mention_count: 0,+ visibility_score: 0,+ mention_position: null,+ mentioned_entity_count: 0,+ competitor_mentions: [+ {+ competitor_id: 'c1',+ name: 'Rival',+ mention_count: 0,+ visibility_score: 0,+ mention_position: null,+ },+ {+ competitor_id: 'c2',+ name: 'Other',+ mention_count: 0,+ visibility_score: 0,+ mention_position: null,+ },+ ],+ ...over,+});++/** The row as it reads once `next` is written. */+const written = (row, next) => ({ ...row, ...next });++describe('recountRow', () => {+ // ChatGPT's tag for Acme, whose disambig names Rival: saved before #1097,+ // Rival counted as named first and Acme second.+ const tagged =+ 'Pick <Entity category="software" value="Acme" disambig="Rival alternative"/> for teams. Other is fine too.';++ it('cleans the tags out of the text and counts what the answer reads', () => {+ const row = stored(tagged, {+ mention_count: 1,+ mention_position: 2,+ mentioned_entity_count: 3,+ competitor_mentions: [+ {+ competitor_id: 'c1',+ name: 'Rival',+ mention_count: 1,+ visibility_score: 30,+ mention_position: 1,+ },+ {+ competitor_id: 'c2',+ name: 'Other',+ mention_count: 1,+ visibility_score: 30,+ mention_position: 3,+ },+ ],+ });+ const next = recountRow(row, brandInfo, competitors, { cleanMarkup: true });++ expect(next.response).toBe('Pick Acme for teams. Other is fine too.');+ expect(next.mention_count).toBe(1);+ expect(next.mention_position).toBe(1);+ expect(next.mentioned_entity_count).toBe(2);+ // Rival was only in the tag's disambig.+ expect(next.competitor_mentions[0]).toMatchObject({+ competitor_id: 'c1',+ mention_count: 0,+ mention_position: null,+ });+ expect(next.competitor_mentions[1]).toMatchObject({+ competitor_id: 'c2',+ mention_count: 1,+ mention_position: 2,+ });+ // Run again over what it wrote: nothing left to change.+ expect(+ recountRow(written(row, next), brandInfo, competitors, { cleanMarkup: true }),+ ).toBeNull();+ });++ it('keeps the text as it is without cleanMarkup, and counts as the parser reads it', () => {+ const next = recountRow(stored(tagged), brandInfo, competitors);+ // The parser reads the answer without its tags either way.+ expect(next.response).toBeUndefined();+ expect(next.mention_count).toBe(1);+ expect(next.competitor_mentions[0].mention_count).toBe(0);+ });++ it('writes nothing for an answer already counted as it reads', () => {+ const row = stored('Acme leads.', {});+ const next = recountRow(row, brandInfo, competitors, { cleanMarkup: true });+ expect(next.response).toBeUndefined();+ expect(+ recountRow(written(row, next), brandInfo, competitors, { cleanMarkup: true }),+ ).toBeNull();+ });++ it('leaves a tag without a value in the text, and writes nothing more for it', () => {+ const text = 'See <Entity category="software"/> and Acme.';+ const row = stored(text);+ const next = recountRow(row, brandInfo, competitors, { cleanMarkup: true });+ expect(next.response).toBeUndefined();+ expect(+ recountRow(written(row, next), brandInfo, competitors, { cleanMarkup: true }),+ ).toBeNull();+ });++ it('reads an answer that no longer names the brand as neutral, as one is saved', () => {+ // The brand only in the tag's disambig: its sentiment was read as if named.+ const row = stored(+ 'Try <Entity category="software" value="Rival" disambig="Acme alternative"/> today.',+ { sentiment: 'positive', mention_count: 1, visibility_score: 55, mention_position: 2 },+ );+ const next = recountRow(row, brandInfo, competitors, { cleanMarkup: true });+ expect(next).toMatchObject({+ response: 'Try Rival today.',+ sentiment: 'neutral',+ mention_count: 0,+ mention_position: null,+ });+ // Positive no longer lifts the score of an answer that doesn't name it.+ expect(next.visibility_score).toBe(+ recountRow(stored('Try Rival today.'), brandInfo, competitors)?.visibility_score ?? 0,+ );+ expect(+ recountRow(written(row, next), brandInfo, competitors, { cleanMarkup: true }),+ ).toBeNull();+ // Still named: its sentiment stays as it was read.+ const named = stored('Acme leads.', { sentiment: 'positive' });+ expect(recountRow(named, brandInfo, competitors)?.sentiment).toBeUndefined();+ });++ it('keeps an entry whose competitor is gone as it was', () => {+ const gone = {+ competitor_id: 'c9',+ name: 'Gone',+ mention_count: 4,+ visibility_score: 50,+ mention_position: 1,+ };+ const row = stored('Acme and Rival.', { competitor_mentions: [gone] });+ const next = recountRow(row, brandInfo, competitors, { cleanMarkup: true });+ expect(next.competitor_mentions).toEqual([gone]);+ });+});diff --git a/server/src/scripts/backfill-mention-counts.js b/server/src/scripts/backfill-mention-counts.jsindex 515a7f01b..dd109d27e 100644--- a/server/src/scripts/backfill-mention-counts.js+++ b/server/src/scripts/backfill-mention-counts.js@@ -1,33 +1,64 @@/**- * Recount brand and competitor mentions on historical prompt_results (#871).+ * Recount brand and competitor mentions on historical prompt_results: re-runs+ * parseResponse over the stored answer text and rewrites what it derives+ * (mention-recount.js).*- * Names starting or ending with a character outside ASCII [A-Za-z0-9_]- * ("Ümit", "İpek", "Acme, Inc.", "星云科技") were never matched, so- * their answers were stored with zero mentions. This re-runs parseResponse- * over the stored answer text and rewrites what it derives:+ * Run: `node src/scripts/backfill-mention-counts.js [--entity-markup] [--brand <id>] [--dry-run]`*- * - mention_count, visibility_score, mention_position, mentioned_entity_count- * - mention_count, visibility_score and mention_position of each existing- * competitor_mentions entry whose competitor still exists+ * Two fixes, two scopes:+ * - by default (#871): names starting or ending with a character outside+ * ASCII [A-Za-z0-9_] ("Ümit", "İpek", "Acme, Inc.", "星云科技") were never+ * matched, so their answers were stored with zero mentions. Brands of+ * active or trialing organizations whose own name or any competitor name+ * is such a name; every other brand's counts are unchanged by the fix.+ * - `--entity-markup` (#1097): ChatGPT answers saved with its+ * `<Entity … value="X" …/>` tags in the text. The tags' attributes counted+ * as mentions and shifted positions. Every brand of an active or trialing+ * organization, and only its answers that still hold a tag: the text is+ * saved without the tags, as new answers are, and recounted.+ * --brand narrows to one brand regardless. Rows are counted by today's parser+ * (mention-recount.js): the ones it rewrites pick up its other changes too.*- * Sentiment is not re-analysed: an answer that now counts as mentioning the- * brand keeps the 'neutral' it was stored with, because re-running the model- * over thousands of answers is a separate decision.+ * Before --entity-markup runs on production, keep a copy: nothing else holds+ * the text with its tags. In a schema of its own (no grants: a table made by+ * hand in public gets that schema's, with row level security off), a week of+ * answers at a time on the date index, dropped once the run is checked:+ * create schema if not exists backfill_backup;+ * create table backfill_backup.prompt_results_entity_markup as+ * select id, brand_id, created_at, response, sentiment, mention_count, visibility_score,+ * mention_position, mentioned_entity_count, competitor_mentions+ * from public.prompt_results+ * where false;+ * -- once per week of answers, oldest first:+ * insert into backfill_backup.prompt_results_entity_markup+ * select id, brand_id, created_at, response, sentiment, mention_count, visibility_score,+ * mention_position, mentioned_entity_count, competitor_mentions+ * from public.prompt_results+ * where created_at >= '<day>' and created_at < '<day + 7>'+ * and response like '%<Entity%';+ * To put it back, in parts as it was filled: one brand at a time (or a week+ * at a time on b.created_at), then run the three refreshes it printed again:+ * update public.prompt_results p+ * set response = b.response, sentiment = b.sentiment, mention_count = b.mention_count,+ * visibility_score = b.visibility_score, mention_position = b.mention_position,+ * mentioned_entity_count = b.mentioned_entity_count,+ * competitor_mentions = b.competitor_mentions+ * from backfill_backup.prompt_results_entity_markup b+ * where p.id = b.id and b.brand_id = '<id>';+ * Once the run is checked, on its own, never with the restore:+ * drop schema backfill_backup cascade;*- * Run: `node src/scripts/backfill-mention-counts.js [--brand <id>] [--dry-run]`- *- * Scope: brands of active or trialing organizations whose own name or any- * competitor name starts or ends outside [A-Za-z0-9_] — every other brand's- * counts are unchanged by the fix. --brand narrows to one brand regardless.- *- * Idempotent: rows whose recomputed values equal the stored ones are skipped.- * Afterwards, refresh each changed brand's rollups with- * `INSIGHTS_BACKFILL_BRAND=<id> node src/scripts/backfill-insights-daily.js`;- * the ids are printed at the end.+ * Idempotent: rows whose recomputed values equal the stored ones are skipped,+ * and a cleaned answer leaves the filter, so a stopped run is run again. Each+ * changed brand's refresh commands are printed as soon as its rows are+ * written: a run stopped later, and run again, finds that brand clean. A dry+ * run prints a few rows per brand, before → after, and its sentiment changes.*/import 'dotenv/config';import supabaseAdmin from '../config/supabase.js';-import { parseResponse } from '../lib/response-parser.js';+import { IN_FILTER_CHUNK, selectInChunks } from '../lib/chunked-in.js';+import { addDays, dayChunks, resultDayBound } from '../lib/citations-rollups.js';+import { recountRow } from '../lib/mention-recount.js';const BATCH_SIZE = 250;const BATCH_PAUSE_MS = 250;@@ -35,15 +66,25 @@ const CONCURRENCY = 20;const args = process.argv.slice(2);const dryRun = args.includes('--dry-run');+const entityMarkup = args.includes('--entity-markup');const brandArg = args.includes('--brand') ? args[args.indexOf('--brand') + 1] : null;const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));-/** Retry transient network failures (connection resets on long walks). */+/**+ * Retry transient network failures (connection resets on long walks): thrown,+ * or returned by supabase-js as an error with no code.+ */async function withRetry(fn, label) {for (let attempt = 1; ; attempt++) {try {- return await fn();+ const result = await fn();+ if (result?.error && !result.error.code && attempt < 4) {+ console.warn(` retry ${attempt}/3 after error in ${label}: ${result.error.message}`);+ await sleep(attempt * 2000);+ continue;+ }+ return result;} catch (err) {if (attempt >= 4) throw err;console.warn(` retry ${attempt}/3 after error in ${label}: ${err.message}`);@@ -75,55 +116,169 @@ async function brandsInScope() {return brands ?? [];}+/**+ * The brand's domains and competitors, as an answer is counted with them. A+ * read that fails stops the run: counted without them, every answer would+ * lose its domain mentions or its competitors, and a cleaned answer leaves+ * the filter, so running again wouldn't revisit it.+ */async function brandContext(brand) {- const [{ data: domains }, { data: competitors }] = await Promise.all([- supabaseAdmin.from('brand_domains').select('domain').eq('brand_id', brand.id),- supabaseAdmin.from('competitors').select('id, name, domain').eq('brand_id', brand.id),+ const [domainsRes, competitorsRes] = await Promise.all([+ withRetry(+ () => supabaseAdmin.from('brand_domains').select('domain').eq('brand_id', brand.id),+ 'brand domains',+ ),+ withRetry(+ () => supabaseAdmin.from('competitors').select('id, name, domain').eq('brand_id', brand.id),+ 'competitors',+ ),]);+ if (domainsRes.error) throw new Error(domainsRes.error.message);+ if (competitorsRes.error) throw new Error(competitorsRes.error.message);return {- brandInfo: { brandName: brand.name, domains: (domains ?? []).map((d) => d.domain) },- competitors: competitors ?? [],+ brandInfo: {+ brandName: brand.name,+ domains: (domainsRes.data ?? []).map((d) => d.domain),+ },+ competitors: competitorsRes.data ?? [],};}-/** The row's new values, or null when nothing changes. */-function recount(row, brandInfo, competitors) {- const metrics = parseResponse(- { text: row.response ?? '', citations: Array.isArray(row.citations) ? row.citations : [] },- brandInfo,- row.sentiment ?? 'neutral',- competitors,+const ROW_COLUMNS =+ 'id, response, citations, sentiment, mention_count, visibility_score, mention_position, mentioned_entity_count, competitor_mentions';++/** Rows a dry run prints per brand, before → after. */+const DRY_RUN_SAMPLES = 3;++/** The competitors an answer names, as its entries count them. */+const namedCompetitors = (entries) =>+ (Array.isArray(entries) ? entries : [])+ .filter((e) => e.mention_count > 0)+ .map((e) => e.name)+ .join(', ') || '—';++/** One row's change, as a dry run prints it. */+function sampleLine(row, next) {+ const moved = (label, before, after) =>+ before === after ? null : `${label} ${before ?? '—'} → ${after ?? '—'}`;+ const parts = [+ moved('mentions', row.mention_count, next.mention_count),+ moved('position', row.mention_position, next.mention_position),+ moved(+ 'competitors named',+ namedCompetitors(row.competitor_mentions),+ namedCompetitors(next.competitor_mentions),+ ),+ next.response !== undefined ? 'text cleaned' : null,+ next.sentiment !== undefined ? `sentiment ${row.sentiment} → ${next.sentiment}` : null,+ ].filter(Boolean);+ return ` ${row.id}: ${parts.join(' · ') || 'score only'}`;+}++/**+ * Recounts these rows and writes the ones that changed (a dry run writes+ * none, and keeps a few as samples); returns [updated, skipped] and adds the+ * sentiment changes to `report`.+ */+async function recountRows(rows, brandInfo, competitors, report) {+ let updated = 0;+ let skipped = 0;+ for (let i = 0; i < rows.length; i += CONCURRENCY) {+ const chunk = rows.slice(i, i + CONCURRENCY);+ await Promise.all(+ chunk.map(async (row) => {+ const next = recountRow(row, brandInfo, competitors, { cleanMarkup: entityMarkup });+ if (!next) {+ skipped++;+ return;+ }+ if (next.sentiment !== undefined) report.sentiment++;+ if (dryRun && report.samples.length < DRY_RUN_SAMPLES) {+ report.samples.push(sampleLine(row, next));+ }+ if (!dryRun) {+ const { error: updateErr } = await withRetry(+ () => supabaseAdmin.from('prompt_results').update(next).eq('id', row.id),+ 'row update',+ );+ if (updateErr) throw new Error(updateErr.message);+ }+ updated++;+ }),+ );+ }+ return [updated, skipped];+}++/** Tagged answers read per page: one UTC day of a brand's holds fewer. */+const ID_PAGE = 1000;++/**+ * The brand's answers that still hold a ChatGPT entity tag, by id, a UTC day+ * of answers at a time (the brand and date index; each answer's text read+ * once). Listed before any is cleaned: a cleaned answer leaves the filter,+ * which would shift offset pages taken during the writes.+ */+async function taggedRowIds(brandId) {+ const [first, last] = await withRetry(+ () => Promise.all([resultDayBound(brandId, true), resultDayBound(brandId, false)]),+ 'answer days',);- const fresh = new Map(metrics.competitorMentions.map((m) => [m.competitor_id, m]));- const entries = Array.isArray(row.competitor_mentions) ? row.competitor_mentions : [];- const competitorMentions = entries.map((entry) => {- const m = fresh.get(entry.competitor_id);- if (!m) return entry;- return {- ...entry,- mention_count: m.mention_count,- visibility_score: m.visibility_score,- mention_position: m.mention_position,- };- });-- const next = {- mention_count: metrics.mentionCount,- visibility_score: metrics.visibilityScore,- mention_position: metrics.mentionPosition,- mentioned_entity_count: metrics.mentionedEntityCount,- competitor_mentions: competitorMentions,- };- const unchanged =- next.mention_count === row.mention_count &&- next.visibility_score === row.visibility_score &&- next.mention_position === row.mention_position &&- next.mentioned_entity_count === row.mentioned_entity_count &&- JSON.stringify(competitorMentions) === JSON.stringify(entries);- return unchanged ? null : next;+ if (!first || !last) return [];+ const ids = [];+ for (const [day] of dayChunks(first, last, 1)) {+ for (let offset = 0; ; offset += ID_PAGE) {+ const { data, error } = await withRetry(+ () =>+ supabaseAdmin+ .from('prompt_results')+ .select('id')+ .eq('brand_id', brandId)+ .gte('created_at', `${day}T00:00:00Z`)+ .lt('created_at', `${addDays(day, 1)}T00:00:00Z`)+ .like('response', '%<Entity%')+ .order('id', { ascending: true })+ .range(offset, offset + ID_PAGE - 1),+ 'tagged ids',+ );+ if (error) throw new Error(error.message);+ ids.push(...(data ?? []).map((r) => r.id));+ if (!data || data.length < ID_PAGE) break;+ }+ }+ return ids;+}++async function backfillTagged(brand, report) {+ const { brandInfo, competitors } = await brandContext(brand);+ const ids = await taggedRowIds(brand.id);+ let updated = 0;+ let skipped = 0;+ for (let i = 0; i < ids.length; i += BATCH_SIZE) {+ // A URL of 100 ids at a time (chunked-in.js).+ const { data: rows, error } = await withRetry(+ () =>+ selectInChunks(+ ids.slice(i, i + BATCH_SIZE),+ (chunk) => supabaseAdmin.from('prompt_results').select(ROW_COLUMNS).in('id', chunk),+ IN_FILTER_CHUNK,+ ),+ 'page fetch',+ );+ if (error) throw new Error(error.message);+ const [u, k] = await recountRows(rows ?? [], brandInfo, competitors, report);+ updated += u;+ skipped += k;+ console.log(+ ` ${brand.name}: ${updated} changed, ${skipped} unchanged of ${ids.length} tagged`,+ );+ await sleep(BATCH_PAUSE_MS);+ }+ return updated;}-async function backfillBrand(brand) {+async function backfillBrand(brand, report) {+ if (entityMarkup) return backfillTagged(brand, report);const { brandInfo, competitors } = await brandContext(brand);let updated = 0;let skipped = 0;@@ -135,9 +290,7 @@ async function backfillBrand(brand) {() =>supabaseAdmin.from('prompt_results')- .select(- 'id, response, citations, sentiment, mention_count, visibility_score, mention_position, mentioned_entity_count, competitor_mentions',- )+ .select(ROW_COLUMNS).eq('brand_id', brand.id).not('response', 'is', null).order('created_at', { ascending: false })@@ -148,26 +301,9 @@ async function backfillBrand(brand) {if (error) throw new Error(error.message);if (!rows?.length) break;- for (let i = 0; i < rows.length; i += CONCURRENCY) {- const chunk = rows.slice(i, i + CONCURRENCY);- await Promise.all(- chunk.map(async (row) => {- const next = recount(row, brandInfo, competitors);- if (!next) {- skipped++;- return;- }- if (!dryRun) {- const { error: updateErr } = await withRetry(- () => supabaseAdmin.from('prompt_results').update(next).eq('id', row.id),- 'row update',- );- if (updateErr) throw new Error(updateErr.message);- }- updated++;- }),- );- }+ const [u, k] = await recountRows(rows, brandInfo, competitors, report);+ updated += u;+ skipped += k;console.log(` ${brand.name}: ${updated} changed, ${skipped} unchanged`);if (rows.length < BATCH_SIZE) break;@@ -180,34 +316,58 @@ async function backfillBrand(brand) {const candidates = await brandsInScope();const brands = [];for (const brand of candidates) {- if (brandArg || isAffectedName(brand.name)) {+ if (brandArg || entityMarkup || isAffectedName(brand.name)) {brands.push(brand);continue;}- const { data: comps } = await supabaseAdmin- .from('competitors')- .select('name')- .eq('brand_id', brand.id);+ // A failed read stops the run, never drops the brand from it unsaid.+ const { data: comps, error: compsError } = await withRetry(+ () => supabaseAdmin.from('competitors').select('name').eq('brand_id', brand.id),+ 'competitors',+ );+ if (compsError) throw new Error(compsError.message);if ((comps ?? []).some((c) => isAffectedName(c.name))) brands.push(brand);}-console.log(`Recounting mentions for ${brands.length} brand(s)${dryRun ? ' — DRY RUN' : ''}`);+console.log(+ `Recounting mentions${entityMarkup ? ' in answers with entity tags' : ''} for ${brands.length} brand(s)${dryRun ? ' — DRY RUN' : ''}`,+);++/**+ * A changed brand's refreshes. Each reads the counts: Insights' rollups;+ * Mentions' with the answers' terms (an entity's terms are kept only where+ * it's mentioned), their rollups and Perception's snapshots; Citations'+ * (we_mention, comp_present).+ */+const refreshCommands = (brand) => [+ ` INSIGHTS_BACKFILL_BRAND=${brand.id} node src/scripts/backfill-insights-daily.js # ${brand.name}`,+ ` MENTIONS_BACKFILL_BRAND=${brand.id} node src/scripts/backfill-mentions-daily.js # ${brand.name}`,+ ` CITATIONS_BACKFILL_BRAND=${brand.id} node src/scripts/backfill-citations-daily.js # ${brand.name}`,+];const changed = [];let total = 0;for (const brand of brands) {- const n = await backfillBrand(brand);+ const report = { samples: [], sentiment: 0 };+ const n = await backfillBrand(brand, report);total += n;- if (n > 0) changed.push(brand);+ if (n === 0) continue;+ changed.push(brand);+ if (dryRun) {+ console.log(+ ` ${brand.name}: ${report.sentiment} sentiment change(s) to neutral; for example:`,+ );+ for (const line of report.samples) console.log(line);+ } else {+ // Now, not at the end: a run stopped later and run again finds this brand clean.+ console.log(` ${brand.name} written; refresh its rollups:`);+ for (const line of refreshCommands(brand)) console.log(line);+ }}console.log(`Done. ${total} row(s) ${dryRun ? 'would be ' : ''}updated.`);if (changed.length && !dryRun) {- console.log('Refresh the rollups for the changed brands:');- for (const brand of changed) {- console.log(- ` INSIGHTS_BACKFILL_BRAND=${brand.id} node src/scripts/backfill-insights-daily.js # ${brand.name}`,- );- }+ console.log('Refresh the rollups for the changed brands (each was printed as it finished):');+ for (const brand of changed) for (const line of refreshCommands(brand)) console.log(line);}process.exit(0);