Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(prompts): read each tracked prompt's real demand every night (PROMPT-014, part 1) (#1650)

Cihan554a31358e04
diff --git a/docs/self-host/env-variables.mdx b/docs/self-host/env-variables.mdx
index 01d99037d..6ab2fc9cb 100644
--- a/docs/self-host/env-variables.mdx
+++ b/docs/self-host/env-variables.mdx
@@ -49,6 +49,7 @@ Fed to the Express API + workers.
| `IS_CLOUD` | optional | `"true"` only for the official cloud build |
| `DATAFORSEO_LOGIN` | optional | DataForSEO login (for prompt volume analysis) |
| `DATAFORSEO_PASSWORD` | optional | DataForSEO password |
+| `DEMAND_SIGNALS` | optional | `"true"` to read each tracked prompt's Google Trends, People Also Ask and YouTube demand from DataForSEO about once a week (the Demand card on a prompt's page). **Costs money:** each read is a paid DataForSEO task, about $0.004 a prompt a week, at most 500 prompts a night. Off by default; needs the DataForSEO keys above. Search Console demand is read every night either way, at no cost. |
| `CLORO_API_KEY` | optional | Cloro web scraper API key (for ChatGPT/Gemini/Perplexity web scraping) |
| `PLATFORM_PROVIDER` | optional | Default `cloro`. Used to pick scraper backend |
| `SCRAPEDO_API_KEY` | Site Audit | Scrape.do API key — required to run **Site Audit** (it fetches target pages via Scrape.do's proxy + JS render). Distinct from `CLORO_API_KEY`, which scrapes the AI answer engines. |
diff --git a/server/.env.example b/server/.env.example
index 01cd739be..2bdba5eb4 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -106,6 +106,12 @@ IS_CLOUD=false
# DataForSEO (for keyword volume data)
DATAFORSEO_LOGIN=
DATAFORSEO_PASSWORD=
+# A prompt's real demand (PROMPT-014): read Google Trends, People Also Ask
+# and YouTube for each tracked prompt's top keyword about once a week, as
+# DataForSEO Standard tasks (about $0.004 a prompt a week; at most 500
+# prompts a night). Off unless "true"; needs the DataForSEO keys above.
+# Search Console demand is read every night either way, at no cost.
+# DEMAND_SIGNALS=
# Cloro (platform web scrapers)
CLORO_API_KEY=
diff --git a/server/src/lib/dataforseo.js b/server/src/lib/dataforseo.js
index 0a5cae73c..ea8429ece 100644
--- a/server/src/lib/dataforseo.js
+++ b/server/src/lib/dataforseo.js
@@ -1,5 +1,13 @@
-const DATAFORSEO_API_URL =
- 'https://api.dataforseo.com/v3/keywords_data/google_ads/search_volume/live';
+const DATAFORSEO_BASE_URL = 'https://api.dataforseo.com';
+const DATAFORSEO_API_URL = `${DATAFORSEO_BASE_URL}/v3/keywords_data/google_ads/search_volume/live`;
+
+/** Tasks one task_post request takes (DataForSEO's own limit). */
+export const MAX_TASKS_PER_POST = 100;
+
+/** Whether the server has DataForSEO's login and password. */
+export function isDataForSeoConfigured() {
+ return Boolean(process.env.DATAFORSEO_LOGIN && process.env.DATAFORSEO_PASSWORD);
+}
function getAuthHeader() {
const login = process.env.DATAFORSEO_LOGIN;
@@ -65,3 +73,112 @@ export async function getSearchVolumes(keywords, options = {}) {
return volumes;
}
+
+/** One request to DataForSEO's API, its whole answer once the request itself went through. */
+async function callApi(path, { method = 'GET', body } = {}) {
+ const response = await fetch(`${DATAFORSEO_BASE_URL}${path}`, {
+ method,
+ headers: {
+ Authorization: getAuthHeader(),
+ ...(body ? { 'Content-Type': 'application/json' } : {}),
+ },
+ body: body ? JSON.stringify(body) : undefined,
+ signal: AbortSignal.timeout(60_000),
+ });
+
+ if (!response.ok) {
+ throw new Error(`DataForSEO API error (${response.status}): ${response.statusText}`);
+ }
+
+ const data = await response.json();
+ if (data.status_code !== 20000) {
+ throw new Error(`DataForSEO error: ${data.status_message || 'Unknown error'}`);
+ }
+ return data;
+}
+
+/**
+ * Standard (queued) tasks, set at once: `path` is the endpoint's own, without
+ * `/task_post` (e.g. /v3/serp/google/organic). Each task carries a `tag`, and
+ * the answer maps each tag to its task's id; a task DataForSEO refused has no
+ * entry. Charged as each task is set.
+ *
+ * Sent MAX_TASKS_PER_POST at a time. A request that fails costs only its own
+ * tasks: the others are still set, and what was set comes back. `onChunk`
+ * is awaited with each request's ids as soon as they're in, so the caller
+ * can keep them before the next one goes (a task set is paid for); an error
+ * it throws stops the rest.
+ *
+ * @param {string} path
+ * @param {Array<{ tag: string } & Record<string, unknown>>} tasks
+ * @param {{ onChunk?: (ids: Map<string, string>) => Promise<void> | void }} [options]
+ * @returns {Promise<{ ids: Map<string, string>, failed: number, errors: string[] }>}
+ * tag → task id; `failed`: tasks in the requests that failed, and why
+ */
+export async function postTasks(path, tasks, { onChunk } = {}) {
+ const ids = new Map();
+ const errors = [];
+ let failed = 0;
+ for (let i = 0; i < tasks.length; i += MAX_TASKS_PER_POST) {
+ const chunk = tasks.slice(i, i + MAX_TASKS_PER_POST);
+ let data;
+ try {
+ data = await callApi(`${path}/task_post`, { method: 'POST', body: chunk });
+ } catch (err) {
+ failed += chunk.length;
+ errors.push(err?.message ?? String(err));
+ continue;
+ }
+ const set = new Map();
+ for (const task of data.tasks ?? []) {
+ // 20100: Task Created.
+ const tag = task?.data?.tag;
+ if (task?.status_code === 20100 && task.id && typeof tag === 'string') set.set(tag, task.id);
+ }
+ for (const [tag, id] of set) ids.set(tag, id);
+ if (onChunk && set.size > 0) await onChunk(set);
+ }
+ return { ids, failed, errors };
+}
+
+/**
+ * The endpoint's tasks that are done and not collected yet (up to 1,000,
+ * from the last three days), each with the tag it was set with. A task
+ * leaves the list once its result is read.
+ *
+ * @param {string} path e.g. /v3/keywords_data/google_trends/explore
+ * @returns {Promise<Map<string, string | null>>} task id → tag
+ */
+export async function readyTasks(path) {
+ const data = await callApi(`${path}/tasks_ready`);
+ const ready = new Map();
+ for (const task of data.tasks ?? []) {
+ for (const done of task?.result ?? []) {
+ if (done?.id) ready.set(done.id, typeof done.tag === 'string' ? done.tag : null);
+ }
+ }
+ return ready;
+}
+
+/** Status codes of a task still being worked on: 40601 Task Handed, 40602 Task In Queue. */
+const PENDING = new Set([40601, 40602]);
+
+/**
+ * One task's result, read from its task_get path (with the id in it).
+ * `done` with the first result, or with null when the search found nothing
+ * (40102); `pending` while it waits its turn; `gone` when DataForSEO no
+ * longer has it (40401, results are kept 30 days).
+ *
+ * @param {string} path e.g. /v3/serp/google/organic/task_get/advanced/<id>
+ * @returns {Promise<{ status: 'done', result: object | null } | { status: 'pending' | 'gone' }>}
+ */
+export async function getTask(path) {
+ const data = await callApi(path);
+ const task = data.tasks?.[0];
+ if (!task) throw new Error('DataForSEO task error: No task returned');
+ if (task.status_code === 20000) return { status: 'done', result: task.result?.[0] ?? null };
+ if (task.status_code === 40102) return { status: 'done', result: null };
+ if (PENDING.has(task.status_code)) return { status: 'pending' };
+ if (task.status_code === 40401) return { status: 'gone' };
+ throw new Error(`DataForSEO task error: ${task.status_message || task.status_code}`);
+}
diff --git a/server/src/lib/dataforseo.test.js b/server/src/lib/dataforseo.test.js
new file mode 100644
index 000000000..7c20906da
--- /dev/null
+++ b/server/src/lib/dataforseo.test.js
@@ -0,0 +1,202 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+/**
+ * DataForSEO's queued (Standard) tasks (PROMPT-014): set with task_post,
+ * listed by tasks_ready once done, read with task_get. No request leaves
+ * the test: fetch answers with DataForSEO's documented bodies.
+ */
+
+const { MAX_TASKS_PER_POST, getTask, isDataForSeoConfigured, postTasks, readyTasks } =
+ await import('./dataforseo.js');
+
+let requests;
+let answers;
+
+beforeEach(() => {
+ vi.stubEnv('DATAFORSEO_LOGIN', 'login');
+ vi.stubEnv('DATAFORSEO_PASSWORD', 'secret');
+ requests = [];
+ answers = [];
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (url, init) => {
+ requests.push({ url, method: init.method, body: init.body ? JSON.parse(init.body) : null });
+ const body = answers.shift();
+ return { ok: true, status: 200, statusText: 'OK', json: async () => body };
+ }),
+ );
+});
+
+afterEach(() => {
+ vi.unstubAllEnvs();
+ vi.unstubAllGlobals();
+});
+
+const envelope = (tasks) => ({ status_code: 20000, status_message: 'Ok.', tasks });
+
+describe('isDataForSeoConfigured', () => {
+ it('needs both the login and the password', () => {
+ expect(isDataForSeoConfigured()).toBe(true);
+ vi.stubEnv('DATAFORSEO_PASSWORD', '');
+ expect(isDataForSeoConfigured()).toBe(false);
+ });
+});
+
+describe('postTasks', () => {
+ it("sets the tasks, at most 100 a request, and maps each tag to its task's id", async () => {
+ const tasks = Array.from({ length: MAX_TASKS_PER_POST + 1 }, (_, i) => ({
+ keyword: `k${i}`,
+ tag: `p${i}`,
+ }));
+ answers.push(
+ envelope(
+ tasks.slice(0, MAX_TASKS_PER_POST).map((t, i) => ({
+ id: `id-${i}`,
+ // One refused: no entry for it.
+ status_code: i === 1 ? 40501 : 20100,
+ status_message: i === 1 ? 'Invalid Field.' : 'Task Created.',
+ data: { keyword: t.keyword, tag: t.tag },
+ result: null,
+ })),
+ ),
+ envelope([
+ {
+ id: 'id-last',
+ status_code: 20100,
+ data: { tag: `p${MAX_TASKS_PER_POST}` },
+ result: null,
+ },
+ ]),
+ );
+
+ const chunks = [];
+ const { ids, failed, errors } = await postTasks('/v3/serp/google/organic', tasks, {
+ onChunk: (set) => chunks.push(set.size),
+ });
+
+ expect(requests.map((r) => [r.url, r.method, r.body.length])).toEqual([
+ ['https://api.dataforseo.com/v3/serp/google/organic/task_post', 'POST', 100],
+ ['https://api.dataforseo.com/v3/serp/google/organic/task_post', 'POST', 1],
+ ]);
+ expect(ids.get('p0')).toBe('id-0');
+ expect(ids.has('p1')).toBe(false);
+ expect(ids.get(`p${MAX_TASKS_PER_POST}`)).toBe('id-last');
+ expect(ids.size).toBe(MAX_TASKS_PER_POST);
+ expect(chunks).toEqual([MAX_TASKS_PER_POST - 1, 1]);
+ expect({ failed, errors }).toEqual({ failed: 0, errors: [] });
+ });
+
+ it('keeps what one request set when another fails, and hands each over as soon as it is set', async () => {
+ const tasks = Array.from({ length: MAX_TASKS_PER_POST * 2 + 1 }, (_, i) => ({ tag: `p${i}` }));
+ const created = (from, to) =>
+ envelope(
+ tasks.slice(from, to).map((t, i) => ({
+ id: `id-${from + i}`,
+ status_code: 20100,
+ data: { tag: t.tag },
+ })),
+ );
+ answers.push(
+ created(0, MAX_TASKS_PER_POST),
+ // The second request is refused: its 100 tasks aren't set.
+ { status_code: 50000, status_message: 'Internal Error.', tasks: [] },
+ created(MAX_TASKS_PER_POST * 2, MAX_TASKS_PER_POST * 2 + 1),
+ );
+ const handed = [];
+
+ const { ids, failed, errors } = await postTasks('/v3/serp/google/organic', tasks, {
+ onChunk: async (set) => handed.push([...set.keys()]),
+ });
+
+ expect(requests).toHaveLength(3);
+ expect(ids.size).toBe(MAX_TASKS_PER_POST + 1);
+ expect(ids.get('p0')).toBe('id-0');
+ expect(ids.has(`p${MAX_TASKS_PER_POST}`)).toBe(false);
+ expect(ids.get(`p${MAX_TASKS_PER_POST * 2}`)).toBe(`id-${MAX_TASKS_PER_POST * 2}`);
+ expect(failed).toBe(MAX_TASKS_PER_POST);
+ expect(errors).toEqual(['DataForSEO error: Internal Error.']);
+ expect(handed.map((tags) => tags.length)).toEqual([MAX_TASKS_PER_POST, 1]);
+ });
+
+ it('answers what failed when every request is refused', async () => {
+ answers.push({ status_code: 40100, status_message: 'You are not authorized.', tasks: [] });
+ const onChunk = vi.fn();
+ expect(await postTasks('/v3/serp/youtube/organic', [{ tag: 'p1' }], { onChunk })).toEqual({
+ ids: new Map(),
+ failed: 1,
+ errors: ['DataForSEO error: You are not authorized.'],
+ });
+ expect(onChunk).not.toHaveBeenCalled();
+ });
+
+ it('stops when the caller cannot keep what was set', async () => {
+ const tasks = Array.from({ length: MAX_TASKS_PER_POST + 1 }, (_, i) => ({ tag: `p${i}` }));
+ answers.push(envelope([{ id: 'id-0', status_code: 20100, data: { tag: 'p0' } }]));
+ await expect(
+ postTasks('/v3/serp/google/organic', tasks, {
+ onChunk: async () => {
+ throw new Error('write failed');
+ },
+ }),
+ ).rejects.toThrow('write failed');
+ // The second request never went.
+ expect(requests).toHaveLength(1);
+ });
+});
+
+describe('readyTasks', () => {
+ it("lists the done tasks' ids with their tags", async () => {
+ answers.push(
+ envelope([
+ {
+ id: 'x',
+ status_code: 20000,
+ result: [
+ {
+ id: '10311535-0696-0093-0000-8fe3770f079b',
+ se: 'google',
+ function: 'explore',
+ date_posted: '2026-10-27 22:00:04 +00:00',
+ tag: 'p1',
+ endpoint:
+ '/v3/keywords_data/google_trends/explore/task_get/10311535-0696-0093-0000-8fe3770f079b',
+ },
+ ],
+ },
+ ]),
+ );
+ const ready = await readyTasks('/v3/keywords_data/google_trends/explore');
+ expect(requests[0].url).toBe(
+ 'https://api.dataforseo.com/v3/keywords_data/google_trends/explore/tasks_ready',
+ );
+ expect([...ready]).toEqual([['10311535-0696-0093-0000-8fe3770f079b', 'p1']]);
+ });
+});
+
+describe('getTask', () => {
+ const read = (status_code, result = null) => {
+ answers.push(envelope([{ id: 't1', status_code, status_message: '', result }]));
+ return getTask('/v3/serp/youtube/organic/task_get/advanced/t1');
+ };
+
+ it('answers done with the first result', async () => {
+ expect(await read(20000, [{ keyword: 'a', items: [] }])).toEqual({
+ status: 'done',
+ result: { keyword: 'a', items: [] },
+ });
+ });
+
+ it('answers done with nothing for a search with no results', async () => {
+ expect(await read(40102)).toEqual({ status: 'done', result: null });
+ });
+
+ it('answers pending while the task waits, and gone once DataForSEO has dropped it', async () => {
+ expect(await read(40602)).toEqual({ status: 'pending' });
+ expect(await read(40601)).toEqual({ status: 'pending' });
+ expect(await read(40401)).toEqual({ status: 'gone' });
+ });
+
+ it('fails on any other status', async () => {
+ await expect(read(50000)).rejects.toThrow('DataForSEO task error');
+ });
+});
diff --git a/server/src/lib/prompt-demand.js b/server/src/lib/prompt-demand.js
new file mode 100644
index 000000000..cc23c0b5f
--- /dev/null
+++ b/server/src/lib/prompt-demand.js
@@ -0,0 +1,958 @@
+/**
+ * Real demand for a tracked prompt (PROMPT-014, 00370): proof that people
+ * ask it, beside the estimate of its AI searches (PROMPT-013).
+ *
+ * Four signals, each kept with the day it was read:
+ *
+ * gsc the brand's Search Console queries that match the prompt, over
+ * the last 28 days: impressions, clicks, how many queries and the
+ * most seen one. Free, from gsc_query_stats, every night. A query
+ * goes to the prompt the Action Center would map it to
+ * (classify.js matchPrompt, `gscMatchOverlap`).
+ * trends Google Trends for the prompt's keyword, 12 months: the weekly
+ * line, rising, steady or falling, and the change on a year ago.
+ * paa the People Also Ask questions Google shows for the keyword.
+ * youtube the videos YouTube lists for the keyword and their views.
+ *
+ * The last three are DataForSEO's queued (Standard) tasks, read once a week
+ * per prompt, at most NIGHTLY_LIMIT prompts a night, and only with
+ * DEMAND_SIGNALS=true. The keyword is the prompt's volume analysis's own
+ * (prompt_volumes): its keyword with the most Google searches, with that
+ * analysis's location and language. A prompt never analysed, or whose
+ * keywords Google has no searches for, has none of them. A task is set one
+ * night and read once it's done: an hour later (scheduleDemandCollect), or
+ * the next night.
+ *
+ * The level weighs the signals read (demandLevel). The Demand card on the
+ * prompt's page shows all of it (GET /api/prompts/:id/demand), and the
+ * Action Center's priority reads the level.
+ *
+ * A workspace whose plan has ended is skipped (2026-10-27, #1608): nothing
+ * is matched, set or read for it.
+ */
+
+import supabaseAdmin from '../config/supabase.js';
+import { resolve } from '../config/action-engine.js';
+import { activeBrandFilter, activeOrgFilter } from './active-subscriptions.js';
+import { getTask, isDataForSeoConfigured, postTasks, readyTasks } from './dataforseo.js';
+import { matchPrompt, tokens } from './signals/library/classify.js';
+import { logger } from './logger.js';
+
+const DAY_MS = 86_400_000;
+
+/** Search Console's window, in days up to today. */
+export const GSC_WINDOW_DAYS = 28;
+/** Search Console queries read per brand, the most seen first. */
+export const GSC_QUERY_LIMIT = 5000;
+/** Days before a prompt's paid signals are set to be read again. */
+export const REFRESH_DAYS = 7;
+/** Prompts whose paid signals one night sets, so a night's cost has a ceiling. */
+export const NIGHTLY_LIMIT = 500;
+/** A task not read by then is given up (DataForSEO lists a done task for three days). */
+export const TASK_EXPIRE_DAYS = 3;
+/** When the night's tasks are read: Standard tasks take up to about 45 minutes. */
+export const COLLECT_DELAY_MS = 60 * 60_000;
+/**
+ * How long one sweep matches brands. The brands it doesn't reach are counted
+ * (`unfinished`) and go first the next night: a night takes the brands whose
+ * rows were written longest ago first (prompt_demand_brand_updates).
+ */
+export const SWEEP_DEADLINE_MS = 20 * 60_000;
+/**
+ * A sweep claims a prompt before it sets its tasks (prompt_demand_claim). A
+ * claim older than this is one whose sweep died, and may be taken again.
+ */
+export const CLAIM_STALE_MINUTES = 120;
+/** People Also Ask questions kept per prompt. */
+export const MAX_PAA_QUESTIONS = 8;
+/** Task results read at once. */
+const READ_CONCURRENCY = 5;
+/** Rows per write. */
+const WRITE_CHUNK = 500;
+/** Ids per `.in()` filter. */
+const ID_CHUNK = 100;
+/** SERP tasks need a location: the United States in English, when the analysis had none. */
+const DEFAULT_LOCATION_CODE = 2840;
+const DEFAULT_LANGUAGE_CODE = 'en';
+
+const { library: matching } = resolve();
+
+/** The paid signals, in the order the card shows them. */
+export const PAID_SIGNALS = ['trends', 'paa', 'youtube'];
+const ALL_SIGNALS = ['gsc', ...PAID_SIGNALS];
+
+/** Whether the weekly DataForSEO reads run: DEMAND_SIGNALS=true and DataForSEO's keys. */
+export function paidSignalsEnabled() {
+ return process.env.DEMAND_SIGNALS === 'true' && isDataForSeoConfigured();
+}
+
+// ─── The level ──────────────────────────────────────────────────────────────
+
+/**
+ * What each signal is worth toward the level: 2 points when it's strong, 1
+ * when there is some, 0 otherwise.
+ *
+ * gsc impressions over 28 days: 1,000 and up 2, 100 and up 1.
+ * trends rising 2, steady 1, falling or too little data 0. Trends is
+ * relative to the keyword's own peak, so only the direction says
+ * anything.
+ * paa questions Google lists: 4 and up 2, 1 and up 1.
+ * youtube views across the videos read: 100,000 and up 2, 5,000 and up 1.
+ */
+export const LEVEL_RULE = Object.freeze({
+ gscImpressions: { strong: 1000, some: 100 },
+ paaQuestions: { strong: 4, some: 1 },
+ youtubeViews: { strong: 100_000, some: 5000 },
+ /** Share of the points read for high (with at least two signals read), and for medium. */
+ high: 0.6,
+ medium: 0.3,
+ highMinSignals: 2,
+});
+
+function byThreshold(value, { strong, some }) {
+ if (value >= strong) return 2;
+ if (value >= some) return 1;
+ return 0;
+}
+
+/** A signal's points (0–2); its value is what is stored for it. */
+export function signalPoints(name, value) {
+ if (!value) return 0;
+ if (name === 'gsc') return byThreshold(Number(value.impressions) || 0, LEVEL_RULE.gscImpressions);
+ if (name === 'paa') {
+ return byThreshold(
+ Array.isArray(value.questions) ? value.questions.length : 0,
+ LEVEL_RULE.paaQuestions,
+ );
+ }
+ if (name === 'youtube') return byThreshold(Number(value.views) || 0, LEVEL_RULE.youtubeViews);
+ if (name === 'trends') return value.trend === 'rising' ? 2 : value.trend === 'steady' ? 1 : 0;
+ return 0;
+}
+
+/**
+ * The level of a prompt's demand and how many of the four signals were read.
+ * Each signal read counts, whatever it found (a read that found nothing is
+ * evidence too); a signal never read doesn't. The level is the share of the
+ * points the read signals could give: high at LEVEL_RULE.high and above
+ * with two signals or more, medium at LEVEL_RULE.medium and above, low
+ * below. Nothing read: no level.
+ *
+ * @param {{ gsc?: object|null, trends?: object|null, paa?: object|null, youtube?: object|null }} row
+ * @returns {{ level: 'high'|'medium'|'low'|null, signals: number }}
+ */
+export function demandLevel(row) {
+ let signals = 0;
+ let points = 0;
+ for (const name of ALL_SIGNALS) {
+ if (!row?.[name]) continue;
+ signals += 1;
+ points += signalPoints(name, row[name]);
+ }
+ if (signals === 0) return { level: null, signals: 0 };
+ const share = points / (2 * signals);
+ const level =
+ share >= LEVEL_RULE.high && signals >= LEVEL_RULE.highMinSignals
+ ? 'high'
+ : share >= LEVEL_RULE.medium
+ ? 'medium'
+ : 'low';
+ return { level, signals };
+}
+
+// ─── Search Console ─────────────────────────────────────────────────────────
+
+/**
+ * Each prompt's Search Console demand: the queries matched to it, summed.
+ * A query goes to one prompt at most, the closest (matchPrompt). Null when
+ * the brand has no query in the window: nothing was read, not "no demand".
+ * A prompt with no matching query gets zeros.
+ *
+ * @param {Array<{ query: string, clicks: number, impressions: number }>} queries
+ * @param {Array<{ id: string, text: string }>} prompts
+ * @param {string} at when it was read
+ * @returns {Map<string, { impressions: number, clicks: number, queries: number, top: object|null, at: string }> | null}
+ */
+export function gscDemand(queries, prompts, at) {
+ if (!Array.isArray(queries) || queries.length === 0) return null;
+ const promptTokens = new Map(prompts.map((p) => [p.id, tokens(p.text)]));
+ const demand = new Map(
+ prompts.map((p) => [p.id, { impressions: 0, clicks: 0, queries: 0, top: null, at }]),
+ );
+ for (const q of queries) {
+ const match = matchPrompt(tokens(q.query), promptTokens, matching);
+ if (!match) continue;
+ const d = demand.get(match.id);
+ const impressions = Number(q.impressions) || 0;
+ const clicks = Number(q.clicks) || 0;
+ d.impressions += impressions;
+ d.clicks += clicks;
+ d.queries += 1;
+ if (!d.top || impressions > d.top.impressions) d.top = { query: q.query, impressions, clicks };
+ }
+ return demand;
+}
+
+/** YYYY-MM-DD, `daysAgo` days before `now`, in UTC. */
+function isoDay(now, daysAgo) {
+ return new Date(now - daysAgo * DAY_MS).toISOString().slice(0, 10);
+}
+
+async function readGscQueries(brandId, now) {
+ const { data, error } = await supabaseAdmin.rpc('prompt_demand_gsc_queries', {
+ p_brand_id: brandId,
+ p_from: isoDay(now, GSC_WINDOW_DAYS - 1),
+ p_to: isoDay(now, 0),
+ p_limit: GSC_QUERY_LIMIT,
+ });
+ if (error) throw new Error(`gsc queries read failed: ${error.message}`);
+ return Array.isArray(data) ? data : [];
+}
+
+// ─── The paid signals' keyword ──────────────────────────────────────────────
+
+/**
+ * What the paid signals are read for: the analysis's keyword with the most
+ * Google searches, and the analysis's location and language. Null without
+ * an analysis or a keyword Google has searches for.
+ *
+ * @param {{ keywords?: string[], google_volumes?: Record<string, number>, location_code?: number|null, language_code?: string|null } | null} volume
+ */
+export function topKeyword(volume) {
+ if (!volume) return null;
+ let best = null;
+ for (const [keyword, searches] of Object.entries(volume.google_volumes ?? {})) {
+ const n = Number(searches) || 0;
+ if (n > 0 && keyword.trim() && (!best || n > best.searches)) {
+ best = { keyword: keyword.trim(), searches: n };
+ }
+ }
+ if (!best) return null;
+ return {
+ keyword: best.keyword,
+ locationCode: Number.isInteger(volume.location_code) ? volume.location_code : null,
+ languageCode: volume.language_code || null,
+ };
+}
+
+/**
+ * Google Trends refuses a keyword over 100 characters or with any of
+ * < > | " - + : *. Those are read as spaces; a keyword still too long has
+ * no Trends.
+ */
+export function trendsKeyword(keyword) {
+ const clean = String(keyword ?? '')
+ .replace(/[<>|"\-+:*]/g, ' ')
+ .replace(/\s+/g, ' ')
+ .trim();
+ return clean && clean.length <= 100 ? clean : null;
+}
+
+// ─── Reading the tasks' results ─────────────────────────────────────────────
+
+const round = (n) => Math.round(n);
+const average = (values) =>
+ values.length ? values.reduce((sum, v) => sum + v, 0) / values.length : 0;
+
+/** Weeks averaged at each end of the year for the change on a year ago. */
+const TREND_EDGE_WEEKS = 4;
+/** A change of this many percent or more is rising (or, down, falling). */
+export const TREND_CHANGE_PERCENT = 20;
+
+/**
+ * Google Trends' 12 months (Explore, google_trends_graph): its weekly
+ * values (0–100, relative to the keyword's own peak), the change between
+ * the first and the last four weeks, and rising, steady or falling. No week
+ * with interest: trend null, too little data.
+ */
+export function trendsSignal(result, at) {
+ const graph = (result?.items ?? []).find((item) => item?.type === 'google_trends_graph');
+ const weeks = (graph?.data ?? []).filter((week) => week && !week.missing_data);
+ const points = weeks.map((week) => {
+ const value = Array.isArray(week.values) ? Number(week.values[0]) : NaN;
+ return Number.isFinite(value) ? Math.max(0, Math.min(100, round(value))) : 0;
+ });
+ const base = { points, from: weeks[0]?.date_from ?? null, to: weeks.at(-1)?.date_to ?? null, at };
+ if (!points.some((p) => p > 0)) return { ...base, trend: null, change: null };
+ const first = average(points.slice(0, TREND_EDGE_WEEKS));
+ const last = average(points.slice(-TREND_EDGE_WEEKS));
+ if (first === 0) return { ...base, trend: last > 0 ? 'rising' : 'steady', change: null };
+ const change = round(((last - first) / first) * 100);
+ const trend =
+ change >= TREND_CHANGE_PERCENT
+ ? 'rising'
+ : change <= -TREND_CHANGE_PERCENT
+ ? 'falling'
+ : 'steady';
+ return { ...base, trend, change };
+}
+
+/**
+ * People Also Ask on Google's first page (SERP organic, advanced): each
+ * question (people_also_ask_element's title) and the page Google answers it
+ * with (its first expanded element's url and domain), once each, at most
+ * MAX_PAA_QUESTIONS.
+ */
+export function paaSignal(result, at) {
+ const questions = [];
+ const seen = new Set();
+ for (const item of result?.items ?? []) {
+ if (item?.type !== 'people_also_ask') continue;
+ for (const element of item.items ?? []) {
+ if (element?.type !== 'people_also_ask_element') continue;
+ const question = String(element.title ?? element.seed_question ?? '')
+ .trim()
+ .slice(0, 300);
+ const key = question.toLowerCase();
+ if (!question || seen.has(key)) continue;
+ seen.add(key);
+ const expanded = Array.isArray(element.expanded_element)
+ ? element.expanded_element[0]
+ : element.expanded_element;
+ questions.push({
+ question,
+ url: typeof expanded?.url === 'string' ? expanded.url : null,
+ domain: typeof expanded?.domain === 'string' ? expanded.domain : null,
+ });
+ }
+ }
+ return { questions: questions.slice(0, MAX_PAA_QUESTIONS), at };
+}
+
+/**
+ * YouTube's results for the keyword (SERP YouTube organic, advanced): the
+ * videos read (youtube_video, the first 20 results), their views summed, the
+ * most viewed, and how many results YouTube says it has (se_results_count).
+ */
+export function youtubeSignal(result, at) {
+ const videos = (result?.items ?? []).filter((item) => item?.type === 'youtube_video');
+ let views = 0;
+ let top = null;
+ for (const video of videos) {
+ const count = Number(video.views_count) || 0;
+ views += count;
+ if (!top || count > top.views) {
+ top = {
+ title: String(video.title ?? '').slice(0, 300),
+ url: typeof video.url === 'string' ? video.url : null,
+ channel: typeof video.channel_name === 'string' ? video.channel_name : null,
+ views: count,
+ published: typeof video.timestamp === 'string' ? video.timestamp : null,
+ };
+ }
+ }
+ const results = Number(result?.se_results_count);
+ return {
+ videos: videos.length,
+ views,
+ results: Number.isFinite(results) && results > 0 ? results : null,
+ top,
+ at,
+ };
+}
+
+// ─── The DataForSEO tasks ───────────────────────────────────────────────────
+
+/**
+ * Each paid signal's endpoint: where its tasks are set, listed when done and
+ * read, what a task asks (null: this keyword can't be asked) and how its
+ * result is read.
+ */
+export const SOURCES = Object.freeze({
+ trends: {
+ path: '/v3/keywords_data/google_trends/explore',
+ getPath: (id) => `/v3/keywords_data/google_trends/explore/task_get/${id}`,
+ task: ({ keyword, locationCode, languageCode, tag }) => {
+ const asked = trendsKeyword(keyword);
+ if (!asked) return null;
+ return {
+ keywords: [asked],
+ // Worldwide when the analysis had no location.
+ ...(locationCode ? { location_code: locationCode } : {}),
+ language_code: languageCode || DEFAULT_LANGUAGE_CODE,
+ type: 'web',
+ time_range: 'past_12_months',
+ item_types: ['google_trends_graph'],
+ tag,
+ };
+ },
+ read: trendsSignal,
+ },
+ paa: {
+ path: '/v3/serp/google/organic',
+ getPath: (id) => `/v3/serp/google/organic/task_get/advanced/${id}`,
+ task: ({ keyword, locationCode, languageCode, tag }) => ({
+ keyword: keyword.slice(0, 700),
+ location_code: locationCode || DEFAULT_LOCATION_CODE,
+ language_code: languageCode || DEFAULT_LANGUAGE_CODE,
+ device: 'desktop',
+ depth: 10,
+ tag,
+ }),
+ read: paaSignal,
+ },
+ youtube: {
+ path: '/v3/serp/youtube/organic',
+ getPath: (id) => `/v3/serp/youtube/organic/task_get/advanced/${id}`,
+ task: ({ keyword, locationCode, languageCode, tag }) => ({
+ keyword: keyword.slice(0, 700),
+ location_code: locationCode || DEFAULT_LOCATION_CODE,
+ language_code: languageCode || DEFAULT_LANGUAGE_CODE,
+ block_depth: 20,
+ tag,
+ }),
+ read: youtubeSignal,
+ },
+});
+
+/**
+ * A prompt is due again once its paid signals were set REFRESH_DAYS ago, or
+ * for a new keyword. Never while its tasks wait, or while another sweep's
+ * claim on it holds (CLAIM_STALE_MINUTES).
+ */
+export function isDue(row, keyword, now) {
+ if (!row) return true;
+ if (row.tasks) return false;
+ if (row.claimed_at && now - Date.parse(row.claimed_at) < CLAIM_STALE_MINUTES * 60_000) {
+ return false;
+ }
+ if (row.keyword !== keyword.keyword) return true;
+ if (!row.posted_at) return true;
+ return now - Date.parse(row.posted_at) >= REFRESH_DAYS * DAY_MS;
+}
+
+/** Never read first, then the longest waiting; at most `limit`. */
+export function pickDue(candidates, limit = NIGHTLY_LIMIT) {
+ const at = (c) => (c.postedAt ? Date.parse(c.postedAt) || 0 : null);
+ return [...candidates]
+ .sort((a, b) => {
+ const x = at(a);
+ const y = at(b);
+ if (x === null || y === null) return (x === null ? 0 : 1) - (y === null ? 0 : 1);
+ return x - y;
+ })
+ .slice(0, limit);
+}
+
+/** Runs `fn` over `items`, `concurrency` at a time. */
+async function eachLimited(items, concurrency, fn) {
+ let next = 0;
+ const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+ while (next < items.length) {
+ const item = items[next];
+ next += 1;
+ await fn(item);
+ }
+ });
+ await Promise.all(workers);
+}
+
+/** Postgres' foreign key violation: here, a prompt deleted since it was read. */
+const FOREIGN_KEY_VIOLATION = '23503';
+
+/** The ids, of those given, whose prompts are still there. */
+async function existingPromptIds(ids) {
+ const found = new Set();
+ for (let i = 0; i < ids.length; i += ID_CHUNK) {
+ const { data, error } = await supabaseAdmin
+ .from('prompts')
+ .select('id')
+ .in('id', ids.slice(i, i + ID_CHUNK));
+ if (error) throw new Error(`prompts read failed: ${error.message}`);
+ for (const row of data ?? []) found.add(row.id);
+ }
+ return found;
+}
+
+/**
+ * Writes the rows, WRITE_CHUNK at a time. A prompt deleted since it was read
+ * fails its whole chunk (the foreign key): the chunk is written again
+ * without the prompts that are gone, so the others' reads (paid for) aren't
+ * lost with it.
+ */
+async function upsertRows(rows) {
+ const write = async (chunk) =>
+ (await supabaseAdmin.from('prompt_demand').upsert(chunk, { onConflict: 'prompt_id' })).error;
+ for (let i = 0; i < rows.length; i += WRITE_CHUNK) {
+ const chunk = rows.slice(i, i + WRITE_CHUNK);
+ let error = await write(chunk);
+ if (error?.code === FOREIGN_KEY_VIOLATION) {
+ const kept = await existingPromptIds(chunk.map((r) => r.prompt_id));
+ const left = chunk.filter((r) => kept.has(r.prompt_id));
+ logger.warn(
+ { promptIds: chunk.filter((r) => !kept.has(r.prompt_id)).map((r) => r.prompt_id) },
+ '[prompt-demand] rows of deleted prompts not written',
+ );
+ error = left.length ? await write(left) : null;
+ }
+ if (error) throw new Error(`prompt demand write failed: ${error.message}`);
+ }
+}
+
+/**
+ * Claims the prompts due before their tasks are set (prompt_demand_claim):
+ * only those claimed are set, so a second sweep running at once sets none
+ * of them again.
+ */
+async function claimDue(due) {
+ const { data, error } = await supabaseAdmin.rpc('prompt_demand_claim', {
+ p_rows: due.map((d) => ({ prompt_id: d.promptId, brand_id: d.brandId })),
+ p_stale_minutes: CLAIM_STALE_MINUTES,
+ });
+ if (error) throw new Error(`prompt demand claim failed: ${error.message}`);
+ const claimed = new Set(Array.isArray(data) ? data : []);
+ return due.filter((d) => claimed.has(d.promptId));
+}
+
+/**
+ * Sets the paid signals' tasks for the prompts due: one task per signal and
+ * prompt, tagged with the prompt's id. The prompts are claimed first, and
+ * only those claimed are set (claimDue). Each request's task ids are
+ * written as soon as it's back, so a failure later on loses none that were
+ * paid for. A request that fails is set again next week; a prompt none
+ * could be set for stays due once its claim lapses.
+ *
+ * @param {Array<{ promptId: string, brandId: string, keyword: string, locationCode: number|null, languageCode: string|null }>} due
+ * @returns {Promise<number>} prompts with at least one task set
+ */
+export async function postDemandTasks(due, { now = Date.now() } = {}) {
+ if (due.length === 0) return 0;
+ const claimed = await claimDue(due);
+ if (claimed.length === 0) return 0;
+ const at = new Date(now).toISOString();
+ const byPrompt = new Map(claimed.map((d) => [d.promptId, d]));
+ // What each prompt's row holds so far: a request's ids are added to it.
+ const rows = new Map();
+ for (const name of PAID_SIGNALS) {
+ const source = SOURCES[name];
+ const tasks = claimed.map((d) => source.task({ ...d, tag: d.promptId })).filter(Boolean);
+ if (tasks.length === 0) continue;
+ const { failed, errors } = await postTasks(source.path, tasks, {
+ onChunk: async (ids) => {
+ const written = [];
+ for (const [promptId, id] of ids) {
+ const d = byPrompt.get(promptId);
+ if (!d) continue;
+ const row = rows.get(promptId) ?? {
+ prompt_id: d.promptId,
+ brand_id: d.brandId,
+ keyword: d.keyword,
+ location_code: d.locationCode,
+ language_code: d.languageCode,
+ tasks: { posted_at: at, trends: null, paa: null, youtube: null },
+ posted_at: at,
+ };
+ const next = { ...row, tasks: { ...row.tasks, [name]: id } };
+ rows.set(promptId, next);
+ written.push(next);
+ }
+ await upsertRows(written);
+ },
+ });
+ if (failed > 0) {
+ logger.warn({ signal: name, failed, errors }, '[prompt-demand] tasks not set');
+ }
+ }
+ return rows.size;
+}
+
+const DEMAND_ROW_COLUMNS = 'prompt_id, brand_id, gsc, trends, paa, youtube, tasks, posted_at';
+
+/** The prompt_demand rows with tasks waiting, paged. */
+async function waitingRows() {
+ const rows = [];
+ for (let from = 0; ; from += 1000) {
+ const { data, error } = await supabaseAdmin
+ .from('prompt_demand')
+ .select(DEMAND_ROW_COLUMNS)
+ .not('tasks', 'is', null)
+ .order('prompt_id', { ascending: true })
+ .range(from, from + 999);
+ if (error) throw new Error(`prompt demand read failed: ${error.message}`);
+ rows.push(...(data ?? []));
+ if ((data ?? []).length < 1000) return rows;
+ }
+}
+
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+/**
+ * Done tasks no row knows: set, and their ids never written (a request that
+ * timed out after DataForSEO took it, a write that failed). Their tag is
+ * their prompt's id, so each goes back to its prompt's row, where that
+ * signal has no task waiting. Rows already waiting are given theirs in
+ * place; the rest are read and returned.
+ */
+async function orphanRows(ready, waiting) {
+ const known = new Set();
+ for (const row of waiting) {
+ for (const name of PAID_SIGNALS) if (row.tasks?.[name]) known.add(row.tasks[name]);
+ }
+ const byPrompt = new Map(waiting.map((r) => [r.prompt_id, r]));
+ const found = new Map();
+ for (const name of PAID_SIGNALS) {
+ for (const [id, tag] of ready[name]) {
+ if (known.has(id) || !tag || !UUID.test(tag)) continue;
+ const row = byPrompt.get(tag);
+ if (row) {
+ if (!row.tasks[name]) row.tasks = { ...row.tasks, [name]: id };
+ continue;
+ }
+ found.set(tag, { ...found.get(tag), [name]: id });
+ }
+ }
+ if (found.size === 0) return [];
+ const ids = [...found.keys()];
+ const rows = [];
+ for (let i = 0; i < ids.length; i += ID_CHUNK) {
+ const { data, error } = await supabaseAdmin
+ .from('prompt_demand')
+ .select(DEMAND_ROW_COLUMNS)
+ .in('prompt_id', ids.slice(i, i + ID_CHUNK));
+ if (error) throw new Error(`prompt demand read failed: ${error.message}`);
+ for (const row of data ?? []) {
+ rows.push({
+ ...row,
+ tasks: { posted_at: row.posted_at ?? null, ...row.tasks, ...found.get(row.prompt_id) },
+ });
+ }
+ }
+ return rows;
+}
+
+/**
+ * Reads the tasks that are done: each endpoint's ready list first, then
+ * each of ours on it, and the done ones no row knew (orphanRows). A task
+ * older than TASK_EXPIRE_DAYS is read whether listed or not, and given up
+ * if it still has nothing; the prompt is set again when it's next due. The
+ * level is worked out again for each prompt read. Workspaces whose plan
+ * has ended are left out.
+ *
+ * @returns {Promise<{ read: number, waiting: number, skipped?: 'off' }>}
+ */
+export async function collectDemandTasks({ now = Date.now() } = {}) {
+ if (!paidSignalsEnabled()) return { read: 0, waiting: 0, skipped: 'off' };
+ const ready = {};
+ for (const name of PAID_SIGNALS) {
+ try {
+ ready[name] = await readyTasks(SOURCES[name].path);
+ } catch (err) {
+ ready[name] = new Map();
+ logger.warn(
+ { err: { message: err?.message }, signal: name },
+ '[prompt-demand] ready list not read',
+ );
+ }
+ }
+ const waitingNow = await waitingRows();
+ const rows = [...waitingNow, ...(await orphanRows(ready, waitingNow))];
+ if (rows.length === 0) return { read: 0, waiting: 0 };
+ const isActive = await activeBrandFilter(rows.map((r) => r.brand_id));
+ const mine = rows.filter((r) => isActive(r.brand_id));
+ if (mine.length === 0) return { read: 0, waiting: 0 };
+
+ const at = new Date(now).toISOString();
+ const writes = [];
+ let waiting = 0;
+ await eachLimited(mine, READ_CONCURRENCY, async (row) => {
+ const tasks = { ...row.tasks };
+ const next = {
+ prompt_id: row.prompt_id,
+ brand_id: row.brand_id,
+ gsc: row.gsc,
+ trends: row.trends,
+ paa: row.paa,
+ youtube: row.youtube,
+ };
+ const expired = now - (Date.parse(tasks.posted_at ?? '') || 0) >= TASK_EXPIRE_DAYS * DAY_MS;
+ let changed = false;
+ for (const name of PAID_SIGNALS) {
+ const id = tasks[name];
+ if (!id) continue;
+ if (!ready[name].has(id) && !expired) continue;
+ try {
+ const got = await getTask(SOURCES[name].getPath(id));
+ if (got.status === 'pending' && !expired) continue;
+ if (got.status === 'done') next[name] = SOURCES[name].read(got.result, at);
+ } catch (err) {
+ logger.warn(
+ { err: { message: err?.message }, signal: name, promptId: row.prompt_id },
+ '[prompt-demand] task not read',
+ );
+ if (!expired) continue;
+ }
+ tasks[name] = null;
+ changed = true;
+ }
+ const left = PAID_SIGNALS.some((name) => tasks[name]);
+ if (left) waiting += 1;
+ if (!changed) return;
+ const { level, signals } = demandLevel(next);
+ writes.push({
+ ...next,
+ tasks: left ? tasks : null,
+ level,
+ signals,
+ updated_at: at,
+ });
+ });
+ await upsertRows(writes);
+ return { read: writes.length, waiting };
+}
+
+// ─── The nightly sweep ──────────────────────────────────────────────────────
+
+/** A select paged past PostgREST's 1,000 rows. */
+async function selectAll(build) {
+ const rows = [];
+ for (let from = 0; ; from += 1000) {
+ const { data, error } = await build().range(from, from + 999);
+ if (error) throw new Error(error.message);
+ rows.push(...(data ?? []));
+ if ((data ?? []).length < 1000) return rows;
+ }
+}
+
+async function activePrompts(brandId) {
+ const rows = await selectAll(() =>
+ supabaseAdmin
+ .from('prompts')
+ .select('id, text, prompt_sets!inner(brand_id)')
+ .eq('prompt_sets.brand_id', brandId)
+ .eq('is_active', true)
+ .order('id', { ascending: true }),
+ );
+ return rows.map((r) => ({ id: r.id, text: r.text }));
+}
+
+async function brandDemandRows(brandId) {
+ return selectAll(() =>
+ supabaseAdmin
+ .from('prompt_demand')
+ .select('*')
+ .eq('brand_id', brandId)
+ .order('prompt_id', { ascending: true }),
+ );
+}
+
+/** Each prompt's newest volume analysis. */
+async function promptVolumes(promptIds) {
+ const volumes = new Map();
+ for (let i = 0; i < promptIds.length; i += ID_CHUNK) {
+ const { data, error } = await supabaseAdmin
+ .from('prompt_volumes')
+ .select('prompt_id, google_volumes, location_code, language_code, fetched_at')
+ .in('prompt_id', promptIds.slice(i, i + ID_CHUNK))
+ .order('fetched_at', { ascending: false });
+ if (error) throw new Error(`prompt volumes read failed: ${error.message}`);
+ for (const row of data ?? []) if (!volumes.has(row.prompt_id)) volumes.set(row.prompt_id, row);
+ }
+ return volumes;
+}
+
+/**
+ * One brand's night: each active prompt's Search Console demand and level
+ * written where they changed, and the prompts whose paid signals are due.
+ * Rows are only made for a prompt with something read.
+ */
+export async function refreshBrandDemand(brand, { now = Date.now(), paid = false } = {}) {
+ const prompts = await activePrompts(brand.id);
+ if (prompts.length === 0) return { written: 0, due: [] };
+ const at = new Date(now).toISOString();
+ const [existing, gsc, volumes] = await Promise.all([
+ brandDemandRows(brand.id).then((rows) => new Map(rows.map((r) => [r.prompt_id, r]))),
+ brand.gsc_property
+ ? readGscQueries(brand.id, now).then((queries) => gscDemand(queries, prompts, at))
+ : null,
+ paid ? promptVolumes(prompts.map((p) => p.id)) : new Map(),
+ ]);
+
+ const writes = [];
+ const due = [];
+ for (const prompt of prompts) {
+ const row = existing.get(prompt.id);
+ const signals = {
+ gsc: gsc?.get(prompt.id) ?? null,
+ trends: row?.trends ?? null,
+ paa: row?.paa ?? null,
+ youtube: row?.youtube ?? null,
+ };
+ const level = demandLevel(signals);
+ // Search Console is read again every night, and says so (its `at`).
+ const changed =
+ signals.gsc !== null ||
+ (row
+ ? row.gsc !== null || row.level !== level.level || row.signals !== level.signals
+ : level.signals > 0);
+ if (changed) {
+ writes.push({
+ prompt_id: prompt.id,
+ brand_id: brand.id,
+ gsc: signals.gsc,
+ level: level.level,
+ signals: level.signals,
+ updated_at: at,
+ });
+ }
+ if (paid) {
+ const keyword = topKeyword(volumes.get(prompt.id));
+ if (keyword && isDue(row, keyword, now)) {
+ due.push({
+ promptId: prompt.id,
+ brandId: brand.id,
+ ...keyword,
+ postedAt: row?.posted_at ?? null,
+ });
+ }
+ }
+ }
+ await upsertRows(writes);
+ return { written: writes.length, due };
+}
+
+/**
+ * When each brand's rows were last written (prompt_demand_brand_updates),
+ * as times. A brand with none isn't in it. When it can't be read, every
+ * brand counts as never written: the night goes in the brands' id order.
+ */
+async function brandUpdates() {
+ const { data, error } = await supabaseAdmin.rpc('prompt_demand_brand_updates');
+ if (error) {
+ logger.warn({ err: { message: error.message } }, '[prompt-demand] brand order not read');
+ return new Map();
+ }
+ return new Map(Object.entries(data ?? {}).map(([brandId, at]) => [brandId, Date.parse(at) || 0]));
+}
+
+/** The brands never written first, then the longest since their last write. */
+export function sweepOrder(brands, updates) {
+ // A time is 0 or more: never written (-1) goes first.
+ const at = (brand) => updates.get(brand.id) ?? -1;
+ return [...brands].sort((a, b) => at(a) - at(b) || String(a.id).localeCompare(String(b.id)));
+}
+
+/**
+ * The night's demand (server.js daily run): last night's tasks read, every
+ * active brand's prompts matched to its Search Console queries and their
+ * levels kept, then the paid signals set for the prompts due, the
+ * NIGHTLY_LIMIT waiting longest. Brands whose workspace's plan has ended
+ * are skipped; one brand's failure is logged and the rest go on.
+ *
+ * The brands go in sweepOrder, for SWEEP_DEADLINE_MS at most: the ones not
+ * reached are `unfinished`, logged, and go first the next night. `due`:
+ * the prompts whose paid signals were due, before the cap.
+ *
+ * @returns {Promise<{ brands: number, written: number, skipped: number, failed: number, unfinished: number, due: number, posted: number, read: number, paid: boolean }>}
+ */
+export async function sweepPromptDemand({
+ now = Date.now(),
+ limit = NIGHTLY_LIMIT,
+ deadlineMs = SWEEP_DEADLINE_MS,
+} = {}) {
+ const paid = paidSignalsEnabled();
+ const outcome = {
+ brands: 0,
+ written: 0,
+ skipped: 0,
+ failed: 0,
+ unfinished: 0,
+ due: 0,
+ posted: 0,
+ read: 0,
+ paid,
+ };
+
+ if (paid) {
+ try {
+ outcome.read = (await collectDemandTasks({ now })).read;
+ } catch (err) {
+ logger.error({ err: { message: err?.message } }, '[prompt-demand] tasks not collected');
+ }
+ }
+
+ const { data: brands, error } = await supabaseAdmin
+ .from('brands')
+ .select('id, organization_id, gsc_property')
+ .eq('is_active', true);
+ if (error) throw new Error(`brands read failed: ${error.message}`);
+ const isActive = await activeOrgFilter((brands ?? []).map((b) => b.organization_id));
+ const active = (brands ?? []).filter((b) => isActive(b.organization_id));
+ outcome.skipped = (brands ?? []).length - active.length;
+ const ordered = sweepOrder(active, await brandUpdates());
+
+ const due = [];
+ const started = Date.now();
+ for (const [i, brand] of ordered.entries()) {
+ if (Date.now() - started > deadlineMs) {
+ outcome.unfinished = ordered.length - i;
+ break;
+ }
+ try {
+ const result = await refreshBrandDemand(brand, { now, paid });
+ outcome.brands += 1;
+ outcome.written += result.written;
+ due.push(...result.due);
+ } catch (err) {
+ outcome.failed += 1;
+ logger.warn(
+ { err: { message: err?.message }, brandId: brand.id },
+ '[prompt-demand] a brand was not read',
+ );
+ }
+ }
+
+ if (outcome.unfinished > 0) {
+ logger.warn(
+ { unfinished: outcome.unfinished, brands: outcome.brands, deadlineMs },
+ '[prompt-demand] sweep ran out of time: brands left for the next night',
+ );
+ }
+ outcome.due = due.length;
+ if (paid && due.length) outcome.posted = await postDemandTasks(pickDue(due, limit), { now });
+ logger.info(outcome, '[prompt-demand] nightly sweep done');
+ return outcome;
+}
+
+/**
+ * Reads tonight's tasks once they've had time to finish (COLLECT_DELAY_MS
+ * after the sweep), so most prompts have their signals the same night.
+ * What isn't done by then is read the next night.
+ */
+export function scheduleDemandCollect(delayMs = COLLECT_DELAY_MS) {
+ if (!paidSignalsEnabled()) return null;
+ const timer = setTimeout(() => {
+ collectDemandTasks().catch((err) =>
+ logger.error({ err: { message: err?.message } }, '[prompt-demand] late collect failed'),
+ );
+ }, delayMs);
+ timer.unref?.();
+ return timer;
+}
+
+// ─── What the card reads ────────────────────────────────────────────────────
+
+/**
+ * What GET /api/prompts/:id/demand answers: the signals read, the level,
+ * which tasks are still waiting, and what the card needs to say about a
+ * missing signal: whether the prompt has a volume analysis and a keyword to
+ * read the paid signals for, and whether they're read at all on this server.
+ */
+export function demandResponse(row, volume, { paid = paidSignalsEnabled() } = {}) {
+ const keyword = topKeyword(volume);
+ return {
+ level: row?.level ?? null,
+ signals: row?.signals ?? 0,
+ updatedAt: row?.updated_at ?? null,
+ keyword: row?.keyword ?? keyword?.keyword ?? null,
+ locationCode: row?.keyword ? (row.location_code ?? null) : (keyword?.locationCode ?? null),
+ languageCode: row?.keyword ? (row.language_code ?? null) : (keyword?.languageCode ?? null),
+ gsc: row?.gsc ?? null,
+ trends: row?.trends ?? null,
+ paa: row?.paa ?? null,
+ youtube: row?.youtube ?? null,
+ pending: Object.fromEntries(PAID_SIGNALS.map((name) => [name, Boolean(row?.tasks?.[name])])),
+ hasVolume: Boolean(volume),
+ hasKeyword: Boolean(keyword),
+ paidSignals: paid,
+ };
+}
diff --git a/server/src/lib/prompt-demand.test.js b/server/src/lib/prompt-demand.test.js
new file mode 100644
index 000000000..fe745b720
--- /dev/null
+++ b/server/src/lib/prompt-demand.test.js
@@ -0,0 +1,1289 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+/**
+ * A prompt's real demand (PROMPT-014): Search Console every night, Google
+ * Trends, People Also Ask and YouTube from DataForSEO's queued tasks once a
+ * week (DEMAND_SIGNALS), and the level they come to.
+ */
+
+// ─── A database in memory ───────────────────────────────────────────────────
+
+let tables;
+let writes;
+let rpcCalls;
+/** The database's now(), for the claims. */
+let dbNow;
+/** Runs on each Search Console read (a brand being matched). */
+let onGscRead;
+
+function valueAt(row, path) {
+ return path.split('.').reduce((v, key) => (v == null ? v : v[key]), row);
+}
+
+function query(table) {
+ const filters = [];
+ let range = null;
+ let op = 'select';
+ let payload = null;
+ const rows = () => {
+ let out = (tables[table] ?? []).filter((row) =>
+ filters.every(([kind, col, value]) => {
+ const v = valueAt(row, col);
+ if (kind === 'eq') return v === value;
+ if (kind === 'in') return value.includes(v);
+ if (kind === 'notNull') return v !== null && v !== undefined;
+ return true;
+ }),
+ );
+ if (range) out = out.slice(range[0], range[1] + 1);
+ return out;
+ };
+ const chain = {
+ select: () => chain,
+ eq: (col, value) => (filters.push(['eq', col, value]), chain),
+ in: (col, value) => (filters.push(['in', col, value]), chain),
+ not: (col, operator, value) => {
+ if (operator === 'is' && value === null) filters.push(['notNull', col]);
+ return chain;
+ },
+ order: () => chain,
+ limit: () => chain,
+ range: (from, to) => ((range = [from, to]), chain),
+ upsert: (data) => ((op = 'upsert'), (payload = data), chain),
+ maybeSingle: async () => ({ data: rows()[0] ?? null, error: null }),
+ then: (resolve, reject) => {
+ if (op === 'upsert') {
+ // The foreign key: a row for a prompt that is gone fails the whole write.
+ const promptIds = new Set((tables.prompts ?? []).map((p) => p.id));
+ if (table === 'prompt_demand' && payload.some((row) => !promptIds.has(row.prompt_id))) {
+ return Promise.resolve({
+ error: { code: '23503', message: 'violates foreign key constraint' },
+ }).then(resolve, reject);
+ }
+ writes.push({ table, rows: payload });
+ const kept = tables[table] ?? (tables[table] = []);
+ for (const row of payload) {
+ const at = kept.findIndex((r) => r.prompt_id === row.prompt_id);
+ if (at === -1) kept.push({ ...row });
+ else kept[at] = { ...kept[at], ...row };
+ }
+ return Promise.resolve({ error: null }).then(resolve, reject);
+ }
+ return Promise.resolve({ data: rows(), error: null }).then(resolve, reject);
+ },
+ };
+ return chain;
+}
+
+/** prompt_demand_claim: no task waiting and no claim younger than p_stale_minutes. */
+function claim({ p_rows, p_stale_minutes }) {
+ const at = new Date(dbNow).toISOString();
+ const claimed = [];
+ for (const { prompt_id, brand_id } of p_rows) {
+ if (!tables.prompts.some((p) => p.id === prompt_id)) continue;
+ const row = tables.prompt_demand.find((r) => r.prompt_id === prompt_id);
+ if (!row) {
+ tables.prompt_demand.push({
+ prompt_id,
+ brand_id,
+ claimed_at: at,
+ tasks: null,
+ gsc: null,
+ level: null,
+ signals: 0,
+ updated_at: at,
+ });
+ } else {
+ if (row.tasks) continue;
+ if (row.claimed_at && Date.parse(row.claimed_at) >= dbNow - p_stale_minutes * 60_000)
+ continue;
+ row.claimed_at = at;
+ }
+ claimed.push(prompt_id);
+ }
+ return claimed;
+}
+
+/** prompt_demand_brand_updates: each brand's last write. */
+function brandUpdates() {
+ const out = {};
+ for (const row of tables.prompt_demand) {
+ if (row.updated_at && !(out[row.brand_id] >= row.updated_at))
+ out[row.brand_id] = row.updated_at;
+ }
+ return out;
+}
+
+vi.mock('../config/supabase.js', () => ({
+ default: {
+ from: (table) => query(table),
+ rpc: async (name, args) => {
+ rpcCalls.push({ name, args });
+ if (name === 'prompt_demand_claim') return { data: claim(args), error: null };
+ if (name === 'prompt_demand_brand_updates') return { data: brandUpdates(), error: null };
+ onGscRead?.(args.p_brand_id);
+ return { data: tables.gsc?.[args.p_brand_id] ?? [], error: null };
+ },
+ },
+}));
+
+const gscReads = () =>
+ rpcCalls.filter((c) => c.name === 'prompt_demand_gsc_queries').map((c) => c.args.p_brand_id);
+
+// The plan check (#1608): the organizations and brands whose plan ended.
+let endedOrgs;
+let endedBrands;
+vi.mock('./active-subscriptions.js', () => ({
+ activeOrgFilter: async () => (orgId) => !endedOrgs.has(orgId),
+ activeBrandFilter: async () => (brandId) => !endedBrands.has(brandId),
+}));
+
+const postTasks = vi.fn();
+const readyTasks = vi.fn();
+const getTask = vi.fn();
+vi.mock('./dataforseo.js', () => ({
+ isDataForSeoConfigured: () => true,
+ postTasks,
+ readyTasks,
+ getTask,
+}));
+
+/** postTasks setting every task, each id `${path}#${tag}`, in one request. */
+async function setAll(path, tasks, { onChunk } = {}) {
+ const ids = new Map(tasks.map((t) => [t.tag, `${path}#${t.tag}`]));
+ if (onChunk) await onChunk(ids);
+ return { ids, failed: 0, errors: [] };
+}
+
+const { logger } = await import('./logger.js');
+
+const {
+ CLAIM_STALE_MINUTES,
+ LEVEL_RULE,
+ NIGHTLY_LIMIT,
+ REFRESH_DAYS,
+ SOURCES,
+ TASK_EXPIRE_DAYS,
+ collectDemandTasks,
+ demandLevel,
+ demandResponse,
+ gscDemand,
+ isDue,
+ paaSignal,
+ pickDue,
+ postDemandTasks,
+ sweepOrder,
+ sweepPromptDemand,
+ topKeyword,
+ trendsKeyword,
+ trendsSignal,
+ youtubeSignal,
+} = await import('./prompt-demand.js');
+
+const NOW = Date.parse('2026-10-28T06:00:00Z');
+const AT = new Date(NOW).toISOString();
+const DAY = 86_400_000;
+const daysAgo = (n) => new Date(NOW - n * DAY).toISOString();
+
+// ─── Realistic DataForSEO results ───────────────────────────────────────────
+
+/** Google Trends Explore, one keyword: 52 weeks, the last ones higher. */
+function trendsResult(values) {
+ return {
+ keywords: ['ai search monitoring'],
+ type: 'web',
+ location_code: 2840,
+ language_code: 'en',
+ check_url: 'https://trends.google.com/trends/explore?q=ai%20search%20monitoring',
+ datetime: '2026-10-27 22:10:01 +00:00',
+ items_count: 1,
+ items: [
+ {
+ position: 1,
+ type: 'google_trends_graph',
+ title: 'Interest over time',
+ keywords: ['ai search monitoring'],
+ data: values.map((value, i) => ({
+ date_from: new Date(Date.UTC(2025, 9, 26) + i * 7 * DAY).toISOString().slice(0, 10),
+ date_to: new Date(Date.UTC(2025, 9, 26) + (i * 7 + 6) * DAY).toISOString().slice(0, 10),
+ timestamp: (Date.UTC(2025, 9, 26) + i * 7 * DAY) / 1000,
+ missing_data: false,
+ values: [value],
+ })),
+ averages: [40],
+ },
+ ],
+ };
+}
+
+/** Google organic (advanced) with a People Also Ask block among the results. */
+const SERP_RESULT = {
+ keyword: 'ai search monitoring',
+ type: 'organic',
+ se_domain: 'google.com',
+ location_code: 2840,
+ language_code: 'en',
+ check_url: 'https://www.google.com/search?q=ai+search+monitoring',
+ datetime: '2026-10-27 22:12:40 +00:00',
+ item_types: ['organic', 'people_also_ask', 'related_searches'],
+ se_results_count: 245000000,
+ items_count: 12,
+ items: [
+ {
+ type: 'organic',
+ rank_group: 1,
+ rank_absolute: 1,
+ domain: 'example.com',
+ title: 'AI Search Monitoring',
+ url: 'https://example.com/',
+ },
+ {
+ type: 'people_also_ask',
+ rank_group: 1,
+ rank_absolute: 3,
+ position: 'left',
+ xpath: '/html[1]/body[1]/div[3]',
+ items: [
+ {
+ type: 'people_also_ask_element',
+ title: 'How do I track my brand in ChatGPT?',
+ seed_question: null,
+ xpath: '/html[1]/body[1]/div[3]/div[1]',
+ expanded_element: [
+ {
+ type: 'people_also_ask_expanded_element',
+ featured_title: null,
+ url: 'https://www.example.org/track-brand-chatgpt',
+ domain: 'www.example.org',
+ title: 'Track your brand in ChatGPT',
+ description: 'To track your brand…',
+ images: null,
+ timestamp: null,
+ table: null,
+ },
+ ],
+ },
+ {
+ type: 'people_also_ask_element',
+ title: 'What is AI search monitoring?',
+ seed_question: null,
+ xpath: '/html[1]/body[1]/div[3]/div[2]',
+ expanded_element: [
+ {
+ type: 'people_also_ask_ai_overview_expanded_element',
+ items: [{ type: 'ai_overview_element', text: '…' }],
+ },
+ ],
+ },
+ {
+ type: 'people_also_ask_element',
+ title: 'how do i track my brand in chatgpt?',
+ seed_question: null,
+ xpath: '/html[1]/body[1]/div[3]/div[3]',
+ expanded_element: null,
+ },
+ {
+ type: 'people_also_ask_element',
+ title: 'Is there a tool to monitor AI answers?',
+ seed_question: null,
+ xpath: '/html[1]/body[1]/div[3]/div[4]',
+ expanded_element: [],
+ },
+ ],
+ },
+ { type: 'related_searches', rank_group: 1, rank_absolute: 11, items: ['ai monitoring tools'] },
+ ],
+};
+
+/** YouTube organic (advanced): videos and a channel. */
+const YOUTUBE_RESULT = {
+ keyword: 'ai search monitoring',
+ type: 'organic',
+ se_domain: 'youtube.com',
+ location_code: 2840,
+ language_code: 'en',
+ check_url: 'https://www.youtube.com/results?search_query=ai+search+monitoring',
+ datetime: '2026-10-27 22:15:02 +00:00',
+ se_results_count: 214,
+ items_count: 3,
+ items: [
+ {
+ type: 'youtube_video',
+ rank_group: 1,
+ rank_absolute: 1,
+ block_rank: 1,
+ block_name: null,
+ title: 'How to track your brand in ChatGPT',
+ url: 'https://www.youtube.com/watch?v=abc123',
+ thumbnail_url: 'https://i.ytimg.com/vi/abc123/hqdefault.jpg',
+ channel_id: 'UC1',
+ channel_name: 'Search Lab',
+ channel_url: 'https://www.youtube.com/@searchlab',
+ channel_logo: null,
+ description: 'A walkthrough',
+ highlighted: ['track'],
+ badges: ['New'],
+ is_live: false,
+ is_movie: null,
+ views_count: 310000,
+ publication_date: '2 months ago',
+ timestamp: '2026-08-20 10:00:00 +00:00',
+ video_id: 'abc123',
+ duration_time: '12:04',
+ duration_time_seconds: 724,
+ is_shorts: false,
+ },
+ {
+ type: 'youtube_channel',
+ rank_group: 1,
+ rank_absolute: 2,
+ block_rank: 1,
+ block_name: null,
+ channel_id: 'UC2',
+ name: 'AI Monitor',
+ video_count: 0,
+ is_verified: true,
+ },
+ {
+ type: 'youtube_video',
+ rank_group: 2,
+ rank_absolute: 3,
+ block_rank: 2,
+ block_name: null,
+ title: 'AI visibility tools compared',
+ url: 'https://www.youtube.com/watch?v=def456',
+ channel_name: 'Growth Notes',
+ views_count: 1490000,
+ publication_date: '5 months ago',
+ timestamp: '2026-05-11 08:00:00 +00:00',
+ video_id: 'def456',
+ is_shorts: false,
+ },
+ ],
+};
+
+// ─── Fixtures ───────────────────────────────────────────────────────────────
+
+const prompt = (id, brandId, text, isActive = true) => ({
+ id,
+ text,
+ is_active: isActive,
+ prompt_sets: { brand_id: brandId },
+});
+
+const volume = (promptId, googleVolumes, extra = {}) => ({
+ prompt_id: promptId,
+ google_volumes: googleVolumes,
+ location_code: 2840,
+ language_code: 'en',
+ fetched_at: daysAgo(20),
+ ...extra,
+});
+
+beforeEach(() => {
+ tables = {
+ brands: [
+ { id: 'b1', organization_id: 'o1', gsc_property: 'sc-domain:ansvisor.com', is_active: true },
+ ],
+ prompts: [
+ prompt('p1', 'b1', 'What is the best AI search monitoring tool?'),
+ prompt('p2', 'b1', 'How do I track my brand in ChatGPT answers?'),
+ prompt('p3', 'b1', 'Paused prompt about pricing', false),
+ ],
+ prompt_demand: [],
+ prompt_volumes: [],
+ gsc: {
+ b1: [
+ { query: 'ai search monitoring tool', clicks: 60, impressions: 2400 },
+ { query: 'best ai search monitoring', clicks: 36, impressions: 1720 },
+ { query: 'track brand chatgpt', clicks: 4, impressions: 90 },
+ { query: 'ansvisor login', clicks: 300, impressions: 900 },
+ ],
+ },
+ };
+ writes = [];
+ rpcCalls = [];
+ dbNow = NOW;
+ onGscRead = null;
+ endedOrgs = new Set();
+ endedBrands = new Set();
+ postTasks.mockReset();
+ readyTasks.mockReset().mockResolvedValue(new Map());
+ getTask.mockReset();
+ vi.stubEnv('DEMAND_SIGNALS', 'false');
+});
+
+afterEach(() => {
+ vi.unstubAllEnvs();
+ vi.restoreAllMocks();
+});
+
+const demandRow = (promptId) => tables.prompt_demand.find((r) => r.prompt_id === promptId);
+
+// ─── Search Console ─────────────────────────────────────────────────────────
+
+describe('gscDemand', () => {
+ const prompts = [
+ { id: 'p1', text: 'What is the best AI search monitoring tool?' },
+ { id: 'p2', text: 'How do I track my brand in ChatGPT answers?' },
+ ];
+
+ it("sums each prompt's matching queries over the window, the most seen as its example", () => {
+ const demand = gscDemand(tables.gsc.b1, prompts, AT);
+ expect(demand.get('p1')).toEqual({
+ impressions: 4120,
+ clicks: 96,
+ queries: 2,
+ top: { query: 'ai search monitoring tool', impressions: 2400, clicks: 60 },
+ at: AT,
+ });
+ expect(demand.get('p2')).toEqual({
+ impressions: 90,
+ clicks: 4,
+ queries: 1,
+ top: { query: 'track brand chatgpt', impressions: 90, clicks: 4 },
+ at: AT,
+ });
+ });
+
+ it('gives a query to the closest prompt only, and a prompt nothing matches zeros', () => {
+ const demand = gscDemand(
+ [{ query: 'monitoring tool', clicks: 1, impressions: 10 }],
+ [
+ { id: 'a', text: 'best monitoring tool' },
+ { id: 'b', text: 'monitoring dashboards for teams' },
+ ],
+ AT,
+ );
+ expect(demand.get('a').queries).toBe(1);
+ expect(demand.get('b')).toEqual({ impressions: 0, clicks: 0, queries: 0, top: null, at: AT });
+ });
+
+ it('reads nothing (null) when the brand has no query in the window', () => {
+ expect(gscDemand([], prompts, AT)).toBeNull();
+ });
+});
+
+// ─── The level ──────────────────────────────────────────────────────────────
+
+describe('demandLevel', () => {
+ const strong = {
+ gsc: { impressions: 4120 },
+ trends: { trend: 'rising' },
+ paa: { questions: [1, 2, 3, 4].map((n) => ({ question: `q${n}` })) },
+ youtube: { views: 1_800_000 },
+ };
+
+ it('is high with every signal strong, and counts the signals read', () => {
+ expect(demandLevel(strong)).toEqual({ level: 'high', signals: 4 });
+ });
+
+ it('has no level with nothing read', () => {
+ expect(demandLevel({ gsc: null, trends: null, paa: null, youtube: null })).toEqual({
+ level: null,
+ signals: 0,
+ });
+ });
+
+ it('is at most medium on one signal, however strong', () => {
+ expect(demandLevel({ gsc: { impressions: 50_000 } })).toEqual({ level: 'medium', signals: 1 });
+ });
+
+ it('counts a read that found nothing as read, and as low', () => {
+ expect(
+ demandLevel({
+ gsc: { impressions: 20 },
+ trends: { trend: null },
+ paa: { questions: [] },
+ youtube: { views: 300 },
+ }),
+ ).toEqual({ level: 'low', signals: 4 });
+ });
+
+ it('follows the rule at its edges', () => {
+ // 3 of 6 points (0.5): medium.
+ expect(
+ demandLevel({
+ gsc: { impressions: LEVEL_RULE.gscImpressions.strong },
+ trends: { trend: 'steady' },
+ paa: { questions: [] },
+ }).level,
+ ).toBe('medium');
+ // 6 of 8 points (0.75): high.
+ expect(
+ demandLevel({
+ gsc: { impressions: LEVEL_RULE.gscImpressions.strong },
+ trends: { trend: 'steady' },
+ paa: { questions: [{}, {}, {}, {}] },
+ youtube: { views: LEVEL_RULE.youtubeViews.some },
+ }).level,
+ ).toBe('high');
+ // 1 of 4 points (0.25): low.
+ expect(
+ demandLevel({
+ gsc: { impressions: LEVEL_RULE.gscImpressions.some },
+ trends: { trend: 'falling' },
+ }).level,
+ ).toBe('low');
+ });
+});
+
+// ─── The keyword ────────────────────────────────────────────────────────────
+
+describe('topKeyword', () => {
+ it("takes the analysis's keyword with the most searches, with its location and language", () => {
+ expect(
+ topKeyword(volume('p1', { 'ai monitoring': 320, 'ai search monitoring': 1900, geo: 0 })),
+ ).toEqual({ keyword: 'ai search monitoring', locationCode: 2840, languageCode: 'en' });
+ });
+
+ it('has none without an analysis or a keyword Google has searches for', () => {
+ expect(topKeyword(null)).toBeNull();
+ expect(topKeyword(volume('p1', { 'ai monitoring': 0 }))).toBeNull();
+ });
+
+ it("reads Trends' refused characters as spaces", () => {
+ expect(trendsKeyword('ai-search "monitoring": tools')).toBe('ai search monitoring tools');
+ expect(trendsKeyword('x'.repeat(101))).toBeNull();
+ });
+});
+
+// ─── Reading the results ────────────────────────────────────────────────────
+
+describe('trendsSignal', () => {
+ it('reads a rising year and its change on a year ago', () => {
+ const values = Array.from({ length: 52 }, (_, i) => (i < 4 ? 25 : i >= 48 ? 41 : 30));
+ const signal = trendsSignal(trendsResult(values), AT);
+ expect(signal).toMatchObject({ trend: 'rising', change: 64, from: '2025-10-26', at: AT });
+ expect(signal.points).toHaveLength(52);
+ });
+
+ it('reads falling and steady years', () => {
+ const falling = Array.from({ length: 52 }, (_, i) => (i < 4 ? 80 : 40));
+ expect(trendsSignal(trendsResult(falling), AT)).toMatchObject({
+ trend: 'falling',
+ change: -50,
+ });
+ const steady = Array.from({ length: 52 }, () => 50);
+ expect(trendsSignal(trendsResult(steady), AT)).toMatchObject({ trend: 'steady', change: 0 });
+ });
+
+ it('says too little data when Google has no interest, and skips missing weeks', () => {
+ const result = trendsResult([0, 0, 0]);
+ result.items[0].data[1].missing_data = true;
+ expect(trendsSignal(result, AT)).toMatchObject({ points: [0, 0], trend: null, change: null });
+ expect(trendsSignal(null, AT)).toMatchObject({ points: [], trend: null });
+ });
+});
+
+describe('paaSignal', () => {
+ it('reads each question once, with the page Google answers it with', () => {
+ expect(paaSignal(SERP_RESULT, AT)).toEqual({
+ questions: [
+ {
+ question: 'How do I track my brand in ChatGPT?',
+ url: 'https://www.example.org/track-brand-chatgpt',
+ domain: 'www.example.org',
+ },
+ { question: 'What is AI search monitoring?', url: null, domain: null },
+ { question: 'Is there a tool to monitor AI answers?', url: null, domain: null },
+ ],
+ at: AT,
+ });
+ });
+
+ it('reads no question from a page without the block, or no result', () => {
+ expect(paaSignal({ items: [SERP_RESULT.items[0]] }, AT)).toEqual({ questions: [], at: AT });
+ expect(paaSignal(null, AT)).toEqual({ questions: [], at: AT });
+ });
+});
+
+describe('youtubeSignal', () => {
+ it("sums the videos' views, keeps the most viewed and YouTube's result count", () => {
+ expect(youtubeSignal(YOUTUBE_RESULT, AT)).toEqual({
+ videos: 2,
+ views: 1_800_000,
+ results: 214,
+ top: {
+ title: 'AI visibility tools compared',
+ url: 'https://www.youtube.com/watch?v=def456',
+ channel: 'Growth Notes',
+ views: 1_490_000,
+ published: '2026-05-11 08:00:00 +00:00',
+ },
+ at: AT,
+ });
+ });
+});
+
+describe('the tasks asked', () => {
+ const asked = {
+ keyword: 'ai search monitoring',
+ locationCode: null,
+ languageCode: null,
+ tag: 'p1',
+ };
+
+ it('asks Trends for 12 months worldwide without a location, and SERP for the US', () => {
+ expect(SOURCES.trends.task(asked)).toEqual({
+ keywords: ['ai search monitoring'],
+ language_code: 'en',
+ type: 'web',
+ time_range: 'past_12_months',
+ item_types: ['google_trends_graph'],
+ tag: 'p1',
+ });
+ expect(SOURCES.paa.task(asked)).toMatchObject({ location_code: 2840, language_code: 'en' });
+ expect(SOURCES.youtube.task({ ...asked, locationCode: 2792, languageCode: 'tr' })).toEqual({
+ keyword: 'ai search monitoring',
+ location_code: 2792,
+ language_code: 'tr',
+ block_depth: 20,
+ tag: 'p1',
+ });
+ });
+});
+
+// ─── The weekly cache and the nightly cap ───────────────────────────────────
+
+describe('isDue and pickDue', () => {
+ const keyword = { keyword: 'ai search monitoring' };
+
+ it('sets a prompt again a week after it was set, at once for a new keyword, never while waiting', () => {
+ expect(isDue(undefined, keyword, NOW)).toBe(true);
+ const row = { keyword: 'ai search monitoring', tasks: null, posted_at: daysAgo(3) };
+ expect(isDue(row, keyword, NOW)).toBe(false);
+ expect(isDue({ ...row, posted_at: daysAgo(REFRESH_DAYS) }, keyword, NOW)).toBe(true);
+ expect(isDue({ ...row, keyword: 'ai monitoring' }, keyword, NOW)).toBe(true);
+ expect(isDue({ ...row, posted_at: daysAgo(9), tasks: { trends: 't1' } }, keyword, NOW)).toBe(
+ false,
+ );
+ });
+
+ it(`takes at most ${NIGHTLY_LIMIT} a night, the never read first, then the longest waiting`, () => {
+ const candidates = Array.from({ length: 600 }, (_, i) => ({
+ promptId: `p${i}`,
+ postedAt: i % 2 ? daysAgo(8 + (i % 30)) : null,
+ }));
+ const picked = pickDue(candidates);
+ expect(picked).toHaveLength(NIGHTLY_LIMIT);
+ expect(picked.slice(0, 300).every((c) => c.postedAt === null)).toBe(true);
+ const dated = picked.slice(300).map((c) => Date.parse(c.postedAt));
+ expect(dated).toEqual([...dated].sort((a, b) => a - b));
+ });
+});
+
+// ─── The sweep ──────────────────────────────────────────────────────────────
+
+describe('sweepPromptDemand', () => {
+ it('with DEMAND_SIGNALS off, writes Search Console and the level, and sets no paid task', async () => {
+ tables.prompt_volumes = [volume('p1', { 'ai search monitoring': 1900 })];
+ const outcome = await sweepPromptDemand({ now: NOW });
+
+ expect(outcome).toMatchObject({ brands: 1, written: 2, posted: 0, paid: false });
+ expect(postTasks).not.toHaveBeenCalled();
+ expect(readyTasks).not.toHaveBeenCalled();
+ expect(getTask).not.toHaveBeenCalled();
+ expect(rpcCalls.filter((c) => c.name === 'prompt_demand_gsc_queries')).toEqual([
+ {
+ name: 'prompt_demand_gsc_queries',
+ args: { p_brand_id: 'b1', p_from: '2026-10-01', p_to: '2026-10-28', p_limit: 5000 },
+ },
+ ]);
+ expect(rpcCalls.some((c) => c.name === 'prompt_demand_claim')).toBe(false);
+ expect(demandRow('p1')).toMatchObject({
+ brand_id: 'b1',
+ gsc: { impressions: 4120, clicks: 96, queries: 2 },
+ level: 'medium',
+ signals: 1,
+ });
+ // The paused prompt isn't read.
+ expect(demandRow('p3')).toBeUndefined();
+ });
+
+ it('makes no row for a brand without Search Console and nothing else read', async () => {
+ tables.brands[0].gsc_property = null;
+ const outcome = await sweepPromptDemand({ now: NOW });
+ expect(outcome.written).toBe(0);
+ expect(gscReads()).toEqual([]);
+ expect(tables.prompt_demand).toEqual([]);
+ });
+
+ it("clears a brand's Search Console once its property is gone", async () => {
+ tables.brands[0].gsc_property = null;
+ tables.prompt_demand = [
+ {
+ prompt_id: 'p1',
+ brand_id: 'b1',
+ gsc: { impressions: 900, clicks: 9, queries: 1, top: null, at: daysAgo(1) },
+ trends: { trend: 'rising', points: [1], change: 30, at: daysAgo(2) },
+ level: 'high',
+ signals: 2,
+ tasks: null,
+ posted_at: daysAgo(2),
+ keyword: 'ai search monitoring',
+ },
+ ];
+ await sweepPromptDemand({ now: NOW });
+ expect(demandRow('p1')).toMatchObject({ gsc: null, level: 'medium', signals: 1 });
+ });
+
+ it('with DEMAND_SIGNALS on, sets the three tasks for each prompt with a measured keyword', async () => {
+ vi.stubEnv('DEMAND_SIGNALS', 'true');
+ tables.prompt_volumes = [volume('p1', { 'ai search monitoring': 1900, 'ai monitoring': 300 })];
+ // p2 was analysed, and Google has no searches for its keywords.
+ tables.prompt_volumes.push(volume('p2', { 'track brand chatgpt answers': 0 }));
+ postTasks.mockImplementation(setAll);
+
+ const outcome = await sweepPromptDemand({ now: NOW });
+
+ expect(outcome).toMatchObject({ paid: true, due: 1, posted: 1 });
+ // Claimed before it was set.
+ expect(rpcCalls.find((c) => c.name === 'prompt_demand_claim').args).toEqual({
+ p_rows: [{ prompt_id: 'p1', brand_id: 'b1' }],
+ p_stale_minutes: CLAIM_STALE_MINUTES,
+ });
+ expect(postTasks).toHaveBeenCalledTimes(3);
+ expect(postTasks.mock.calls.map(([path]) => path)).toEqual([
+ '/v3/keywords_data/google_trends/explore',
+ '/v3/serp/google/organic',
+ '/v3/serp/youtube/organic',
+ ]);
+ expect(postTasks.mock.calls[1][1]).toEqual([
+ {
+ keyword: 'ai search monitoring',
+ location_code: 2840,
+ language_code: 'en',
+ device: 'desktop',
+ depth: 10,
+ tag: 'p1',
+ },
+ ]);
+ expect(demandRow('p1')).toMatchObject({
+ keyword: 'ai search monitoring',
+ location_code: 2840,
+ language_code: 'en',
+ posted_at: AT,
+ tasks: {
+ posted_at: AT,
+ trends: '/v3/keywords_data/google_trends/explore#p1',
+ paa: '/v3/serp/google/organic#p1',
+ youtube: '/v3/serp/youtube/organic#p1',
+ },
+ });
+ expect(demandRow('p2')?.tasks ?? null).toBeNull();
+ });
+
+ it('does not set a prompt read this week again, nor one whose tasks are waiting', async () => {
+ vi.stubEnv('DEMAND_SIGNALS', 'true');
+ tables.prompt_volumes = [
+ volume('p1', { 'ai search monitoring': 1900 }),
+ volume('p2', { 'track brand chatgpt': 500 }),
+ ];
+ tables.prompt_demand = [
+ {
+ prompt_id: 'p1',
+ brand_id: 'b1',
+ keyword: 'ai search monitoring',
+ posted_at: daysAgo(3),
+ tasks: null,
+ signals: 0,
+ level: null,
+ gsc: null,
+ },
+ {
+ prompt_id: 'p2',
+ brand_id: 'b1',
+ keyword: 'track brand chatgpt',
+ posted_at: daysAgo(1),
+ tasks: { posted_at: daysAgo(1), trends: 't-p2', paa: null, youtube: null },
+ signals: 0,
+ level: null,
+ gsc: null,
+ },
+ ];
+ await sweepPromptDemand({ now: NOW });
+ expect(postTasks).not.toHaveBeenCalled();
+ });
+
+ it('sets at most the limit a night, the longest waiting first', async () => {
+ vi.stubEnv('DEMAND_SIGNALS', 'true');
+ tables.prompts = Array.from({ length: 5 }, (_, i) => prompt(`p${i}`, 'b1', `prompt ${i}`));
+ tables.prompt_volumes = tables.prompts.map((p) => volume(p.id, { [`keyword ${p.id}`]: 100 }));
+ tables.prompt_demand = tables.prompts.map((p, i) => ({
+ prompt_id: p.id,
+ brand_id: 'b1',
+ keyword: `keyword ${p.id}`,
+ posted_at: daysAgo(8 + i),
+ tasks: null,
+ signals: 0,
+ level: null,
+ gsc: null,
+ }));
+ postTasks.mockImplementation(setAll);
+
+ const outcome = await sweepPromptDemand({ now: NOW, limit: 2 });
+
+ // Five were due; the two waiting longest were set.
+ expect(outcome).toMatchObject({ due: 5, posted: 2 });
+ expect(postTasks.mock.calls[0][1].map((t) => t.tag)).toEqual(['p4', 'p3']);
+ });
+
+ it('skips a workspace whose plan has ended: nothing matched, set or read for it', async () => {
+ vi.stubEnv('DEMAND_SIGNALS', 'true');
+ endedOrgs.add('o1');
+ endedBrands.add('b1');
+ tables.prompt_volumes = [volume('p1', { 'ai search monitoring': 1900 })];
+ tables.prompt_demand = [
+ {
+ prompt_id: 'p2',
+ brand_id: 'b1',
+ tasks: { posted_at: daysAgo(1), trends: 't-p2', paa: null, youtube: null },
+ },
+ ];
+ readyTasks.mockResolvedValue(new Map([['t-p2', 'p2']]));
+
+ const outcome = await sweepPromptDemand({ now: NOW });
+
+ expect(outcome).toMatchObject({ brands: 0, skipped: 1, posted: 0, read: 0 });
+ expect(gscReads()).toEqual([]);
+ expect(postTasks).not.toHaveBeenCalled();
+ expect(getTask).not.toHaveBeenCalled();
+ expect(writes).toEqual([]);
+ });
+});
+
+// ─── Setting the tasks: nothing paid for twice, nothing paid for lost ───────
+
+describe('setting the paid tasks', () => {
+ beforeEach(() => {
+ vi.stubEnv('DEMAND_SIGNALS', 'true');
+ tables.prompt_volumes = [
+ volume('p1', { 'ai search monitoring': 1900 }),
+ volume('p2', { 'track brand chatgpt': 500 }),
+ ];
+ });
+
+ const tasksOf = (promptId) => demandRow(promptId)?.tasks ?? null;
+
+ it("writes each request's ids as soon as it's back, and keeps them when a later one fails", async () => {
+ const warn = vi.spyOn(logger, 'warn').mockImplementation(() => {});
+ postTasks.mockImplementation(async (path, tasks, { onChunk }) => {
+ // People Also Ask: p1's request goes through, p2's fails.
+ if (path === SOURCES.paa.path) {
+ await onChunk(new Map([['p1', `${path}#p1`]]));
+ return {
+ ids: new Map([['p1', `${path}#p1`]]),
+ failed: 1,
+ errors: ['DataForSEO API error (502): Bad Gateway'],
+ };
+ }
+ // The others, one request per prompt.
+ for (const t of tasks) await onChunk(new Map([[t.tag, `${path}#${t.tag}`]]));
+ return { ids: new Map(tasks.map((t) => [t.tag, `${path}#${t.tag}`])), failed: 0, errors: [] };
+ });
+
+ const outcome = await sweepPromptDemand({ now: NOW });
+
+ expect(outcome).toMatchObject({ due: 2, posted: 2 });
+ // The first request's id was written before the next request went.
+ const taskWrites = writes.filter((w) => w.rows.some((r) => r.tasks));
+ expect(taskWrites[0].rows).toEqual([
+ expect.objectContaining({
+ prompt_id: 'p1',
+ tasks: {
+ posted_at: AT,
+ trends: '/v3/keywords_data/google_trends/explore#p1',
+ paa: null,
+ youtube: null,
+ },
+ }),
+ ]);
+ expect(tasksOf('p1')).toEqual({
+ posted_at: AT,
+ trends: '/v3/keywords_data/google_trends/explore#p1',
+ paa: '/v3/serp/google/organic#p1',
+ youtube: '/v3/serp/youtube/organic#p1',
+ });
+ expect(tasksOf('p2')).toEqual({
+ posted_at: AT,
+ trends: '/v3/keywords_data/google_trends/explore#p2',
+ paa: null,
+ youtube: '/v3/serp/youtube/organic#p2',
+ });
+ expect(warn).toHaveBeenCalledWith(
+ expect.objectContaining({ signal: 'paa', failed: 1 }),
+ '[prompt-demand] tasks not set',
+ );
+ });
+
+ it("keeps the other prompts' ids when one prompt was deleted meanwhile", async () => {
+ const warn = vi.spyOn(logger, 'warn').mockImplementation(() => {});
+ postTasks.mockImplementation(async (path, tasks, options) => {
+ // p2 is deleted while its tasks are being set (its row goes with it).
+ tables.prompts = tables.prompts.filter((p) => p.id !== 'p2');
+ tables.prompt_demand = tables.prompt_demand.filter((r) => r.prompt_id !== 'p2');
+ return setAll(path, tasks, options);
+ });
+
+ const outcome = await sweepPromptDemand({ now: NOW });
+
+ expect(outcome.posted).toBe(2);
+ expect(tasksOf('p1')).toMatchObject({
+ trends: '/v3/keywords_data/google_trends/explore#p1',
+ paa: '/v3/serp/google/organic#p1',
+ youtube: '/v3/serp/youtube/organic#p1',
+ });
+ expect(demandRow('p2')).toBeUndefined();
+ expect(warn).toHaveBeenCalledWith(
+ { promptIds: ['p2'] },
+ '[prompt-demand] rows of deleted prompts not written',
+ );
+ });
+
+ it('sets each prompt once when two sweeps run at once: the second finds them claimed', async () => {
+ postTasks.mockImplementation(setAll);
+
+ const outcomes = await Promise.all([
+ sweepPromptDemand({ now: NOW }),
+ sweepPromptDemand({ now: NOW }),
+ ]);
+
+ expect(outcomes.map((o) => o.posted).sort()).toEqual([0, 2]);
+ // Three requests (one per signal), each with both prompts once.
+ expect(postTasks).toHaveBeenCalledTimes(3);
+ for (const [, tasks] of postTasks.mock.calls) {
+ expect(tasks.map((t) => t.tag)).toEqual(['p1', 'p2']);
+ }
+ expect(rpcCalls.filter((c) => c.name === 'prompt_demand_claim')).toHaveLength(2);
+ });
+
+ it("doesn't set a prompt another sweep claimed, until the claim is older than its window", async () => {
+ postTasks.mockImplementation(setAll);
+ const claimedBy = (minutesAgo) => ({
+ prompt_id: 'p1',
+ brand_id: 'b1',
+ keyword: 'ai search monitoring',
+ posted_at: daysAgo(8),
+ claimed_at: new Date(NOW - minutesAgo * 60_000).toISOString(),
+ tasks: null,
+ gsc: null,
+ level: null,
+ signals: 0,
+ });
+ const due = [
+ {
+ promptId: 'p1',
+ brandId: 'b1',
+ keyword: 'ai search monitoring',
+ locationCode: 2840,
+ languageCode: 'en',
+ },
+ ];
+
+ // A sweep claimed it half an hour ago and died before setting it.
+ tables.prompt_demand = [claimedBy(30)];
+ expect(isDue(tables.prompt_demand[0], { keyword: 'ai search monitoring' }, NOW)).toBe(false);
+ expect(await postDemandTasks(due, { now: NOW })).toBe(0);
+ expect(postTasks).not.toHaveBeenCalled();
+
+ // Past the window, it's taken again.
+ tables.prompt_demand = [claimedBy(CLAIM_STALE_MINUTES + 1)];
+ expect(isDue(tables.prompt_demand[0], { keyword: 'ai search monitoring' }, NOW)).toBe(true);
+ expect(await postDemandTasks(due, { now: NOW })).toBe(1);
+ expect(postTasks).toHaveBeenCalledTimes(3);
+ expect(demandRow('p1').claimed_at).toBe(AT);
+ });
+});
+
+// ─── The brands' order and the night's time ─────────────────────────────────
+
+describe("the night's brands", () => {
+ beforeEach(() => {
+ tables.brands = ['b1', 'b2', 'b3'].map((id) => ({
+ id,
+ organization_id: 'o1',
+ gsc_property: `sc-domain:${id}.example`,
+ is_active: true,
+ }));
+ tables.prompts = [
+ prompt('p1', 'b1', 'ai search monitoring tool'),
+ prompt('p2', 'b2', 'ai search monitoring tool'),
+ prompt('p3', 'b3', 'ai search monitoring tool'),
+ ];
+ tables.gsc = Object.fromEntries(
+ ['b1', 'b2', 'b3'].map((id) => [
+ id,
+ [{ query: 'ai search monitoring tool', clicks: 1, impressions: 200 }],
+ ]),
+ );
+ // b1 written last night, b3 two nights ago, b2 never.
+ tables.prompt_demand = [
+ {
+ prompt_id: 'p1',
+ brand_id: 'b1',
+ gsc: null,
+ level: null,
+ signals: 0,
+ updated_at: daysAgo(1),
+ },
+ {
+ prompt_id: 'p3',
+ brand_id: 'b3',
+ gsc: null,
+ level: null,
+ signals: 0,
+ updated_at: daysAgo(2),
+ },
+ ];
+ });
+
+ it('takes the brands never written first, then the ones written longest ago', async () => {
+ expect(
+ sweepOrder(
+ tables.brands,
+ new Map([
+ ['b1', NOW - DAY],
+ ['b3', NOW - 2 * DAY],
+ ]),
+ ).map((b) => b.id),
+ ).toEqual(['b2', 'b3', 'b1']);
+
+ const outcome = await sweepPromptDemand({ now: NOW });
+
+ expect(gscReads()).toEqual(['b2', 'b3', 'b1']);
+ expect(outcome).toMatchObject({ brands: 3, unfinished: 0 });
+ });
+
+ it('counts the brands it ran out of time for and says so; they go first the next night', async () => {
+ const warn = vi.spyOn(logger, 'warn').mockImplementation(() => {});
+ let clock = 0;
+ vi.spyOn(Date, 'now').mockImplementation(() => clock);
+ // Matching the first brand takes the whole 20 minutes.
+ onGscRead = () => {
+ clock += 21 * 60_000;
+ };
+
+ const outcome = await sweepPromptDemand({ now: NOW });
+
+ expect(gscReads()).toEqual(['b2']);
+ expect(outcome).toMatchObject({ brands: 1, unfinished: 2 });
+ expect(warn).toHaveBeenCalledWith(
+ expect.objectContaining({ unfinished: 2, brands: 1 }),
+ '[prompt-demand] sweep ran out of time: brands left for the next night',
+ );
+
+ // The next night starts with the two left.
+ clock = 0;
+ rpcCalls = [];
+ onGscRead = null;
+ await sweepPromptDemand({ now: NOW + DAY });
+ expect(gscReads()).toEqual(['b3', 'b1', 'b2']);
+ });
+});
+
+// ─── Reading the tasks ──────────────────────────────────────────────────────
+
+describe('collectDemandTasks', () => {
+ const waiting = (tasks, extra = {}) => ({
+ prompt_id: 'p1',
+ brand_id: 'b1',
+ gsc: { impressions: 4120, clicks: 96, queries: 7, top: null, at: daysAgo(1) },
+ trends: null,
+ paa: null,
+ youtube: null,
+ tasks,
+ level: 'medium',
+ signals: 1,
+ ...extra,
+ });
+
+ it('does nothing with DEMAND_SIGNALS off', async () => {
+ tables.prompt_demand = [waiting({ posted_at: daysAgo(1), trends: 't1' })];
+ expect(await collectDemandTasks({ now: NOW })).toEqual({ read: 0, waiting: 0, skipped: 'off' });
+ expect(readyTasks).not.toHaveBeenCalled();
+ });
+
+ it('reads the tasks that are done, keeps the level, and clears what was read', async () => {
+ vi.stubEnv('DEMAND_SIGNALS', 'true');
+ tables.prompt_demand = [
+ waiting({ posted_at: daysAgo(1), trends: 't1', paa: 's1', youtube: 'y1' }),
+ ];
+ readyTasks.mockImplementation(async (path) =>
+ path.includes('youtube')
+ ? new Map()
+ : new Map([
+ ['t1', 'p1'],
+ ['s1', 'p1'],
+ ]),
+ );
+ const rising = Array.from({ length: 52 }, (_, i) => (i < 4 ? 25 : i >= 48 ? 41 : 30));
+ getTask.mockImplementation(async (path) => {
+ if (path === '/v3/keywords_data/google_trends/explore/task_get/t1') {
+ return { status: 'done', result: trendsResult(rising) };
+ }
+ if (path === '/v3/serp/google/organic/task_get/advanced/s1') {
+ return { status: 'done', result: SERP_RESULT };
+ }
+ throw new Error(`unexpected ${path}`);
+ });
+
+ expect(await collectDemandTasks({ now: NOW })).toEqual({ read: 1, waiting: 1 });
+
+ const row = demandRow('p1');
+ expect(row.trends).toMatchObject({ trend: 'rising', change: 64, at: AT });
+ expect(row.paa.questions).toHaveLength(3);
+ expect(row.youtube).toBeNull();
+ // YouTube isn't done: its task waits for the next read.
+ expect(row.tasks).toEqual({ posted_at: daysAgo(1), trends: null, paa: null, youtube: 'y1' });
+ // Search Console 2 + Trends 2 + PAA 1 of 6: high, three signals read.
+ expect(row).toMatchObject({ level: 'high', signals: 3 });
+ });
+
+ it('gives up a task not done after three days, keeping what was read before', async () => {
+ vi.stubEnv('DEMAND_SIGNALS', 'true');
+ const before = { videos: 3, views: 900, results: 3, top: null, at: daysAgo(9) };
+ tables.prompt_demand = [
+ waiting(
+ { posted_at: daysAgo(TASK_EXPIRE_DAYS), trends: null, paa: null, youtube: 'y1' },
+ { youtube: before },
+ ),
+ ];
+ getTask.mockResolvedValue({ status: 'pending' });
+
+ await collectDemandTasks({ now: NOW });
+
+ expect(getTask).toHaveBeenCalledWith('/v3/serp/youtube/organic/task_get/advanced/y1');
+ expect(demandRow('p1')).toMatchObject({ tasks: null, youtube: before });
+ });
+
+ it('reads a done search that found nothing as read, with nothing in it', async () => {
+ vi.stubEnv('DEMAND_SIGNALS', 'true');
+ tables.prompt_demand = [
+ waiting({ posted_at: daysAgo(1), trends: null, paa: 's1', youtube: null }),
+ ];
+ readyTasks.mockResolvedValue(new Map([['s1', 'p1']]));
+ getTask.mockResolvedValue({ status: 'done', result: null });
+
+ await collectDemandTasks({ now: NOW });
+
+ expect(demandRow('p1')).toMatchObject({
+ paa: { questions: [], at: AT },
+ tasks: null,
+ signals: 2,
+ });
+ });
+});
+
+describe('collecting the tasks whose ids were lost', () => {
+ // A prompt's id is the tag its tasks are set with.
+ const P1 = '6f1c2a3b-0d4e-4f5a-9b6c-7d8e9f0a1b2c';
+ const OTHER = '0a0b0c0d-1e1f-4a2b-8c3d-4e5f6a7b8c9d';
+
+ beforeEach(() => {
+ vi.stubEnv('DEMAND_SIGNALS', 'true');
+ tables.prompts = [prompt(P1, 'b1', 'What is the best AI search monitoring tool?')];
+ });
+
+ it('reads a done task no row knows when its tag is one of our prompts, and nothing else', async () => {
+ // Claimed and set; the request timed out, so its ids were never written.
+ tables.prompt_demand = [
+ {
+ prompt_id: P1,
+ brand_id: 'b1',
+ gsc: null,
+ trends: null,
+ paa: null,
+ youtube: null,
+ tasks: null,
+ posted_at: daysAgo(8),
+ level: null,
+ signals: 0,
+ },
+ ];
+ readyTasks.mockImplementation(async (path) =>
+ path === SOURCES.paa.path
+ ? new Map([
+ ['lost-paa', P1],
+ // Someone else's: a prompt that isn't ours, and a tag that isn't a prompt.
+ ['theirs-1', OTHER],
+ ['theirs-2', 'weekly-report'],
+ ])
+ : new Map(),
+ );
+ getTask.mockResolvedValue({ status: 'done', result: SERP_RESULT });
+
+ expect(await collectDemandTasks({ now: NOW })).toEqual({ read: 1, waiting: 0 });
+
+ expect(getTask.mock.calls).toEqual([['/v3/serp/google/organic/task_get/advanced/lost-paa']]);
+ expect(demandRow(P1)).toMatchObject({ tasks: null, signals: 1 });
+ expect(demandRow(P1).paa.questions).toHaveLength(3);
+ });
+
+ it("gives a waiting row the lost task of a signal it isn't waiting for", async () => {
+ tables.prompt_demand = [
+ {
+ prompt_id: P1,
+ brand_id: 'b1',
+ gsc: null,
+ trends: null,
+ paa: null,
+ youtube: null,
+ tasks: { posted_at: daysAgo(1), trends: 't1', paa: null, youtube: null },
+ posted_at: daysAgo(1),
+ level: null,
+ signals: 0,
+ },
+ ];
+ readyTasks.mockImplementation(async (path) =>
+ path === SOURCES.paa.path ? new Map([['lost-paa', P1]]) : new Map(),
+ );
+ getTask.mockResolvedValue({ status: 'done', result: SERP_RESULT });
+
+ expect(await collectDemandTasks({ now: NOW })).toEqual({ read: 1, waiting: 1 });
+
+ expect(demandRow(P1).tasks).toEqual({
+ posted_at: daysAgo(1),
+ trends: 't1',
+ paa: null,
+ youtube: null,
+ });
+ expect(demandRow(P1).paa.questions).toHaveLength(3);
+ });
+});
+
+describe('demandResponse', () => {
+ it("says what the card needs about the signals that aren't there", () => {
+ expect(demandResponse(null, null, { paid: false })).toEqual({
+ level: null,
+ signals: 0,
+ updatedAt: null,
+ keyword: null,
+ locationCode: null,
+ languageCode: null,
+ gsc: null,
+ trends: null,
+ paa: null,
+ youtube: null,
+ pending: { trends: false, paa: false, youtube: false },
+ hasVolume: false,
+ hasKeyword: false,
+ paidSignals: false,
+ });
+ expect(
+ demandResponse(
+ { level: 'high', signals: 2, tasks: { youtube: 'y1' }, updated_at: AT },
+ volume('p1', { 'ai search monitoring': 1900 }),
+ { paid: true },
+ ),
+ ).toMatchObject({
+ level: 'high',
+ keyword: 'ai search monitoring',
+ locationCode: 2840,
+ pending: { trends: false, paa: false, youtube: true },
+ hasVolume: true,
+ hasKeyword: true,
+ paidSignals: true,
+ });
+ });
+});
diff --git a/server/src/routes/prompts-demand.test.js b/server/src/routes/prompts-demand.test.js
new file mode 100644
index 000000000..975574494
--- /dev/null
+++ b/server/src/routes/prompts-demand.test.js
@@ -0,0 +1,142 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import express from 'express';
+
+/**
+ * GET /api/prompts/:promptId/demand (PROMPT-014): what the nightly sweep
+ * read for the prompt's Demand card. Only for a person of the brand's
+ * organization; a read, so a workspace whose plan has ended still gets it.
+ */
+
+let demand;
+let profileOrg;
+
+function table(name) {
+ const result = () => {
+ if (name === 'prompts') {
+ return { data: { id: 'p1', text: 'best crm', prompt_sets: { brand_id: 'b1' } }, error: null };
+ }
+ if (name === 'brands') return { data: { id: 'b1', organization_id: 'o1' }, error: null };
+ if (name === 'profiles') return { data: { organization_id: profileOrg }, error: null };
+ if (name === 'prompt_demand') return { data: demand, error: null };
+ if (name === 'prompt_volumes') {
+ return {
+ data: {
+ google_volumes: { 'best crm': 9900, 'crm software': 2400 },
+ location_code: 2840,
+ language_code: 'en',
+ fetched_at: '2026-10-20T00:00:00.000Z',
+ },
+ error: null,
+ };
+ }
+ return { data: [], error: null };
+ };
+ const chain = new Proxy(
+ {},
+ {
+ get: (_t, prop) => {
+ if (prop === 'then') {
+ return (resolve, reject) => Promise.resolve(result()).then(resolve, reject);
+ }
+ if (prop === 'single' || prop === 'maybeSingle') return async () => result();
+ return () => chain;
+ },
+ },
+ );
+ return chain;
+}
+
+vi.mock('../config/supabase.js', () => ({ default: { from: (name) => table(name) } }));
+vi.mock('../lib/plan-guard.js', () => ({
+ PlanLimitError: class extends Error {},
+ enforceFeature: vi.fn(),
+ reserveAiBudget: vi.fn(),
+ getImproveQuotaStatus: vi.fn(),
+ requireFeature: () => (_req, _res, next) => next(),
+ reserveRelevanceAllowance: vi.fn(),
+ settleRelevanceUsage: vi.fn(),
+}));
+vi.mock('../lib/action-center/from-prompt-improve.js', () => ({
+ liveImproveSends: vi.fn(),
+ sendImproveToActionCenter: vi.fn(),
+}));
+
+const { default: router } = await import('./prompts.js');
+
+let server;
+let baseUrl;
+
+beforeEach(async () => {
+ demand = null;
+ profileOrg = 'o1';
+ vi.stubEnv('DEMAND_SIGNALS', 'false');
+ const app = express();
+ app.use((req, _res, next) => {
+ req.user = { id: 'u1' };
+ req.log = { error: vi.fn(), info: vi.fn() };
+ next();
+ });
+ app.use('/api/prompts', router);
+ await new Promise((resolve) => {
+ server = app.listen(0, resolve);
+ });
+ baseUrl = `http://127.0.0.1:${server.address().port}/api/prompts`;
+});
+
+afterEach(async () => {
+ vi.unstubAllEnvs();
+ await new Promise((resolve) => server.close(resolve));
+});
+
+describe('GET /api/prompts/:promptId/demand', () => {
+ it('answers the signals read and the level', async () => {
+ demand = {
+ prompt_id: 'p1',
+ brand_id: 'b1',
+ keyword: 'best crm',
+ location_code: 2840,
+ language_code: 'en',
+ gsc: { impressions: 4120, clicks: 96, queries: 7, top: null, at: '2026-10-28T06:00:00Z' },
+ trends: { trend: 'rising', change: 64, points: [25, 41], at: '2026-10-27T07:00:00Z' },
+ paa: null,
+ youtube: null,
+ tasks: { posted_at: '2026-10-27T06:00:00Z', paa: 's1', youtube: 'y1' },
+ level: 'high',
+ signals: 2,
+ updated_at: '2026-10-28T06:00:00Z',
+ };
+ const res = await fetch(`${baseUrl}/p1/demand`);
+ expect(res.status).toBe(200);
+ const body = await res.json();
+ expect(body.demand).toMatchObject({
+ level: 'high',
+ signals: 2,
+ keyword: 'best crm',
+ gsc: { impressions: 4120, clicks: 96, queries: 7 },
+ trends: { trend: 'rising', change: 64 },
+ pending: { trends: false, paa: true, youtube: true },
+ hasVolume: true,
+ hasKeyword: true,
+ paidSignals: false,
+ });
+ });
+
+ it('answers an empty card for a prompt nothing was read for', async () => {
+ const res = await fetch(`${baseUrl}/p1/demand`);
+ expect(res.status).toBe(200);
+ expect((await res.json()).demand).toMatchObject({
+ level: null,
+ signals: 0,
+ keyword: 'best crm',
+ gsc: null,
+ hasVolume: true,
+ hasKeyword: true,
+ });
+ });
+
+ it('refuses a person from another organization', async () => {
+ profileOrg = 'o2';
+ const res = await fetch(`${baseUrl}/p1/demand`);
+ expect(res.status).toBe(403);
+ });
+});
diff --git a/server/src/routes/prompts.js b/server/src/routes/prompts.js
index 0781e6840..5c2b40251 100644
--- a/server/src/routes/prompts.js
+++ b/server/src/routes/prompts.js
@@ -38,6 +38,7 @@ import {
loadImproveRounds,
sendImproveToActionCenter,
} from '../lib/action-center/from-prompt-improve.js';
+import { demandResponse } from '../lib/prompt-demand.js';
const router = Router();
@@ -1085,6 +1086,34 @@ router.get('/:promptId/improve', readImprove, async (req, res) => {
}
});
+// The prompt's real demand (PROMPT-014): the signals the nightly sweep read
+// and their level, for the Demand card. A read: a workspace whose plan has
+// ended still sees what was read.
+router.get('/:promptId/demand', async (req, res) => {
+ try {
+ const promptId = req.params.promptId;
+ await loadImprovePrompt(promptId, req.user.id);
+ const [demand, volume] = await Promise.all([
+ supabaseAdmin.from('prompt_demand').select('*').eq('prompt_id', promptId).maybeSingle(),
+ supabaseAdmin
+ .from('prompt_volumes')
+ .select('google_volumes, location_code, language_code, fetched_at')
+ .eq('prompt_id', promptId)
+ .order('fetched_at', { ascending: false })
+ .limit(1)
+ .maybeSingle(),
+ ]);
+ if (demand.error) throw demand.error;
+ if (volume.error) throw volume.error;
+ return res.json({ demand: demandResponse(demand.data, volume.data) });
+ } catch (error) {
+ req.log.error({ err: error }, 'prompt demand read error');
+ return res
+ .status(error.status || 500)
+ .json({ error: error.status ? error.message : 'Failed to load demand' });
+ }
+});
+
// Build (or rebuild) the set for the range and cache it.
router.post('/:promptId/improve', requireFeature('prompt_improve'), async (req, res) => {
try {
diff --git a/server/src/server.js b/server/src/server.js
index 70c7db6ab..f9bc38352 100644
--- a/server/src/server.js
+++ b/server/src/server.js
@@ -45,6 +45,7 @@ import supabaseAdmin from './config/supabase.js';
import { getPlan, hasFeature, isCloud, isSubscriptionActive } from './config/plans.js';
import { analyzeBrandVolumes } from './lib/volume-analysis.js';
import { runGscSync } from './lib/gsc-sync.js';
+import { scheduleDemandCollect, sweepPromptDemand } from './lib/prompt-demand.js';
import { runGaSync } from './lib/ga-sync.js';
import { sendTrafficDigests } from './lib/traffic-digest/engine.js';
import { runPageOpportunityDetection } from './lib/page-opportunities.js';
@@ -287,9 +288,20 @@ async function runDailyTracking() {
// Search Console sync (#644) — independent follow-up step; never blocks or
// fails the tracking run. No-ops when Composio isn't configured.
- runGscSync().catch((err) => {
- logger.error({ err }, '[gsc-sync] daily run failed');
- });
+ //
+ // Each prompt's real demand (PROMPT-014) follows it, so tonight's queries
+ // are the ones matched: Search Console every night, and the weekly Google
+ // Trends, People Also Ask and YouTube tasks when DEMAND_SIGNALS is on, read
+ // an hour later or the next night. It runs whether or not the sync did.
+ runGscSync()
+ .catch((err) => {
+ logger.error({ err }, '[gsc-sync] daily run failed');
+ })
+ .then(() => sweepPromptDemand())
+ .then(() => scheduleDemandCollect())
+ .catch((err) => {
+ logger.error({ err }, '[prompt-demand] daily sweep crashed');
+ });
// AI-bot rows past their retention (AT-010): 90 days, counters 400.
pruneBotEvents().catch((err) => {
diff --git a/supabase/migrations/00370_prompt_demand.sql b/supabase/migrations/00370_prompt_demand.sql
new file mode 100644
index 000000000..4b7bbd4ab
--- /dev/null
+++ b/supabase/migrations/00370_prompt_demand.sql
@@ -0,0 +1,181 @@
+-- ============================================================================
+-- 00370: real demand for a tracked prompt (PROMPT-014)
+-- ============================================================================
+-- A prompt's AI search volume is an estimate (PROMPT-013). Its Demand card
+-- shows the signs that people really ask it, each with its source and date:
+--
+-- * Search Console: impressions and clicks over 28 days for the brand's
+-- queries that match the prompt (gsc_query_stats, read every night).
+-- * Google Trends: the 12 months of its keyword, rising, steady or falling.
+-- * People Also Ask: the questions Google lists for its keyword.
+-- * YouTube: the videos for its keyword, and their views.
+--
+-- The last three are DataForSEO's queued (Standard) tasks, read about once a
+-- week: the keyword is the prompt's volume analysis's own (prompt_volumes),
+-- where it is measured, so a prompt never analysed has none of them. A task
+-- is set one night and read when it's done, that night or the next (`tasks`).
+-- A sweep claims a prompt before it sets its tasks (prompt_demand_claim), so
+-- two sweeps at once never pay for the same prompt twice.
+--
+-- The level (high, medium, low) weighs the signals there are; `signals` is
+-- how many of the four were read. The Action Center's priority reads it.
+--
+-- * Members read their brands' rows; the server's nightly sweep writes
+-- them (service role). The app reads them through the server
+-- (GET /api/prompts/:id/demand).
+-- * prompt_demand_gsc_queries: a brand's Search Console queries over a
+-- window, with their clicks and impressions, for the sweep to match to
+-- its prompts. Service role only.
+-- * prompt_demand_claim: claims the prompts a sweep is about to set tasks
+-- for. Service role only.
+-- * prompt_demand_brand_updates: when each brand's rows were last written,
+-- so a night starts with the brands waiting longest. Service role only.
+
+create table if not exists public.prompt_demand (
+ prompt_id uuid primary key references public.prompts(id) on delete cascade,
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ -- What the paid signals were read for: the volume analysis's top keyword,
+ -- its DataForSEO location and language.
+ keyword text,
+ location_code integer,
+ language_code text,
+ -- Each { ..., at }: what was read and when. Null: never read, or the
+ -- source isn't there (no Search Console property, no keyword).
+ gsc jsonb,
+ trends jsonb,
+ paa jsonb,
+ youtube jsonb,
+ -- The DataForSEO tasks set and not read yet: { posted_at, trends, paa, youtube } (task ids).
+ tasks jsonb,
+ -- When the paid signals were last set to be read: about once a week.
+ posted_at timestamptz,
+ -- When a sweep last claimed the prompt to set its tasks
+ -- (prompt_demand_claim). A claim older than the sweep's window is one
+ -- that died, and may be taken again.
+ claimed_at timestamptz,
+ level text
+ constraint prompt_demand_level_check check (level in ('high', 'medium', 'low')),
+ signals smallint not null default 0
+ constraint prompt_demand_signals_check check (signals between 0 and 4),
+ updated_at timestamptz not null default now()
+);
+
+create index if not exists prompt_demand_brand_idx on public.prompt_demand (brand_id);
+-- The tasks waiting to be read: few at a time, looked up every night.
+create index if not exists prompt_demand_tasks_idx
+ on public.prompt_demand (prompt_id) where tasks is not null;
+
+-- Read by the brand's members (the policy below); written by the server.
+alter table public.prompt_demand enable row level security;
+revoke all on table public.prompt_demand from anon, authenticated;
+grant select on table public.prompt_demand to authenticated;
+grant all on table public.prompt_demand to service_role;
+
+drop policy if exists "prompt_demand: member select" on public.prompt_demand;
+create policy "prompt_demand: member select"
+ on public.prompt_demand
+ for select
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid()
+ )
+ );
+
+comment on table public.prompt_demand is
+ 'Real demand for a tracked prompt (PROMPT-014): Search Console impressions and clicks (nightly), Google Trends, People Also Ask and YouTube (DataForSEO, about weekly), and the level they come to. Written by the server''s nightly sweep; members read.';
+
+-- A brand's Search Console queries over a window, the most seen first, with
+-- their clicks and impressions: the sweep matches each one to a prompt.
+-- jsonb, so PostgREST's row cap doesn't cut the list at 1,000.
+create or replace function public.prompt_demand_gsc_queries(
+ p_brand_id uuid, p_from date, p_to date, p_limit integer default 5000
+) returns jsonb
+language sql
+stable
+set search_path = public
+as $$
+ select coalesce(jsonb_agg(row_to_json(x)), '[]'::jsonb) from (
+ select g.query,
+ sum(g.clicks)::bigint as clicks,
+ sum(g.impressions)::bigint as impressions
+ from public.gsc_query_stats g
+ where g.brand_id = p_brand_id and g.date between p_from and p_to
+ group by g.query
+ order by sum(g.impressions) desc, g.query
+ limit greatest(least(coalesce(p_limit, 5000), 20000), 1)
+ ) x
+$$;
+
+revoke all on function public.prompt_demand_gsc_queries(uuid, date, date, integer) from public, anon, authenticated;
+grant execute on function public.prompt_demand_gsc_queries(uuid, date, date, integer) to service_role;
+
+comment on function public.prompt_demand_gsc_queries(uuid, date, date, integer) is
+ 'PROMPT-014 (00370): a brand''s Search Console queries between two days with their clicks and impressions, the most seen first, at most p_limit (default 5,000). The demand sweep matches them to prompts. Service role only.';
+
+-- Claims the prompts a sweep is about to set DataForSEO tasks for, before
+-- it sets them: a prompt is claimed when no task of its is waiting and no
+-- other sweep claimed it in the last p_stale_minutes. Makes the row of a
+-- prompt that has none yet; a prompt deleted since is left out. Returns
+-- the prompt ids claimed (jsonb, past PostgREST's row cap): only those are
+-- set. Two sweeps at once: the row's lock makes the second wait, and then
+-- find the claim.
+create or replace function public.prompt_demand_claim(
+ p_rows jsonb, p_stale_minutes integer default 120
+) returns jsonb
+language sql
+volatile
+set search_path = public
+as $$
+ with asked as (
+ select distinct on (r.prompt_id) r.prompt_id, r.brand_id
+ from jsonb_to_recordset(coalesce(p_rows, '[]'::jsonb)) as r(prompt_id uuid, brand_id uuid)
+ join public.prompts p on p.id = r.prompt_id
+ where r.brand_id is not null
+ order by r.prompt_id
+ ), claimed as (
+ insert into public.prompt_demand as d (prompt_id, brand_id, claimed_at)
+ select a.prompt_id, a.brand_id, now() from asked a
+ on conflict (prompt_id) do update
+ set claimed_at = excluded.claimed_at
+ where d.tasks is null
+ and (
+ d.claimed_at is null
+ or d.claimed_at < now() - make_interval(mins => greatest(coalesce(p_stale_minutes, 120), 1))
+ )
+ returning d.prompt_id
+ )
+ select coalesce(jsonb_agg(c.prompt_id), '[]'::jsonb) from claimed c
+$$;
+
+revoke all on function public.prompt_demand_claim(jsonb, integer) from public, anon, authenticated;
+grant execute on function public.prompt_demand_claim(jsonb, integer) to service_role;
+
+comment on function public.prompt_demand_claim(jsonb, integer) is
+ 'PROMPT-014 (00370): claims [{prompt_id, brand_id}] for a sweep about to set their DataForSEO tasks: those with no task waiting and no claim in the last p_stale_minutes (default 120). Returns the prompt ids claimed. Service role only.';
+
+-- When each brand's demand rows were last written, { brand_id: timestamptz }:
+-- a night takes the brands never written first, then the ones waiting
+-- longest, so a sweep that runs out of time leaves different brands each
+-- night.
+create or replace function public.prompt_demand_brand_updates()
+returns jsonb
+language sql
+stable
+set search_path = public
+as $$
+ select coalesce(jsonb_object_agg(x.brand_id, x.updated_at), '{}'::jsonb) from (
+ select d.brand_id, max(d.updated_at) as updated_at
+ from public.prompt_demand d
+ group by d.brand_id
+ ) x
+$$;
+
+revoke all on function public.prompt_demand_brand_updates() from public, anon, authenticated;
+grant execute on function public.prompt_demand_brand_updates() to service_role;
+
+comment on function public.prompt_demand_brand_updates() is
+ 'PROMPT-014 (00370): when each brand''s prompt_demand rows were last written, as { brand_id: updated_at }, for the nightly sweep''s brand order. Service role only.';
+
+notify pgrst, 'reload schema';
diff --git a/supabase/schema.sql b/supabase/schema.sql
index cd7d9efc4..081ee94a7 100644
--- a/supabase/schema.sql
+++ b/supabase/schema.sql
@@ -42082,3 +42082,188 @@ comment on function public.action_scorecard(uuid, timestamptz) is
notify pgrst, 'reload schema';
+-- ─────────────────────────────────────────────────────────────────────────
+-- migrations/00370_prompt_demand.sql
+-- ─────────────────────────────────────────────────────────────────────────
+-- ============================================================================
+-- 00370: real demand for a tracked prompt (PROMPT-014)
+-- ============================================================================
+-- A prompt's AI search volume is an estimate (PROMPT-013). Its Demand card
+-- shows the signs that people really ask it, each with its source and date:
+--
+-- * Search Console: impressions and clicks over 28 days for the brand's
+-- queries that match the prompt (gsc_query_stats, read every night).
+-- * Google Trends: the 12 months of its keyword, rising, steady or falling.
+-- * People Also Ask: the questions Google lists for its keyword.
+-- * YouTube: the videos for its keyword, and their views.
+--
+-- The last three are DataForSEO's queued (Standard) tasks, read about once a
+-- week: the keyword is the prompt's volume analysis's own (prompt_volumes),
+-- where it is measured, so a prompt never analysed has none of them. A task
+-- is set one night and read when it's done, that night or the next (`tasks`).
+-- A sweep claims a prompt before it sets its tasks (prompt_demand_claim), so
+-- two sweeps at once never pay for the same prompt twice.
+--
+-- The level (high, medium, low) weighs the signals there are; `signals` is
+-- how many of the four were read. The Action Center's priority reads it.
+--
+-- * Members read their brands' rows; the server's nightly sweep writes
+-- them (service role). The app reads them through the server
+-- (GET /api/prompts/:id/demand).
+-- * prompt_demand_gsc_queries: a brand's Search Console queries over a
+-- window, with their clicks and impressions, for the sweep to match to
+-- its prompts. Service role only.
+-- * prompt_demand_claim: claims the prompts a sweep is about to set tasks
+-- for. Service role only.
+-- * prompt_demand_brand_updates: when each brand's rows were last written,
+-- so a night starts with the brands waiting longest. Service role only.
+
+create table if not exists public.prompt_demand (
+ prompt_id uuid primary key references public.prompts(id) on delete cascade,
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ -- What the paid signals were read for: the volume analysis's top keyword,
+ -- its DataForSEO location and language.
+ keyword text,
+ location_code integer,
+ language_code text,
+ -- Each { ..., at }: what was read and when. Null: never read, or the
+ -- source isn't there (no Search Console property, no keyword).
+ gsc jsonb,
+ trends jsonb,
+ paa jsonb,
+ youtube jsonb,
+ -- The DataForSEO tasks set and not read yet: { posted_at, trends, paa, youtube } (task ids).
+ tasks jsonb,
+ -- When the paid signals were last set to be read: about once a week.
+ posted_at timestamptz,
+ -- When a sweep last claimed the prompt to set its tasks
+ -- (prompt_demand_claim). A claim older than the sweep's window is one
+ -- that died, and may be taken again.
+ claimed_at timestamptz,
+ level text
+ constraint prompt_demand_level_check check (level in ('high', 'medium', 'low')),
+ signals smallint not null default 0
+ constraint prompt_demand_signals_check check (signals between 0 and 4),
+ updated_at timestamptz not null default now()
+);
+
+create index if not exists prompt_demand_brand_idx on public.prompt_demand (brand_id);
+-- The tasks waiting to be read: few at a time, looked up every night.
+create index if not exists prompt_demand_tasks_idx
+ on public.prompt_demand (prompt_id) where tasks is not null;
+
+-- Read by the brand's members (the policy below); written by the server.
+alter table public.prompt_demand enable row level security;
+revoke all on table public.prompt_demand from anon, authenticated;
+grant select on table public.prompt_demand to authenticated;
+grant all on table public.prompt_demand to service_role;
+
+drop policy if exists "prompt_demand: member select" on public.prompt_demand;
+create policy "prompt_demand: member select"
+ on public.prompt_demand
+ for select
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid()
+ )
+ );
+
+comment on table public.prompt_demand is
+ 'Real demand for a tracked prompt (PROMPT-014): Search Console impressions and clicks (nightly), Google Trends, People Also Ask and YouTube (DataForSEO, about weekly), and the level they come to. Written by the server''s nightly sweep; members read.';
+
+-- A brand's Search Console queries over a window, the most seen first, with
+-- their clicks and impressions: the sweep matches each one to a prompt.
+-- jsonb, so PostgREST's row cap doesn't cut the list at 1,000.
+create or replace function public.prompt_demand_gsc_queries(
+ p_brand_id uuid, p_from date, p_to date, p_limit integer default 5000
+) returns jsonb
+language sql
+stable
+set search_path = public
+as $$
+ select coalesce(jsonb_agg(row_to_json(x)), '[]'::jsonb) from (
+ select g.query,
+ sum(g.clicks)::bigint as clicks,
+ sum(g.impressions)::bigint as impressions
+ from public.gsc_query_stats g
+ where g.brand_id = p_brand_id and g.date between p_from and p_to
+ group by g.query
+ order by sum(g.impressions) desc, g.query
+ limit greatest(least(coalesce(p_limit, 5000), 20000), 1)
+ ) x
+$$;
+
+revoke all on function public.prompt_demand_gsc_queries(uuid, date, date, integer) from public, anon, authenticated;
+grant execute on function public.prompt_demand_gsc_queries(uuid, date, date, integer) to service_role;
+
+comment on function public.prompt_demand_gsc_queries(uuid, date, date, integer) is
+ 'PROMPT-014 (00370): a brand''s Search Console queries between two days with their clicks and impressions, the most seen first, at most p_limit (default 5,000). The demand sweep matches them to prompts. Service role only.';
+
+-- Claims the prompts a sweep is about to set DataForSEO tasks for, before
+-- it sets them: a prompt is claimed when no task of its is waiting and no
+-- other sweep claimed it in the last p_stale_minutes. Makes the row of a
+-- prompt that has none yet; a prompt deleted since is left out. Returns
+-- the prompt ids claimed (jsonb, past PostgREST's row cap): only those are
+-- set. Two sweeps at once: the row's lock makes the second wait, and then
+-- find the claim.
+create or replace function public.prompt_demand_claim(
+ p_rows jsonb, p_stale_minutes integer default 120
+) returns jsonb
+language sql
+volatile
+set search_path = public
+as $$
+ with asked as (
+ select distinct on (r.prompt_id) r.prompt_id, r.brand_id
+ from jsonb_to_recordset(coalesce(p_rows, '[]'::jsonb)) as r(prompt_id uuid, brand_id uuid)
+ join public.prompts p on p.id = r.prompt_id
+ where r.brand_id is not null
+ order by r.prompt_id
+ ), claimed as (
+ insert into public.prompt_demand as d (prompt_id, brand_id, claimed_at)
+ select a.prompt_id, a.brand_id, now() from asked a
+ on conflict (prompt_id) do update
+ set claimed_at = excluded.claimed_at
+ where d.tasks is null
+ and (
+ d.claimed_at is null
+ or d.claimed_at < now() - make_interval(mins => greatest(coalesce(p_stale_minutes, 120), 1))
+ )
+ returning d.prompt_id
+ )
+ select coalesce(jsonb_agg(c.prompt_id), '[]'::jsonb) from claimed c
+$$;
+
+revoke all on function public.prompt_demand_claim(jsonb, integer) from public, anon, authenticated;
+grant execute on function public.prompt_demand_claim(jsonb, integer) to service_role;
+
+comment on function public.prompt_demand_claim(jsonb, integer) is
+ 'PROMPT-014 (00370): claims [{prompt_id, brand_id}] for a sweep about to set their DataForSEO tasks: those with no task waiting and no claim in the last p_stale_minutes (default 120). Returns the prompt ids claimed. Service role only.';
+
+-- When each brand's demand rows were last written, { brand_id: timestamptz }:
+-- a night takes the brands never written first, then the ones waiting
+-- longest, so a sweep that runs out of time leaves different brands each
+-- night.
+create or replace function public.prompt_demand_brand_updates()
+returns jsonb
+language sql
+stable
+set search_path = public
+as $$
+ select coalesce(jsonb_object_agg(x.brand_id, x.updated_at), '{}'::jsonb) from (
+ select d.brand_id, max(d.updated_at) as updated_at
+ from public.prompt_demand d
+ group by d.brand_id
+ ) x
+$$;
+
+revoke all on function public.prompt_demand_brand_updates() from public, anon, authenticated;
+grant execute on function public.prompt_demand_brand_updates() to service_role;
+
+comment on function public.prompt_demand_brand_updates() is
+ 'PROMPT-014 (00370): when each brand''s prompt_demand rows were last written, as { brand_id: updated_at }, for the nightly sweep''s brand order. Service role only.';
+
+notify pgrst, 'reload schema';
+