Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(signals): stop the nightly pass dying on the brands that need it most (#829)
Gökhan Geyik03ec2edf74a6
diff --git a/server/src/lib/job-runner.js b/server/src/lib/job-runner.jsindex 8a7b81b1a..013ffe06c 100644--- a/server/src/lib/job-runner.js+++ b/server/src/lib/job-runner.js@@ -16,8 +16,7 @@ import {import { processTrackingJob } from '../workers/tracking-worker.js';import { processContentJob } from '../workers/content-worker.js';import { generatePulseForBrand } from './pulse/engine.js';-import { recordSignalsForBrand } from './signals/record.js';-import { generateActionsForBrand } from './action-center/generate.js';+import { runSignalPass } from './signals/pass.js';import logger from './logger.js';// Concurrency counters (the default of 2 per queue is inherited from the Bull@@ -98,14 +97,15 @@ export async function runTrackingJob(jobId, io) {});// Signal recording (Action Center) rides the same trigger but is// deliberately NOT inside the pulse engine: signals are a product- // surface, so a brand with pulse emails off still gets them. Action- // generation chains after it — it consolidates the signals the- // recording just wrote, so running concurrently would read stale.- recordSignalsForBrand(brandId)- .then(() => generateActionsForBrand(brandId))- .catch((err) => {- logger.error({ err, brandId }, '[signals] recording or action generation failed');- });+ // surface, so a brand with pulse emails off still gets them.+ //+ // Still fire-and-forget, but no longer forgettable: the pass logs+ // itself against the run it covered, and the nightly catch-up sweep+ // re-runs whatever has no row. A failure here costs the brand a few+ // hours now, not the night.+ runSignalPass(brandId).catch((err) => {+ logger.error({ err, brandId }, '[signals] pass failed — catch-up will retry');+ });} else {logger.warn({ brandId, resultCount: result?.resultCount },diff --git a/server/src/lib/pulse/metrics.js b/server/src/lib/pulse/metrics.jsindex 0edbbd5b2..94fcaed98 100644--- a/server/src/lib/pulse/metrics.js+++ b/server/src/lib/pulse/metrics.js@@ -16,6 +16,36 @@ import { resolve } from '../../config/action-engine.js';const DAY_MS = 86_400_000;+function utcDay(date) {+ return date.toISOString().slice(0, 10);+}++function addDays(day, n) {+ return new Date(Date.parse(`${day}T00:00:00Z`) + n * DAY_MS).toISOString().slice(0, 10);+}++/** The instants a whole-day window spans, for callers with no daily RPC. */+function dayWindowBounds(win) {+ return [new Date(`${win.from}T00:00:00.000Z`), new Date(`${win.to}T23:59:59.999Z`)];+}++/**+ * The trailing `length` whole UTC days ending today, and the equal window+ * before it.+ *+ * Same convention the dashboard's delta math uses (deltaDayWindows in+ * web/src/lib/actions/tracking.ts), so a comparison the pulse reports and one+ * the user reads on Insights cover the same days.+ */+export function trailingDayWindows(now, length) {+ const to = utcDay(now);+ const from = addDays(to, -(length - 1));+ return {+ cur: { from, to },+ prev: { from: addDays(from, -length), to: addDays(from, -1) },+ };+}+// Detector thresholds live in config/action-engine.js, with every other// number that decides what counts (#818 phase 2.5). Read through `resolve()`// so per-definition and per-workspace overrides land without touching this@@ -87,16 +117,30 @@ async function rpc(name, params, attempt = 0) {return data;}-async function visibilityRate(brandId, from, to) {- const params = {- p_brand_id: brandId,- p_date_from: from.toISOString(),- p_date_to: to.toISOString(),- };+/**+ * A window the rollups can answer, or the raw timestamps.+ *+ * `days` is a whole-UTC-day window ({ from: 'YYYY-MM-DD', to: 'YYYY-MM-DD' },+ * inclusive on both ends). When one is given the call is served from the+ * pre-aggregated daily tables (00066), whose cost scales with the number of+ * days rather than with everything the brand has ever collected. Without one+ * the raw RPC answers, which is what a window that does not fall on day+ * boundaries — the 24-hour run-anchored one — needs.+ */+export function windowParams(brandId, from, to, days) {+ return days+ ? { p_brand_id: brandId, p_day_from: days.from, p_day_to: days.to }+ : { p_brand_id: brandId, p_date_from: from.toISOString(), p_date_to: to.toISOString() };+}++export const daily = (name, days) => (days ? `${name}_daily` : name);++async function visibilityRate(brandId, from, to, days = null) {+ const params = windowParams(brandId, from, to, days);const [stats, tracked, vis] = await Promise.all([- rpc('visible_prompt_stats', params),- rpc('tracked_prompt_count', params),- rpc('ai_visibility_aggregates', params),+ rpc(daily('visible_prompt_stats', days), params),+ rpc(daily('tracked_prompt_count', days), params),+ rpc(daily('ai_visibility_aggregates', days), params),]);const visible = stats?.visible_prompts ?? 0;const total = tracked ?? 0;@@ -112,12 +156,8 @@ async function visibilityRate(brandId, from, to) {return { visible, total, rate: score };}-async function insightsWindow(brandId, from, to) {- const agg = await rpc('insights_aggregates', {- p_brand_id: brandId,- p_date_from: from.toISOString(),- p_date_to: to.toISOString(),- });+async function insightsWindow(brandId, from, to, days = null) {+ const agg = await rpc(daily('insights_aggregates', days), windowParams(brandId, from, to, days));const mentioning = agg?.mentioning_results ?? 0;return {totalResults: agg?.total_results ?? 0,@@ -127,12 +167,11 @@ async function insightsWindow(brandId, from, to) {};}-async function competitorRates(brandId, from, to) {- const agg = await rpc('ai_visibility_aggregates', {- p_brand_id: brandId,- p_date_from: from.toISOString(),- p_date_to: to.toISOString(),- });+async function competitorRates(brandId, from, to, days = null) {+ const agg = await rpc(+ daily('ai_visibility_aggregates', days),+ windowParams(brandId, from, to, days),+ );// Shared denominator (the brand's answers), same as the web leaderboard.const answers = agg?.answers ?? 0;const brandRate =@@ -416,9 +455,9 @@ export async function computePulseMetrics(brandId, { windowDays = 1, now = new D// windows the Insights 24h view resolves (getTrackingWindow) — so the// email can never disagree with the dashboard. A wall-clock [now-24h]// window double-counts whenever two runs land within 24h of each other- // (e.g. the day the cron time moved earlier). Weekly pulses and the- // detector windows below stay clock-based per the issue spec; brands- // with no completed run yet fall back to the clock window.+ // (e.g. the day the cron time moved earlier). Brands with no completed run+ // yet fall back to the clock window. Everything wider than a day is read in+ // whole UTC days instead — see below.let curTo = now;let curFrom = new Date(now.getTime() - windowDays * DAY_MS);let prevTo = curFrom;@@ -438,8 +477,26 @@ export async function computePulseMetrics(brandId, { windowDays = 1, now = new D}}- const weekFrom = new Date(now.getTime() - 7 * DAY_MS);- const twoWeekFrom = new Date(now.getTime() - 14 * DAY_MS);+ // Everything measured in days is read from the daily rollups, and is+ // therefore expressed in whole UTC days rather than in clock offsets.+ //+ // This is the fix for a silent failure, not a tidy-up. `insights_aggregates`+ // and `ai_visibility_aggregates` rescan every result a brand has ever+ // collected; past roughly 25k results that exceeds PostgREST's 8s statement+ // timeout, and on the largest brand it did so on every single call. The+ // pulse survived on its retries and its catch-up sweep. Signal recording,+ // which shares this function, had neither — so that brand recorded no+ // signals at all for six days, and with them no actions. The rollup answers+ // the same window in a tenth of a second.+ //+ // The 24-hour window stays on the raw RPCs: it is anchored to the+ // tracking-run ledger rather than to midnight, so no whole-day window can+ // express it, and it is cheap for exactly the same reason the wide ones+ // are not.+ const week = trailingDayWindows(now, 7);+ const detector = windowDays === 1 ? null : trailingDayWindows(now, windowDays);+ const [weekFrom, weekTo] = dayWindowBounds(week.cur);+ const [twoWeekFrom, twoWeekTo] = dayWindowBounds(week.prev);const prompts = await brandPrompts(brandId);const promptById = new Map(prompts.map((p) => [p.id, p]));@@ -459,15 +516,18 @@ export async function computePulseMetrics(brandId, { windowDays = 1, now = new DlostCitations,degraded,] = await series([- () => visibilityRate(brandId, curFrom, curTo),- () => visibilityRate(brandId, weekFrom, now),- () => visibilityRate(brandId, twoWeekFrom, weekFrom),- () => insightsWindow(brandId, curFrom, curTo),- () => insightsWindow(brandId, prevFrom, prevTo),- () => competitorRates(brandId, weekFrom, now),- () => competitorRates(brandId, twoWeekFrom, weekFrom),- () => promptScores(brandId, weekFrom, now),- () => promptScores(brandId, twoWeekFrom, weekFrom),+ () => visibilityRate(brandId, curFrom, curTo, detector?.cur ?? null),+ () => visibilityRate(brandId, weekFrom, weekTo, week.cur),+ () => visibilityRate(brandId, twoWeekFrom, twoWeekTo, week.prev),+ () => insightsWindow(brandId, curFrom, curTo, detector?.cur ?? null),+ () => insightsWindow(brandId, prevFrom, prevTo, detector?.prev ?? null),+ () => competitorRates(brandId, weekFrom, weekTo, week.cur),+ () => competitorRates(brandId, twoWeekFrom, twoWeekTo, week.prev),+ // No daily variant exists for the per-prompt summaries, so these stay on+ // the raw RPC — but over the same days as everything above, so the+ // movers cannot describe a different week from the rates beside them.+ () => promptScores(brandId, weekFrom, weekTo),+ () => promptScores(brandId, twoWeekFrom, twoWeekTo),() => firstTimeCitations(promptById, curFrom),() => newEngineAppearances(brandId, curFrom),() => lostCitationPrompts(brandId, promptById, now),diff --git a/server/src/lib/pulse/metrics.test.js b/server/src/lib/pulse/metrics.test.jsindex a5f2ace03..4aea362c3 100644--- a/server/src/lib/pulse/metrics.test.js+++ b/server/src/lib/pulse/metrics.test.js@@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest';// SUPABASE_* env is absent (as in CI) — stub it before the import chain.vi.mock('../../config/supabase.js', () => ({ default: {} }));-import { isTransientDbError, series } from './metrics.js';+import { daily, isTransientDbError, series, trailingDayWindows, windowParams } from './metrics.js';/*** Pulse metric execution shape (#687).@@ -76,3 +76,71 @@ describe('isTransientDbError', () => {expect(isTransientDbError('')).toBe(false);});});++/**+ * Window shape (#818).+ *+ * The wide windows moved off the raw aggregate RPCs, which rescan everything+ * a brand has ever collected and cross the database's 8s statement timeout+ * once a brand is large enough. They are now expressed in whole UTC days and+ * answered from the daily rollups. What is worth pinning is the arithmetic:+ * which days a comparison covers decides what every detector reads.+ */+describe('trailingDayWindows', () => {+ const now = new Date('2026-09-25T02:13:00Z');++ it('covers the trailing seven days, today included', () => {+ expect(trailingDayWindows(now, 7).cur).toEqual({ from: '2026-09-19', to: '2026-09-25' });+ });++ it('puts the previous window immediately before, never overlapping', () => {+ const { cur, prev } = trailingDayWindows(now, 7);+ expect(prev).toEqual({ from: '2026-09-12', to: '2026-09-18' });+ expect(prev.to < cur.from).toBe(true);+ });++ it('gives both windows the same length, so the comparison is like for like', () => {+ const { cur, prev } = trailingDayWindows(now, 7);+ const span = (w) => (Date.parse(w.to) - Date.parse(w.from)) / 86_400_000;+ expect(span(cur)).toBe(span(prev));+ });++ /** The hour the nightly run happens must not change which days it reads. */+ it('does not depend on the time of day', () => {+ const early = trailingDayWindows(new Date('2026-09-25T00:04:00Z'), 7);+ const late = trailingDayWindows(new Date('2026-09-25T23:58:00Z'), 7);+ expect(early).toEqual(late);+ });++ it('crosses a month boundary by date, not by arithmetic on the day number', () => {+ expect(trailingDayWindows(new Date('2026-03-02T05:00:00Z'), 7).cur.from).toBe('2026-02-24');+ });+});++describe('rollup routing', () => {+ it('sends a whole-day window to the rollup variant', () => {+ const days = { from: '2026-09-19', to: '2026-09-25' };+ expect(daily('ai_visibility_aggregates', days)).toBe('ai_visibility_aggregates_daily');+ expect(windowParams('b1', null, null, days)).toEqual({+ p_brand_id: 'b1',+ p_day_from: '2026-09-19',+ p_day_to: '2026-09-25',+ });+ });++ /**+ * The 24-hour window is anchored to the tracking-run ledger rather than to+ * midnight, so no whole-day window can express it — and it is cheap, being+ * one run's worth of rows.+ */+ it('leaves a run-anchored window on the raw RPC', () => {+ const from = new Date('2026-09-24T02:10:00Z');+ const to = new Date('2026-09-25T02:13:00Z');+ expect(daily('ai_visibility_aggregates', null)).toBe('ai_visibility_aggregates');+ expect(windowParams('b1', from, to, null)).toEqual({+ p_brand_id: 'b1',+ p_date_from: from.toISOString(),+ p_date_to: to.toISOString(),+ });+ });+});diff --git a/server/src/lib/signals/pass.js b/server/src/lib/signals/pass.jsnew file mode 100644index 000000000..44cf1fbe0--- /dev/null+++ b/server/src/lib/signals/pass.js@@ -0,0 +1,145 @@+/**+ * The nightly signal pass, and the sweep that picks up the ones it lost.+ *+ * Recording signals and consolidating them into actions are two steps of one+ * thing: a night either produced both or produced nothing useful. They were+ * chained inline in the job runner, fire-and-forget, with a `.catch` that+ * logged. That is how the biggest brand went six days without a single signal+ * — every night its metric queries crossed the database's statement timeout,+ * every night the chain threw, and every night the error went into a log file+ * nobody reads while the Action Center quietly showed week-old work.+ *+ * Two things change here. The pass writes a row when it finishes, so "did it+ * run?" is a question the database can answer. And a sweep re-runs the passes+ * that have no row, so a night lost to a timeout, a deploy or an OOM is+ * recovered the next morning instead of never.+ *+ * The daily pulse has had both since #702. This is the same shape for the+ * half of the same trigger that never got it.+ */++import supabaseAdmin from '../../config/supabase.js';+import { logger } from '../logger.js';+import { recordSignalsForBrand } from './record.js';+import { generateActionsForBrand } from '../action-center/generate.js';++/**+ * How far back the sweep looks. Long enough to cover a missed night plus the+ * gap between two runs, short enough that a brand paused a week ago does not+ * get a pass it has no use for.+ */+const CATCH_UP_WINDOW_MS = 36 * 60 * 60 * 1000;++/** The brand's most recent stamped nightly run, or null. */+async function latestStampedRun(brandId) {+ const { data, error } = await supabaseAdmin+ .from('tracking_runs')+ .select('completed_at')+ .eq('brand_id', brandId)+ .eq('source', 'cron')+ .not('completed_at', 'is', null)+ .order('completed_at', { ascending: false })+ .limit(1);+ if (error) throw new Error(error.message);+ return data?.[0]?.completed_at ?? null;+}++/**+ * Record signals and consolidate them into actions for one brand, then log+ * the pass against the run it covered.+ *+ * The ledger row is written last and only on success: a pass that threw+ * halfway leaves no row, which is precisely what makes the sweep able to find+ * it. Idempotent on (brand, run) so a sweep racing the inline trigger cannot+ * run the same night twice.+ *+ * @param {string} brandId+ * @param {{ runAt?: string, now?: Date }} [options] runAt names the tracking+ * run this pass covers; omitted, the brand's most recent stamped run.+ */+export async function runSignalPass(brandId, { runAt, now = new Date() } = {}) {+ const trackingRunAt = runAt ?? (await latestStampedRun(brandId));+ if (!trackingRunAt) {+ logger.info({ brandId }, '[signals] no stamped run to record against');+ return { skipped: 'no_stamped_run' };+ }++ const recorded = await recordSignalsForBrand(brandId, { now });+ const generated = recorded.skipped ? null : await generateActionsForBrand(brandId, { now });++ const { error } = await supabaseAdmin.from('signal_runs').upsert(+ {+ brand_id: brandId,+ tracking_run_at: trackingRunAt,+ ran_at: now.toISOString(),+ outcome: recorded.skipped ? 'skipped' : 'recorded',+ detail: { signals: recorded, actions: generated },+ },+ { onConflict: 'brand_id,tracking_run_at' },+ );+ if (error) throw new Error(error.message);++ return { trackingRunAt, signals: recorded, actions: generated };+}++/**+ * Re-run the passes that never finished.+ *+ * Every brand whose nightly run stamped within the window but has no pass+ * logged against that run. A brand whose pass succeeded is skipped by the+ * ledger row, so the sweep costs one query on a healthy night.+ */+export async function runSignalCatchUp({ now = new Date() } = {}) {+ try {+ const since = new Date(now.getTime() - CATCH_UP_WINDOW_MS).toISOString();+ const { data: runs, error } = await supabaseAdmin+ .from('tracking_runs')+ .select('brand_id, completed_at')+ .eq('source', 'cron')+ .not('completed_at', 'is', null)+ .gte('completed_at', since)+ .order('completed_at', { ascending: false });+ if (error) throw new Error(error.message);++ const latestByBrand = new Map();+ for (const run of runs ?? []) {+ if (!latestByBrand.has(run.brand_id)) latestByBrand.set(run.brand_id, run.completed_at);+ }+ if (latestByBrand.size === 0) return { recovered: 0, healthy: 0, failed: 0 };++ const { data: passes, error: passErr } = await supabaseAdmin+ .from('signal_runs')+ .select('brand_id, tracking_run_at')+ .gte('tracking_run_at', since);+ if (passErr) throw new Error(passErr.message);+ const done = new Set((passes ?? []).map((row) => `${row.brand_id}|${row.tracking_run_at}`));++ let recovered = 0;+ let healthy = 0;+ let failed = 0;++ for (const [brandId, completedAt] of latestByBrand) {+ if (done.has(`${brandId}|${completedAt}`)) {+ healthy += 1;+ continue;+ }+ try {+ await runSignalPass(brandId, { runAt: completedAt, now });+ logger.info({ brandId, runCompletedAt: completedAt }, '[signals] catch-up pass completed');+ recovered += 1;+ } catch (err) {+ // One brand's failure must not cost every brand behind it in the loop.+ logger.error({ err, brandId }, '[signals] catch-up pass failed');+ failed += 1;+ }+ }++ if (recovered > 0 || failed > 0) {+ logger.info({ recovered, healthy, failed }, '[signals] catch-up sweep completed');+ }+ return { recovered, healthy, failed };+ } catch (err) {+ logger.error({ err }, '[signals] catch-up sweep failed');+ return { error: true };+ }+}diff --git a/server/src/lib/signals/pass.test.js b/server/src/lib/signals/pass.test.jsnew file mode 100644index 000000000..dbcf29aae--- /dev/null+++ b/server/src/lib/signals/pass.test.js@@ -0,0 +1,182 @@+import { beforeEach, describe, expect, it, vi } from 'vitest';++const from = vi.fn();+vi.mock('../../config/supabase.js', () => ({ default: { from: (...a) => from(...a) } }));+vi.mock('../logger.js', () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++const recordSignalsForBrand = vi.fn();+const generateActionsForBrand = vi.fn();+vi.mock('./record.js', () => ({ recordSignalsForBrand: (...a) => recordSignalsForBrand(...a) }));+vi.mock('../action-center/generate.js', () => ({+ generateActionsForBrand: (...a) => generateActionsForBrand(...a),+}));++import { runSignalCatchUp, runSignalPass } from './pass.js';++const BRAND = 'brand-1';+const NOW = new Date('2026-09-25T06:00:00Z');+const RUN_AT = '2026-09-25T02:13:00.000Z';++/**+ * Enough PostgREST to drive the pass: `tracking_runs` and `signal_runs`+ * answer from the fixtures, and every write is recorded.+ */+function mockDb({ runs = [], passes = [] } = {}) {+ const writes = [];+ from.mockImplementation((table) => {+ const builder = {+ select: () => builder,+ eq: () => builder,+ not: () => builder,+ gte: () => builder,+ order: () => builder,+ limit: () => builder,+ upsert: (row) => {+ writes.push({ table, row });+ return Promise.resolve({ error: null });+ },+ then: (resolve, reject) => {+ const data = table === 'tracking_runs' ? runs : table === 'signal_runs' ? passes : [];+ return Promise.resolve({ data, error: null }).then(resolve, reject);+ },+ };+ return builder;+ });+ return writes;+}++beforeEach(() => {+ from.mockReset();+ recordSignalsForBrand.mockReset();+ generateActionsForBrand.mockReset();+ recordSignalsForBrand.mockResolvedValue({ inserted: 2, refreshed: 1 });+ generateActionsForBrand.mockResolvedValue({ created: 1, refreshed: 0 });+});++describe('runSignalPass', () => {+ it('records signals, generates actions, and logs the pass against the run', async () => {+ const writes = mockDb({ runs: [{ completed_at: RUN_AT }] });++ const result = await runSignalPass(BRAND, { now: NOW });++ expect(recordSignalsForBrand).toHaveBeenCalledOnce();+ expect(generateActionsForBrand).toHaveBeenCalledOnce();+ expect(result.trackingRunAt).toBe(RUN_AT);+ expect(writes).toHaveLength(1);+ expect(writes[0]).toMatchObject({+ table: 'signal_runs',+ row: { brand_id: BRAND, tracking_run_at: RUN_AT, outcome: 'recorded' },+ });+ });++ /**+ * The row is the whole point of the ledger: written only when the pass+ * finished, so a pass that threw is exactly what the sweep can find.+ */+ it('logs nothing when recording throws, so the sweep can find the night', async () => {+ const writes = mockDb({ runs: [{ completed_at: RUN_AT }] });+ recordSignalsForBrand.mockRejectedValue(new Error('canceling statement due to timeout'));++ await expect(runSignalPass(BRAND, { now: NOW })).rejects.toThrow(/timeout/);++ expect(writes).toHaveLength(0);+ });++ it('logs nothing when action generation throws', async () => {+ const writes = mockDb({ runs: [{ completed_at: RUN_AT }] });+ generateActionsForBrand.mockRejectedValue(new Error('boom'));++ await expect(runSignalPass(BRAND, { now: NOW })).rejects.toThrow('boom');++ expect(writes).toHaveLength(0);+ });++ /**+ * A run with no fresh results is a decision, not a failure — recording it+ * stops the sweep from retrying the same empty night every morning.+ */+ it('logs a deliberate skip and does not generate actions', async () => {+ const writes = mockDb({ runs: [{ completed_at: RUN_AT }] });+ recordSignalsForBrand.mockResolvedValue({ skipped: 'no_fresh_results' });++ await runSignalPass(BRAND, { now: NOW });++ expect(generateActionsForBrand).not.toHaveBeenCalled();+ expect(writes[0].row.outcome).toBe('skipped');+ });++ it('does nothing for a brand whose nightly run has never stamped', async () => {+ const writes = mockDb({ runs: [] });++ const result = await runSignalPass(BRAND, { now: NOW });++ expect(result).toEqual({ skipped: 'no_stamped_run' });+ expect(recordSignalsForBrand).not.toHaveBeenCalled();+ expect(writes).toHaveLength(0);+ });+});++describe('runSignalCatchUp', () => {+ it('re-runs a brand whose run stamped but whose pass never finished', async () => {+ const writes = mockDb({ runs: [{ brand_id: BRAND, completed_at: RUN_AT }], passes: [] });++ const summary = await runSignalCatchUp({ now: NOW });++ expect(summary).toMatchObject({ recovered: 1, healthy: 0, failed: 0 });+ expect(writes).toHaveLength(1);+ });++ it('leaves a brand alone once its pass is logged', async () => {+ const writes = mockDb({+ runs: [{ brand_id: BRAND, completed_at: RUN_AT }],+ passes: [{ brand_id: BRAND, tracking_run_at: RUN_AT }],+ });++ const summary = await runSignalCatchUp({ now: NOW });++ expect(summary).toMatchObject({ recovered: 0, healthy: 1 });+ expect(recordSignalsForBrand).not.toHaveBeenCalled();+ expect(writes).toHaveLength(0);+ });++ /** One brand's timeout must not cost every brand behind it in the loop. */+ it('carries on past a brand that fails', async () => {+ mockDb({+ runs: [+ { brand_id: 'big', completed_at: RUN_AT },+ { brand_id: 'small', completed_at: RUN_AT },+ ],+ });+ recordSignalsForBrand.mockImplementation((brandId) =>+ brandId === 'big' ? Promise.reject(new Error('statement timeout')) : Promise.resolve({}),+ );++ const summary = await runSignalCatchUp({ now: NOW });++ expect(summary).toMatchObject({ recovered: 1, failed: 1 });+ expect(recordSignalsForBrand).toHaveBeenCalledTimes(2);+ });++ it('considers only the newest run per brand', async () => {+ mockDb({+ runs: [+ { brand_id: BRAND, completed_at: RUN_AT },+ { brand_id: BRAND, completed_at: '2026-09-24T02:10:00.000Z' },+ ],+ });++ const summary = await runSignalCatchUp({ now: NOW });++ expect(summary.recovered).toBe(1);+ });++ it('reports rather than throws when the sweep itself fails', async () => {+ from.mockImplementation(() => {+ throw new Error('connection refused');+ });++ expect(await runSignalCatchUp({ now: NOW })).toEqual({ error: true });+ });+});diff --git a/server/src/server.js b/server/src/server.jsindex c1ea46bba..3cabb2ae8 100644--- a/server/src/server.js+++ b/server/src/server.js@@ -36,6 +36,7 @@ import { runGscSync } from './lib/gsc-sync.js';import { runGaSync } from './lib/ga-sync.js';import { runPageOpportunityDetection } from './lib/page-opportunities.js';import { runPulseCatchUp } from './lib/pulse/engine.js';+import { runSignalCatchUp } from './lib/signals/pass.js';const app = express();app.set('trust proxy', 1);@@ -124,6 +125,15 @@ async function runDailyTracking() {logger.error({ err }, '[pulse] catch-up sweep crashed');});+ // The same recovery for signals and actions (#818). A pass that threw last+ // night — a statement timeout on a large brand, a deploy, an OOM — left no+ // row in signal_runs, and this re-runs it. Without it a lost night was lost+ // for good: the Action Center simply showed nothing new and said nothing+ // about why.+ runSignalCatchUp().catch((err) => {+ logger.error({ err }, '[signals] catch-up sweep crashed');+ });+// Insights rollup backstop (00066). Re-refreshes the trailing days for// every brand, so days whose run never stamped (partial, crashed) and// out-of-run writes (a prompt analyzed right after being added) appear thediff --git a/supabase/migrations/00087_signal_runs.sql b/supabase/migrations/00087_signal_runs.sqlnew file mode 100644index 000000000..e92b68591--- /dev/null+++ b/supabase/migrations/00087_signal_runs.sql@@ -0,0 +1,49 @@+-- A ledger for the nightly signal pass (#818).+--+-- The pass — record signals, then consolidate them into actions — has been+-- fire-and-forget since it was written: a failure logs and the night is gone.+-- Nothing recorded that it had run, so nothing could tell a brand with no+-- conditions firing from a brand whose pass threw, and nothing could pick a+-- lost night back up. On the largest brand it threw every night for six days+-- (its aggregate queries crossed the statement timeout) and the first anyone+-- knew was the Action Center looking stale.+--+-- One row per (brand, tracking run) once the pass has finished for that run.+-- That makes two things possible: the catch-up sweep can ask "which stamped+-- runs have no pass?", and anyone can answer "did it run last night?" without+-- reading server logs.+--+-- The pulse has had exactly this since #702, in `sent_pulses`. This is its+-- counterpart for the half of the trigger that never got one.++create table if not exists public.signal_runs (+ id uuid primary key default gen_random_uuid(),+ brand_id uuid not null references public.brands(id) on delete cascade,++ -- The tracking run this pass covered, by its completion stamp. The pair is+ -- unique so a retry is idempotent and a catch-up cannot double-run.+ tracking_run_at timestamptz not null,++ ran_at timestamptz not null default now(),++ -- 'recorded' — the pass ran and wrote what it found+ -- 'skipped' — it ran and deliberately did nothing (no fresh results)+ outcome text not null check (outcome in ('recorded', 'skipped')),++ -- Counts from the pass, for answering "what did last night do" cheaply.+ detail jsonb not null default '{}'::jsonb,++ unique (brand_id, tracking_run_at)+);++create index if not exists signal_runs_brand_ran_idx+ on public.signal_runs (brand_id, ran_at desc);++-- Service-role only, and deliberately with no policies rather than a policy+-- named for the service role: such a policy applies to anon and authenticated+-- (service_role bypasses RLS entirely), which is the opposite of its name.+-- No policy means the web cannot read this table; nothing in the web does.+alter table public.signal_runs enable row level security;++comment on table public.signal_runs is+ 'One row per completed nightly signal pass. Drives the catch-up sweep and answers whether the pass ran for a given tracking run.';diff --git a/supabase/schema.sql b/supabase/schema.sqlindex 9a4375a5f..7a4198bdc 100644--- a/supabase/schema.sql+++ b/supabase/schema.sql@@ -10449,3 +10449,56 @@ alter table public.actionscomment on column public.actions.definition_version is'Version of the action definition (server/src/lib/action-center/definitions) that raised this action. Historical rows keep the rules they were raised under.';+-- ─────────────────────────────────────────────────────────────────────────+-- migrations/00087_signal_runs.sql+-- ─────────────────────────────────────────────────────────────────────────+-- A ledger for the nightly signal pass (#818).+--+-- The pass — record signals, then consolidate them into actions — has been+-- fire-and-forget since it was written: a failure logs and the night is gone.+-- Nothing recorded that it had run, so nothing could tell a brand with no+-- conditions firing from a brand whose pass threw, and nothing could pick a+-- lost night back up. On the largest brand it threw every night for six days+-- (its aggregate queries crossed the statement timeout) and the first anyone+-- knew was the Action Center looking stale.+--+-- One row per (brand, tracking run) once the pass has finished for that run.+-- That makes two things possible: the catch-up sweep can ask "which stamped+-- runs have no pass?", and anyone can answer "did it run last night?" without+-- reading server logs.+--+-- The pulse has had exactly this since #702, in `sent_pulses`. This is its+-- counterpart for the half of the trigger that never got one.++create table if not exists public.signal_runs (+ id uuid primary key default gen_random_uuid(),+ brand_id uuid not null references public.brands(id) on delete cascade,++ -- The tracking run this pass covered, by its completion stamp. The pair is+ -- unique so a retry is idempotent and a catch-up cannot double-run.+ tracking_run_at timestamptz not null,++ ran_at timestamptz not null default now(),++ -- 'recorded' — the pass ran and wrote what it found+ -- 'skipped' — it ran and deliberately did nothing (no fresh results)+ outcome text not null check (outcome in ('recorded', 'skipped')),++ -- Counts from the pass, for answering "what did last night do" cheaply.+ detail jsonb not null default '{}'::jsonb,++ unique (brand_id, tracking_run_at)+);++create index if not exists signal_runs_brand_ran_idx+ on public.signal_runs (brand_id, ran_at desc);++-- Service-role only, and deliberately with no policies rather than a policy+-- named for the service role: such a policy applies to anon and authenticated+-- (service_role bypasses RLS entirely), which is the opposite of its name.+-- No policy means the web cannot read this table; nothing in the web does.+alter table public.signal_runs enable row level security;++comment on table public.signal_runs is+ 'One row per completed nightly signal pass. Drives the catch-up sweep and answers whether the pass ran for a given tracking run.';+