Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(traffic): read Search Console pages a week at a time, and resume their backfill (#1253)

Cihandd830feee04d
diff --git a/server/src/lib/gsc-sync-pages.test.js b/server/src/lib/gsc-sync-pages.test.js
new file mode 100644
index 000000000..8c53fac68
--- /dev/null
+++ b/server/src/lib/gsc-sync-pages.test.js
@@ -0,0 +1,124 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+/**
+ * The Search Console page sync (AT-008, 00201) end to end over its windows:
+ * a week at a time, and a backfill cut short resumes the next day after the
+ * last window it read instead of starting over (#1177's review).
+ */
+
+const state = vi.hoisted(() => ({ brand: null, pulls: [], updates: [], failOn: null }));
+
+vi.mock('./composio.js', () => ({
+ isComposioConfigured: () => true,
+ queryGscSearchAnalytics: async (_entity, args) => {
+ if (args.dimensions[0] === 'page') state.pulls.push([args.start_date, args.end_date]);
+ if (state.failOn && args.dimensions[0] === 'page' && args.start_date === state.failOn) {
+ throw new Error('GSC 500');
+ }
+ return [];
+ },
+}));
+vi.mock('./logger.js', () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }));
+vi.mock('../config/supabase.js', () => ({
+ default: {
+ from: (table) => {
+ let update = null;
+ const chain = {
+ select: () => chain,
+ eq: () => chain,
+ not: () => chain,
+ in: () => chain,
+ lt: () => chain,
+ delete: () => chain,
+ upsert: async () => ({ error: null }),
+ update: (fields) => {
+ update = fields;
+ return chain;
+ },
+ then: (resolve) => {
+ if (update) {
+ state.updates.push(update);
+ Object.assign(state.brand, update);
+ return resolve({ error: null });
+ }
+ if (table === 'integration_connections') {
+ return resolve({ data: [{ organization_id: 'o1' }], error: null });
+ }
+ if (table === 'brands') return resolve({ data: [{ ...state.brand }], error: null });
+ return resolve({ error: null });
+ },
+ };
+ return chain;
+ },
+ },
+}));
+
+const { runGscSync, PAGE_BACKFILL_DAYS } = await import('./gsc-sync.js');
+
+const day = (daysAgo, now) => new Date(now - daysAgo * 86_400_000).toISOString().slice(0, 10);
+
+beforeEach(() => {
+ vi.useFakeTimers();
+ vi.setSystemTime(new Date('2026-10-20T06:00:00Z'));
+ Object.assign(state, {
+ brand: {
+ id: 'b1',
+ organization_id: 'o1',
+ gsc_property: 'sc-domain:acme.com',
+ gsc_pages_backfilled_property: null,
+ gsc_pages_backfill_from: null,
+ },
+ pulls: [],
+ updates: [],
+ failOn: null,
+ });
+});
+
+describe('the page sync', () => {
+ it('reads a week at a time, then the backfill, and marks it through', async () => {
+ const now = Date.now();
+ await runGscSync({ brandId: 'b1' });
+ // The last 28 days in five pulls, then 190 days back a week at a time.
+ expect(state.pulls[0]).toEqual([day(6, now), day(0, now)]);
+ expect(state.pulls[4]).toEqual([day(28, now), day(28, now)]);
+ expect(state.pulls[5]).toEqual([day(35, now), day(29, now)]);
+ expect(state.pulls.at(-1)[0]).toBe(day(PAGE_BACKFILL_DAYS, now));
+ expect(state.brand).toMatchObject({
+ gsc_pages_backfilled_property: 'sc-domain:acme.com',
+ gsc_pages_backfill_from: null,
+ });
+
+ // The next day: the last 28 days only.
+ vi.setSystemTime(new Date('2026-10-21T06:00:00Z'));
+ state.pulls = [];
+ await runGscSync({ brandId: 'b1' });
+ expect(state.pulls).toHaveLength(5);
+ });
+
+ it('resumes a backfill cut short after the last window it read', async () => {
+ const now = Date.now();
+ // The fourth backfill window (57–63 days ago) fails today.
+ state.failOn = day(63, now);
+ await runGscSync({ brandId: 'b1' });
+ expect(state.brand.gsc_pages_backfill_from).toBe(day(56, now));
+
+ // Tomorrow it works: the recent days, then on from 57 days ago as of today.
+ vi.setSystemTime(new Date('2026-10-21T06:00:00Z'));
+ state.failOn = null;
+ state.pulls = [];
+ await runGscSync({ brandId: 'b1' });
+ expect(state.pulls.slice(0, 5)).toHaveLength(5);
+ expect(state.pulls[5][1]).toBe(day(57, now));
+ // No backfill window read yesterday is read again.
+ for (const [, end] of state.pulls.slice(5)) expect(end <= day(57, now)).toBe(true);
+ expect(state.brand.gsc_pages_backfill_from).toBeNull();
+ });
+
+ it('starts over when a first sync stops in the recent days', async () => {
+ state.failOn = day(13, Date.now());
+ await runGscSync({ brandId: 'b1' });
+ // Nothing marked: the next sync reads the property from the start.
+ expect(state.updates).toEqual([]);
+ expect(state.brand.gsc_pages_backfilled_property).toBeNull();
+ });
+});
diff --git a/server/src/lib/gsc-sync.js b/server/src/lib/gsc-sync.js
index 6647f5020..c7f008506 100644
--- a/server/src/lib/gsc-sync.js
+++ b/server/src/lib/gsc-sync.js
@@ -5,8 +5,9 @@
* live), pulls the last SYNC_WINDOW_DAYS of per-query daily stats through
* Composio and upserts them into gsc_query_stats, and the same per page into
* gsc_page_stats (AT-008: which pages lose clicks while impressions rise). A
- * brand's page sync reads PAGE_BACKFILL_DAYS until one gets through them all
- * for its property, so a 90-day range has a period before it (pagesSyncPlan).
+ * brand's page sync reads a week at a time, and reads its property's
+ * PAGE_BACKFILL_DAYS once, resuming where a sync stopped, so a 90-day range
+ * has a period before it (pagesSyncPlan).
* Parameters are fully deterministic — no LLM anywhere in
* this path. Per-brand failures are logged and skipped; one broken property
* never aborts the run.
@@ -23,10 +24,14 @@ const ROW_LIMIT = 5000;
// — hitting it is logged, never silent.
const MAX_PAGES = 5;
const UPSERT_CHUNK = 500;
-// Page stats (AT-008): the first sync reads 190 days — a 90-day range and
-// the 90 days before it, plus Google's delay — in SYNC_WINDOW_DAYS steps so
-// each step keeps the row cap; the daily prune keeps as many.
+// Page stats (AT-008): the first syncs read 190 days — a 90-day range and
+// the 90 days before it, plus Google's delay; the daily prune keeps as many.
export const PAGE_BACKFILL_DAYS = 190;
+// A week per [page, date] pull: GSC sorts its rows by clicks, so a pull that
+// hits the row cap drops the zero-click page-days. Over a week the cap is
+// spread evenly instead of falling on the older days of a 28-day pull, which
+// are read once while recent days are re-read daily (#1177's review).
+export const PAGE_WINDOW_DAYS = 7;
/** YYYY-MM-DD in UTC, `daysAgo` days before now. */
function utcDateString(daysAgo = 0) {
@@ -59,19 +64,25 @@ export function mapPageRows(brandId, rows) {
}
/**
- * The windows a brand's page sync reads, as days ago: the last
- * SYNC_WINDOW_DAYS once its PAGE_BACKFILL_DAYS have been read, else those in
- * SYNC_WINDOW_DAYS steps.
+ * The page windows from `newest` to `oldest` days ago, both included,
+ * PAGE_WINDOW_DAYS at a time and newest first, with no gap.
*/
-export function pageSyncWindows(backfilled) {
- if (backfilled) return [{ from: SYNC_WINDOW_DAYS, to: 0 }];
+export function pageWindows(newest, oldest) {
const windows = [];
- for (let to = 0; to < PAGE_BACKFILL_DAYS; to += SYNC_WINDOW_DAYS + 1) {
- windows.push({ from: Math.min(to + SYNC_WINDOW_DAYS, PAGE_BACKFILL_DAYS), to });
+ for (let to = newest; to <= oldest; to += PAGE_WINDOW_DAYS) {
+ windows.push({ from: Math.min(to + PAGE_WINDOW_DAYS - 1, oldest), to });
}
return windows;
}
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+/** Whole UTC days from a YYYY-MM-DD date to `now`'s. */
+function daysSince(date, now) {
+ const today = Date.parse(new Date(now).toISOString().slice(0, 10));
+ return Math.round((today - Date.parse(date)) / DAY_MS);
+}
+
/**
* Pull one window of daily rows by `dimension` and upsert them, paging
* through the result set up to the row cap. Returns the rows written.
@@ -131,23 +142,38 @@ async function syncBrand(brand) {
}
/**
- * What a brand's page sync does (AT-008). Its PAGE_BACKFILL_DAYS are read
- * once per Search Console property: until a sync gets through them all
- * (gsc_pages_backfilled_property names the property), every sync reads them
- * again, so one cut short resumes the next day rather than leaving the brand
- * 28 days of history, and its 90-day range `incomplete`, for months. A brand
- * mapped to another property drops the old one's pages first. Then the last
- * SYNC_WINDOW_DAYS.
+ * What a brand's page sync does (AT-008, 00201). Every sync reads the last
+ * SYNC_WINDOW_DAYS. A property's PAGE_BACKFILL_DAYS are read once, older
+ * windows after those: gsc_pages_backfilled_property names the property the
+ * page rows belong to, and gsc_pages_backfill_from how far back its backfill
+ * has read (null once through). So a sync cut short resumes after its last
+ * window the next day, rather than reading the whole history again, and a
+ * window that keeps failing is the only one tried again. A brand mapped to
+ * another property drops the old one's pages and starts over.
*/
-export function pagesSyncPlan(brand) {
- const backfilled = brand.gsc_pages_backfilled_property === brand.gsc_property;
+export function pagesSyncPlan(brand, now = Date.now()) {
+ const recent = pageWindows(0, SYNC_WINDOW_DAYS);
+ const started = brand.gsc_pages_backfilled_property === brand.gsc_property;
+ if (started && !brand.gsc_pages_backfill_from) {
+ return { recent, backfill: [], start: false, dropOldPages: false };
+ }
+ // Days up to SYNC_WINDOW_DAYS ago are the recent windows' every day.
+ const readTo = started
+ ? Math.max(daysSince(brand.gsc_pages_backfill_from, now), SYNC_WINDOW_DAYS)
+ : SYNC_WINDOW_DAYS;
return {
- windows: pageSyncWindows(backfilled),
- dropOldPages: !backfilled && brand.gsc_pages_backfilled_property != null,
- markBackfilled: !backfilled,
+ recent,
+ backfill: readTo < PAGE_BACKFILL_DAYS ? pageWindows(readTo + 1, PAGE_BACKFILL_DAYS) : [],
+ start: !started,
+ dropOldPages: !started && brand.gsc_pages_backfilled_property != null,
};
}
+async function updateBrand(brand, fields) {
+ const { error } = await supabaseAdmin.from('brands').update(fields).eq('id', brand.id);
+ if (error) throw new Error(error.message);
+}
+
/** The brand's page stats (AT-008), as pagesSyncPlan says. */
async function syncBrandPages(brand) {
const plan = pagesSyncPlan(brand);
@@ -156,15 +182,25 @@ async function syncBrandPages(brand) {
if (error) throw new Error(error.message);
}
let total = 0;
- for (const window of plan.windows) {
+ for (const window of plan.recent) {
total += await pullWindow(brand, PAGES, window);
}
- if (plan.markBackfilled) {
- const { error } = await supabaseAdmin
- .from('brands')
- .update({ gsc_pages_backfilled_property: brand.gsc_property })
- .eq('id', brand.id);
- if (error) throw new Error(error.message);
+ if (plan.start) {
+ // Only once the recent days are in, so a first sync cut short there
+ // starts over rather than leaving them out.
+ await updateBrand(brand, {
+ gsc_pages_backfilled_property: brand.gsc_property,
+ gsc_pages_backfill_from: utcDateString(SYNC_WINDOW_DAYS),
+ });
+ }
+ for (const window of plan.backfill) {
+ total += await pullWindow(brand, PAGES, window);
+ // Kept after each window, so the next sync resumes after it.
+ await updateBrand(brand, { gsc_pages_backfill_from: utcDateString(window.from) });
+ }
+ // Through the PAGE_BACKFILL_DAYS: from now on the recent windows only.
+ if (plan.start || brand.gsc_pages_backfill_from) {
+ await updateBrand(brand, { gsc_pages_backfill_from: null });
}
return total;
}
@@ -192,7 +228,9 @@ export async function runGscSync({ organizationId, brandId } = {}) {
// …and their active brands with a mapped property.
let brandQuery = supabaseAdmin
.from('brands')
- .select('id, organization_id, gsc_property, gsc_pages_backfilled_property')
+ .select(
+ 'id, organization_id, gsc_property, gsc_pages_backfilled_property, gsc_pages_backfill_from',
+ )
.eq('is_active', true)
.not('gsc_property', 'is', null)
.in('organization_id', connectedOrgIds);
diff --git a/server/src/lib/gsc-sync.test.js b/server/src/lib/gsc-sync.test.js
index 1a08b7a89..3359a5796 100644
--- a/server/src/lib/gsc-sync.test.js
+++ b/server/src/lib/gsc-sync.test.js
@@ -8,7 +8,8 @@ import {
mapAnalyticsRows,
mapPageRows,
PAGE_BACKFILL_DAYS,
- pageSyncWindows,
+ PAGE_WINDOW_DAYS,
+ pageWindows,
pagesSyncPlan,
} from './gsc-sync.js';
@@ -107,47 +108,107 @@ describe('mapPageRows', () => {
});
});
-describe('pageSyncWindows', () => {
- it('reads the last 28 days once the 190 have been read', () => {
- expect(pageSyncWindows(true)).toEqual([{ from: 28, to: 0 }]);
- });
-
- it('reads 190 days in 28-day steps until then, with no gap', () => {
- const windows = pageSyncWindows(false);
- expect(windows[0]).toEqual({ from: 28, to: 0 });
+describe('pageWindows', () => {
+ it('reads a week at a time, newest first, with no gap', () => {
+ const windows = pageWindows(29, PAGE_BACKFILL_DAYS);
+ expect(windows[0]).toEqual({ from: 35, to: 29 });
expect(windows.at(-1).from).toBe(PAGE_BACKFILL_DAYS);
for (let i = 1; i < windows.length; i++) {
expect(windows[i].to).toBe(windows[i - 1].from + 1);
- expect(windows[i].from - windows[i].to).toBeLessThanOrEqual(28);
+ expect(windows[i].from - windows[i].to + 1).toBeLessThanOrEqual(PAGE_WINDOW_DAYS);
}
});
+
+ it('reads the last 28 days in five windows, today included', () => {
+ expect(pageWindows(0, 28)).toEqual([
+ { from: 6, to: 0 },
+ { from: 13, to: 7 },
+ { from: 20, to: 14 },
+ { from: 27, to: 21 },
+ { from: 28, to: 28 },
+ ]);
+ });
});
describe('pagesSyncPlan', () => {
const property = 'sc-domain:acme.com';
+ const now = Date.parse('2026-10-20T06:00:00Z');
+ const recent = pageWindows(0, 28);
- it('reads the 190 days, and marks them read, until a sync gets through them for the property', () => {
- // A first sync, or one cut short after its first window: no marker yet.
- const plan = pagesSyncPlan({ gsc_property: property, gsc_pages_backfilled_property: null });
+ it("reads a new property's 190 days after its last 28, and starts its backfill", () => {
+ const plan = pagesSyncPlan(
+ {
+ gsc_property: property,
+ gsc_pages_backfilled_property: null,
+ gsc_pages_backfill_from: null,
+ },
+ now,
+ );
expect(plan).toEqual({
- windows: pageSyncWindows(false),
+ recent,
+ backfill: pageWindows(29, PAGE_BACKFILL_DAYS),
+ start: true,
dropOldPages: false,
- markBackfilled: true,
});
});
- it('reads the last 28 days once the property has been read in full', () => {
+ it('reads only the last 28 days once the backfill is through', () => {
expect(
- pagesSyncPlan({ gsc_property: property, gsc_pages_backfilled_property: property }),
- ).toEqual({ windows: [{ from: 28, to: 0 }], dropOldPages: false, markBackfilled: false });
+ pagesSyncPlan(
+ {
+ gsc_property: property,
+ gsc_pages_backfilled_property: property,
+ gsc_pages_backfill_from: null,
+ },
+ now,
+ ),
+ ).toEqual({ recent, backfill: [], start: false, dropOldPages: false });
});
- it("drops the old property's pages before reading a new one's 190 days", () => {
- expect(
- pagesSyncPlan({
+ // gkhngyk on #1177: the backfill restarted from the newest window, so one
+ // window that kept failing made every run read the whole history again.
+ it('resumes a backfill the day after the last window it read', () => {
+ // A sync read back to 2026-08-21 (60 days ago) and stopped.
+ const plan = pagesSyncPlan(
+ {
gsc_property: property,
- gsc_pages_backfilled_property: 'https://old.example/',
- }),
- ).toEqual({ windows: pageSyncWindows(false), dropOldPages: true, markBackfilled: true });
+ gsc_pages_backfilled_property: property,
+ gsc_pages_backfill_from: '2026-08-21',
+ },
+ now,
+ );
+ expect(plan.start).toBe(false);
+ expect(plan.backfill[0]).toEqual({ from: 67, to: 61 });
+ expect(plan.backfill.at(-1).from).toBe(PAGE_BACKFILL_DAYS);
+ });
+
+ it('reads nothing more once the backfill reached its 190 days', () => {
+ const plan = pagesSyncPlan(
+ {
+ gsc_property: property,
+ gsc_pages_backfilled_property: property,
+ gsc_pages_backfill_from: '2026-04-13',
+ },
+ now,
+ );
+ expect(plan.backfill).toEqual([]);
+ });
+
+ it("drops the old property's pages before reading a new one's", () => {
+ expect(
+ pagesSyncPlan(
+ {
+ gsc_property: property,
+ gsc_pages_backfilled_property: 'https://old.example/',
+ gsc_pages_backfill_from: '2026-09-01',
+ },
+ now,
+ ),
+ ).toEqual({
+ recent,
+ backfill: pageWindows(29, PAGE_BACKFILL_DAYS),
+ start: true,
+ dropOldPages: true,
+ });
});
});
diff --git a/supabase/migrations/00201_gsc_pages_backfill_from.sql b/supabase/migrations/00201_gsc_pages_backfill_from.sql
new file mode 100644
index 000000000..0a34818a0
--- /dev/null
+++ b/supabase/migrations/00201_gsc_pages_backfill_from.sql
@@ -0,0 +1,24 @@
+-- ============================================================================
+-- 00201: the Search Console page backfill resumes where it stopped
+-- ============================================================================
+-- Follow-up from #1177's review (AT-008). The page sync read the property's
+-- 190 days again on every run until one got through them all
+-- (gsc_pages_backfilled_property, 00181), so a window that kept failing made
+-- every daily run read the whole history again. It now reads a week at a
+-- time (gsc-sync.js), and keeps how far back the backfill has read:
+--
+-- * gsc_pages_backfilled_property: the property the brand's page rows
+-- belong to. Set once its last 28 days are read, and when it names
+-- another property, that one's page rows are dropped first.
+-- * gsc_pages_backfill_from: the oldest day the property's backfill has
+-- read so far, while it hasn't reached its 190 days; null once it has.
+-- A sync cut short resumes the day after.
+--
+-- A brand backfilled before this column (property set, this null) reads as
+-- done, as it was.
+
+alter table public.brands
+ add column if not exists gsc_pages_backfill_from date;
+
+comment on column public.brands.gsc_pages_backfill_from is
+ 'AI Traffic (AT-008): the oldest day the Search Console page backfill for gsc_pages_backfilled_property has read while it hasn''t reached its 190 days; null once it has (gsc-sync.js).';
diff --git a/supabase/schema.sql b/supabase/schema.sql
index c67a84fba..89802616b 100644
--- a/supabase/schema.sql
+++ b/supabase/schema.sql
@@ -27586,6 +27586,34 @@ create unique index if not exists citation_url_daily_key
on public.citation_url_daily (brand_id, day, model_used, platform, region, topic_id, url_id)
nulls not distinct;
+-- ─────────────────────────────────────────────────────────────────────────
+-- migrations/00201_gsc_pages_backfill_from.sql
+-- ─────────────────────────────────────────────────────────────────────────
+-- ============================================================================
+-- 00201: the Search Console page backfill resumes where it stopped
+-- ============================================================================
+-- Follow-up from #1177's review (AT-008). The page sync read the property's
+-- 190 days again on every run until one got through them all
+-- (gsc_pages_backfilled_property, 00181), so a window that kept failing made
+-- every daily run read the whole history again. It now reads a week at a
+-- time (gsc-sync.js), and keeps how far back the backfill has read:
+--
+-- * gsc_pages_backfilled_property: the property the brand's page rows
+-- belong to. Set once its last 28 days are read, and when it names
+-- another property, that one's page rows are dropped first.
+-- * gsc_pages_backfill_from: the oldest day the property's backfill has
+-- read so far, while it hasn't reached its 190 days; null once it has.
+-- A sync cut short resumes the day after.
+--
+-- A brand backfilled before this column (property set, this null) reads as
+-- done, as it was.
+
+alter table public.brands
+ add column if not exists gsc_pages_backfill_from date;
+
+comment on column public.brands.gsc_pages_backfill_from is
+ 'AI Traffic (AT-008): the oldest day the Search Console page backfill for gsc_pages_backfilled_property has read while it hasn''t reached its 190 days; null once it has (gsc-sync.js).';
+
-- ─────────────────────────────────────────────────────────────────────────
-- migrations/00202_getting_started_steps_update_using.sql
-- ─────────────────────────────────────────────────────────────────────────
diff --git a/web/src/types/supabase.ts b/web/src/types/supabase.ts
index 0be1efd18..377f1d408 100644
--- a/web/src/types/supabase.ts
+++ b/web/src/types/supabase.ts
@@ -610,6 +610,7 @@ export type Database = {
created_at: string
description: string | null
ga_property_id: string | null
+ gsc_pages_backfill_from: string | null
gsc_pages_backfilled_property: string | null
gsc_property: string | null
id: string
@@ -632,6 +633,7 @@ export type Database = {
created_at?: string
description?: string | null
ga_property_id?: string | null
+ gsc_pages_backfill_from?: string | null
gsc_pages_backfilled_property?: string | null
gsc_property?: string | null
id?: string
@@ -654,6 +656,7 @@ export type Database = {
created_at?: string
description?: string | null
ga_property_id?: string | null
+ gsc_pages_backfill_from?: string | null
gsc_pages_backfilled_property?: string | null
gsc_property?: string | null
id?: string