Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M5-002): gsc module: service + db client + trpc + /gsc callback + gsc queue + gscSync cron into core

ralphb49c55f2ec62
diff --git a/apps/api/src/controllers/gsc-oauth-callback.controller.ts b/apps/api/src/controllers/gsc-oauth-callback.controller.ts
index 24cbb502f..0b50ecd58 100644
--- a/apps/api/src/controllers/gsc-oauth-callback.controller.ts
+++ b/apps/api/src/controllers/gsc-oauth-callback.controller.ts
@@ -1,5 +1,9 @@
-import { googleGsc } from '@openpanel/core';
-import { db, encrypt } from '@openpanel/db';
+// Dissolved into @openpanel/core's gsc module (M5-002): state verification,
+// the Google token exchange and the gscConnection upsert moved to
+// packages/core/src/modules/gsc/gsc.service.ts#completeGscOAuthCallback.
+// This controller stays (DELEGATE PATTERN) — it keeps Fastify's signed-cookie
+// verification and the redirect shape, and delegates everything else.
+import { completeGscOAuthCallback } from '@openpanel/core';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { z } from 'zod';
import { LogError } from '@/utils/errors';
@@ -83,65 +87,30 @@ export async function gscGoogleCallback(
const codeVerifierStr = codeVerifierResult?.value;
const projectIdStr = projectIdResult?.value;
- if (state !== stateStr) {
- throw new LogError('GSC OAuth state mismatch', {
- hasState: true,
- hasStoredState: true,
- stateMismatch: true,
- });
- }
-
- const tokens = await googleGsc.validateAuthorizationCode(
- code,
- codeVerifierStr
- );
-
- const accessToken = tokens.accessToken();
- const refreshToken = tokens.hasRefreshToken()
- ? tokens.refreshToken()
- : null;
- const accessTokenExpiresAt = tokens.accessTokenExpiresAt();
-
- if (!refreshToken) {
- throw new LogError('No refresh token returned from Google GSC OAuth');
- }
-
- const project = await db.project.findUnique({
- where: { id: projectIdStr },
- select: { id: true, organizationId: true },
- });
-
- if (!project) {
- throw new LogError('Project not found for GSC connection', {
+ let organizationId: string;
+ try {
+ ({ organizationId } = await completeGscOAuthCallback({
+ code,
+ state,
+ storedState: stateStr,
+ codeVerifier: codeVerifierStr,
projectId: projectIdStr,
- });
+ }));
+ } catch (error) {
+ // Re-wrapped as LogError so redirectWithError shows the same
+ // per-failure message it always has, sourced now from
+ // gsc.service.ts#completeGscOAuthCallback instead of inline.
+ const message = error instanceof Error ? error.message : String(error);
+ throw new LogError(message, { projectId: projectIdStr });
}
- await db.gscConnection.upsert({
- where: { projectId: projectIdStr },
- create: {
- projectId: projectIdStr,
- accessToken: encrypt(accessToken),
- refreshToken: encrypt(refreshToken),
- accessTokenExpiresAt,
- siteUrl: '',
- },
- update: {
- accessToken: encrypt(accessToken),
- refreshToken: encrypt(refreshToken),
- accessTokenExpiresAt,
- lastSyncStatus: null,
- lastSyncError: null,
- },
- });
-
reply.clearCookie('gsc_oauth_state');
reply.clearCookie('gsc_code_verifier');
reply.clearCookie('gsc_project_id');
const dashboardUrl =
process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL!;
- const redirectUrl = `${dashboardUrl}/${project.organizationId}/${projectIdStr}/settings/gsc`;
+ const redirectUrl = `${dashboardUrl}/${organizationId}/${projectIdStr}/settings/gsc`;
return reply.redirect(redirectUrl);
} catch (error) {
req.log.error({ err: error }, 'GSC OAuth callback error');
diff --git a/apps/worker/src/jobs/gsc.ts b/apps/worker/src/jobs/gsc.ts
index 37c9f7070..9480be5db 100644
--- a/apps/worker/src/jobs/gsc.ts
+++ b/apps/worker/src/jobs/gsc.ts
@@ -1,145 +1,36 @@
-import { db, syncGscData } from '@openpanel/db';
+// Dissolved into @openpanel/core's gsc module (M5-002): the sync/backfill job
+// bodies and their status-update bookkeeping moved to
+// packages/core/src/modules/gsc/gsc.service.ts. This file stays (DELEGATE
+// PATTERN) — it is the BullMQ job body V1's worker registers
+// (boot-workers.ts) and the `cron.ts` dispatcher's `gscSync` case, both thin
+// wrappers around the core functions.
+import {
+ listGscConnectionsForSync,
+ runGscProjectBackfill,
+ runGscProjectSync,
+} from '@openpanel/core';
import type { GscQueuePayload } from '@openpanel/queue';
import { gscQueue } from '@openpanel/queue';
import type { Job } from 'bullmq';
import { logger } from '../utils/logger';
-const BACKFILL_MONTHS = 6;
-const CHUNK_DAYS = 14;
-
export function gscJob(job: Job<GscQueuePayload>) {
switch (job.data.type) {
case 'gscProjectSync':
- return gscProjectSyncJob(job.data.payload.projectId);
+ return runGscProjectSync(job.data.payload.projectId);
case 'gscProjectBackfill':
- return gscProjectBackfillJob(job.data.payload.projectId);
+ return runGscProjectBackfill(job.data.payload.projectId);
default:
throw new Error('Unknown GSC job type');
}
}
-async function gscProjectSyncJob(projectId: string) {
- const conn = await db.gscConnection.findUnique({ where: { projectId } });
- if (!conn?.siteUrl) {
- logger.warn({ projectId }, 'GSC sync skipped: no connection or siteUrl');
- return;
- }
-
- try {
- // Sync rolling 3-day window (GSC data can arrive late)
- const endDate = new Date();
- endDate.setDate(endDate.getDate() - 1); // yesterday
- const startDate = new Date(endDate);
- startDate.setDate(startDate.getDate() - 2); // 3 days total
-
- await syncGscData(projectId, startDate, endDate);
-
- await db.gscConnection.update({
- where: { projectId },
- data: {
- lastSyncedAt: new Date(),
- lastSyncStatus: 'success',
- lastSyncError: null,
- },
- });
- logger.info({ projectId }, 'GSC sync completed');
- } catch (error) {
- const message = error instanceof Error ? error.message : String(error);
- await db.gscConnection.update({
- where: { projectId },
- data: {
- lastSyncedAt: new Date(),
- lastSyncStatus: 'error',
- lastSyncError: message,
- },
- });
- logger.error({ err: error, projectId }, 'GSC sync failed');
- throw error;
- }
-}
-
-async function gscProjectBackfillJob(projectId: string) {
- const conn = await db.gscConnection.findUnique({ where: { projectId } });
- if (!conn?.siteUrl) {
- logger.warn(
- { projectId },
- 'GSC backfill skipped: no connection or siteUrl',
- );
- return;
- }
-
- await db.gscConnection.update({
- where: { projectId },
- data: { backfillStatus: 'running' },
- });
-
- try {
- const endDate = new Date();
- endDate.setDate(endDate.getDate() - 1); // yesterday
-
- const startDate = new Date(endDate);
- startDate.setMonth(startDate.getMonth() - BACKFILL_MONTHS);
-
- // Process in chunks to avoid timeouts and respect API limits
- let chunkEnd = new Date(endDate);
- while (chunkEnd > startDate) {
- const chunkStart = new Date(chunkEnd);
- chunkStart.setDate(chunkStart.getDate() - CHUNK_DAYS + 1);
- if (chunkStart < startDate) {
- chunkStart.setTime(startDate.getTime());
- }
-
- logger.info(
- {
- projectId,
- from: chunkStart.toISOString().slice(0, 10),
- to: chunkEnd.toISOString().slice(0, 10),
- },
- 'GSC backfill chunk',
- );
-
- await syncGscData(projectId, chunkStart, chunkEnd);
-
- chunkEnd = new Date(chunkStart);
- chunkEnd.setDate(chunkEnd.getDate() - 1);
- }
-
- await db.gscConnection.update({
- where: { projectId },
- data: {
- backfillStatus: 'completed',
- lastSyncedAt: new Date(),
- lastSyncStatus: 'success',
- lastSyncError: null,
- },
- });
- logger.info({ projectId }, 'GSC backfill completed');
- } catch (error) {
- const message = error instanceof Error ? error.message : String(error);
- await db.gscConnection.update({
- where: { projectId },
- data: {
- backfillStatus: 'failed',
- lastSyncStatus: 'error',
- lastSyncError: message,
- },
- });
- logger.error({ err: error, projectId }, 'GSC backfill failed');
- throw error;
- }
-}
-
export async function gscSyncAllJob() {
- const connections = await db.gscConnection.findMany({
- where: {
- siteUrl: { not: '' },
- },
- select: { projectId: true },
- });
+ const connections = await listGscConnectionsForSync();
logger.info(
{ count: connections.length },
- 'GSC nightly sync: enqueuing projects',
+ 'GSC nightly sync: enqueuing projects'
);
for (const conn of connections) {
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index dd4992e9b..e8819ced7 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -145,6 +145,49 @@ export {
verifyPasswordHash,
verifyTotpCode,
} from './modules/auth/auth.service';
+// Dissolved from @openpanel/db's src/gsc.ts + services/gsc.service.ts
+// (M5-002) — apps/worker's gsc job file, apps/api's gsc OAuth callback
+// controller and packages/trpc's gsc router call these directly, the same
+// way V1 reaches every other dissolved service here. packages/db/src/gsc.ts
+// and packages/db/src/services/gsc.service.ts re-export the subset MCP's gsc
+// tools and the assistant's SEO tools still reach via `@openpanel/db`.
+export type {
+ GscCannibalizedQuery,
+ GscConnectionSummary,
+ GscDateRangeInput,
+ GscOAuthCallbackInput,
+ GscOAuthCallbackResult,
+ GscQueryOpportunity,
+ GscSite,
+} from './modules/gsc/gsc.service';
+export {
+ completeGscOAuthCallback,
+ disconnectGscConnection,
+ getGscAiEngines,
+ getGscCannibalization,
+ getGscConnection,
+ getGscOverview,
+ getGscPageDetails,
+ getGscPages,
+ getGscPreviousOverview,
+ getGscQueries,
+ getGscQueryDetails,
+ getGscSearchEngines,
+ gscGetCannibalizationCore,
+ gscGetOverviewCore,
+ gscGetPageDetailsCore,
+ gscGetQueryDetailsCore,
+ gscGetQueryOpportunitiesCore,
+ gscGetTopPagesCore,
+ gscGetTopQueriesCore,
+ listGscConnectionsForSync,
+ listGscSites,
+ resolveGscDateRange,
+ runGscProjectBackfill,
+ runGscProjectSync,
+ selectGscSite,
+ syncGscData,
+} from './modules/gsc/gsc.service';
// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service
// (M5-001) — apps/worker's insight job files and packages/trpc's insight
// router call these directly, the same way V1 reaches every other dissolved
diff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.ts
index 5c3f79545..f0c8be344 100644
--- a/packages/core/src/jobs.registry.test.ts
+++ b/packages/core/src/jobs.registry.test.ts
@@ -82,7 +82,7 @@ test('the insights queue carries the insight module job', () => {
// job's `type` straight onto its name, so these three ids double as job names.
test('the cron queue carries the insight module cron fragment', () => {
expect(Object.keys(queues.cron.jobs).sort()).toEqual(
- ['insightCleanup', 'insightsDaily', 'weeklyDigest'].sort()
+ ['gscSync', 'insightCleanup', 'insightsDaily', 'weeklyDigest'].sort()
);
for (const name of ['insightsDaily', 'insightCleanup', 'weeklyDigest']) {
expect(
@@ -90,3 +90,24 @@ test('the cron queue carries the insight module cron fragment', () => {
).toMatchObject({ queue: 'cron', name });
}
});
+
+// M5-002: the gsc module's own queue job, plus its cron fan-out fragment.
+// legacyCompat.gsc already discriminates on these exact names (jobs/compat.ts),
+// so this pins the two in agreement.
+test('the gsc queue carries the gsc module jobs', () => {
+ expect(Object.keys(queues.gsc.jobs).sort()).toEqual(
+ ['gscProjectBackfill', 'gscProjectSync'].sort()
+ );
+ for (const name of ['gscProjectSync', 'gscProjectBackfill']) {
+ expect(queues.gsc.jobs[name as keyof typeof queues.gsc.jobs]).toMatchObject(
+ { queue: 'gsc', name }
+ );
+ }
+});
+
+test('the cron queue carries the gsc module cron fragment', () => {
+ expect(queues.cron.jobs.gscSync).toMatchObject({
+ queue: 'cron',
+ name: 'gscSync',
+ });
+});
diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.ts
index 5e3c41877..79f898545 100644
--- a/packages/core/src/jobs.registry.ts
+++ b/packages/core/src/jobs.registry.ts
@@ -14,6 +14,7 @@ import { legacyCompat } from './jobs/compat';
import type { Producers } from './jobs/define';
import { defineQueue } from './jobs/define';
import type { ProducerHandle } from './jobs/producers';
+import { gscCronJobs, gscQueueJobs } from './modules/gsc/gsc.jobs';
import {
insightCronJobs,
insightQueueJobs,
@@ -53,6 +54,7 @@ const cron = defineQueue('cron', {
// Every module's cron fragment spreads in here (ADR-005: "no cron module").
jobs: {
...insightCronJobs,
+ ...gscCronJobs,
},
});
@@ -83,7 +85,9 @@ const gsc = defineQueue('gsc', {
defaults: { removeOnComplete: 50, removeOnFail: 100 },
worker: { concurrency: CONCURRENCY.gsc },
compat: legacyCompat.gsc,
- jobs: {},
+ jobs: {
+ ...gscQueueJobs,
+ },
});
const cohortCompute = defineQueue('cohortCompute', {
diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.ts
index 81c291543..b94cfc668 100644
--- a/packages/core/src/jobs/schedulers.ts
+++ b/packages/core/src/jobs/schedulers.ts
@@ -5,6 +5,7 @@
// locking of our own needed.
import type { Logger } from '../logger';
+import { gscCronSchedules } from '../modules/gsc/gsc.jobs';
import { insightCronSchedules } from '../modules/insight/insight.jobs';
import { wrap } from './envelope';
@@ -63,7 +64,9 @@ export const CRON_SCHEDULES: readonly SchedulerDefinition[] = [
{ id: 'flushReplay', schedule: { every: 10_000 } },
{ id: 'flushGroups', schedule: { every: 10_000 } },
{ id: 'onboarding', schedule: { pattern: '0 * * * *' } },
- { id: 'gscSync', schedule: { pattern: '0 3 * * *' } },
+ // gscSync — owned by the gsc module, declared next to its jobs
+ // (modules/gsc/gsc.jobs.ts).
+ ...gscCronSchedules,
{ id: 'cohortRefresh', schedule: { pattern: '*/30 * * * *' } },
{ id: 'sessionReaper', schedule: { every: 5 * MINUTE_MS } },
// Daily 04:00 UTC — backstop for cleanup leaks.
diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.ts
new file mode 100644
index 000000000..36ceeaccb
--- /dev/null
+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts
@@ -0,0 +1,151 @@
+// Job/cron-fragment wiring — no ClickHouse/Postgres touched. The service
+// methods themselves are exercised in gsc.service.test.ts.
+
+import { expect, test } from 'bun:test';
+import type { JobCtx } from '../../context';
+import { createRecordingProducers } from '../../jobs/testing';
+import { queues } from '../../jobs.registry';
+import type { Logger } from '../../logger';
+import type { Services } from '../../services';
+import { gscCronJobs, gscCronSchedules, gscQueueJobs } from './gsc.jobs';
+import type { GscService } from './gsc.service';
+
+function stubLogger(): Logger {
+ const noop = () => undefined;
+ const logger: Logger = {
+ fatal: noop,
+ error: noop,
+ warn: noop,
+ info: noop,
+ debug: noop,
+ trace: noop,
+ child: () => logger,
+ };
+ return logger;
+}
+
+function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {
+ const services: Services = {
+ auth: {} as Services['auth'],
+ insight: {} as Services['insight'],
+ gsc: gsc as GscService,
+ };
+ return {
+ db: {},
+ ch: {},
+ redis: {},
+ clients: {},
+ buffers: {},
+ logger: stubLogger(),
+ queues: createRecordingProducers(queues).queues,
+ services,
+ requestId: 'req_1',
+ job: { id: 'job_1', attempt: 0, queue: 'gsc', name: jobName },
+ };
+}
+
+test('the gsc jobs registry declares gscProjectSync and gscProjectBackfill on the gsc queue', () => {
+ expect(queues.gsc.jobs.gscProjectSync).toMatchObject({
+ queue: 'gsc',
+ name: 'gscProjectSync',
+ });
+ expect(queues.gsc.jobs.gscProjectBackfill).toMatchObject({
+ queue: 'gsc',
+ name: 'gscProjectBackfill',
+ });
+});
+
+test('the gsc cron fragment is spread into the cron queue', () => {
+ expect(queues.cron.jobs.gscSync).toBeDefined();
+});
+
+test('gscProjectSync validates its payload', () => {
+ expect(() => gscQueueJobs.gscProjectSync.payload.parse({})).toThrow();
+ expect(
+ gscQueueJobs.gscProjectSync.payload.parse({ projectId: 'p1' })
+ ).toEqual({ projectId: 'p1' });
+});
+
+test('gscProjectBackfill validates its payload', () => {
+ expect(() => gscQueueJobs.gscProjectBackfill.payload.parse({})).toThrow();
+ expect(
+ gscQueueJobs.gscProjectBackfill.payload.parse({ projectId: 'p1' })
+ ).toEqual({ projectId: 'p1' });
+});
+
+test('gscProjectSync delegates to runProjectSync', async () => {
+ const calls: unknown[] = [];
+ const ctx = stubJobCtx({
+ runProjectSync: async (projectId) => {
+ calls.push(projectId);
+ },
+ });
+
+ await gscQueueJobs.gscProjectSync.handler({
+ payload: { projectId: 'p1' },
+ ctx,
+ });
+
+ expect(calls).toEqual(['p1']);
+});
+
+test('gscProjectBackfill delegates to runProjectBackfill', async () => {
+ const calls: unknown[] = [];
+ const ctx = stubJobCtx({
+ runProjectBackfill: async (projectId) => {
+ calls.push(projectId);
+ },
+ });
+
+ await gscQueueJobs.gscProjectBackfill.handler({
+ payload: { projectId: 'p1' },
+ ctx,
+ });
+
+ expect(calls).toEqual(['p1']);
+});
+
+test('gscSync fans out one gscProjectSync enqueue per connected project', async () => {
+ const producers = createRecordingProducers(queues);
+ const ctx: JobCtx = {
+ db: {},
+ ch: {},
+ redis: {},
+ clients: {},
+ buffers: {},
+ logger: stubLogger(),
+ queues: producers.queues,
+ services: {
+ auth: {} as Services['auth'],
+ insight: {} as Services['insight'],
+ gsc: {
+ listConnectionsForSync: async () => [
+ { projectId: 'p1' },
+ { projectId: 'p2' },
+ ],
+ } as GscService,
+ },
+ requestId: 'req_1',
+ job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'gscSync' },
+ };
+
+ await gscCronJobs.gscSync.handler({ payload: null, ctx });
+
+ expect(producers.recorded).toHaveLength(2);
+ expect(producers.recorded.map((r) => r.job)).toEqual([
+ 'gscProjectSync',
+ 'gscProjectSync',
+ ]);
+ expect(producers.recorded.map((r) => r.payload)).toEqual([
+ { projectId: 'p1' },
+ { projectId: 'p2' },
+ ]);
+});
+
+// Byte-identity with the id/cadence schedulers.test.ts's golden snapshot pins
+// (apps/worker/src/boot-cron.ts).
+test('the gsc cron fragment carries V1 id and cadence unchanged', () => {
+ expect(gscCronSchedules).toEqual([
+ { id: 'gscSync', schedule: { pattern: '0 3 * * *' } },
+ ]);
+});
diff --git a/packages/core/src/modules/gsc/gsc.jobs.ts b/packages/core/src/modules/gsc/gsc.jobs.ts
new file mode 100644
index 000000000..bb7ee11cb
--- /dev/null
+++ b/packages/core/src/modules/gsc/gsc.jobs.ts
@@ -0,0 +1,52 @@
+// Ported from apps/worker/src/jobs/gsc.ts + boot-cron.ts's gscSync schedule
+// (M5-002).
+//
+// `gscProjectSync` / `gscProjectBackfill` are this module's own queue (`gsc`
+// in the registry — ADR-005's registry key). `gscSync` is a cron fragment:
+// declared here, spread into the ONE `cron` queue by jobs.registry.ts and
+// into `CRON_SCHEDULES` by jobs/schedulers.ts. Scheduler id and cadence are
+// V1's, unchanged (apps/worker/src/boot-cron.ts).
+
+import { z } from 'zod';
+import { defineJob } from '../../jobs/define';
+import type { SchedulerDefinition } from '../../jobs/schedulers';
+
+const gscProjectPayload = z.object({ projectId: z.string() });
+
+/** The `gsc` queue's own jobs. */
+export const gscQueueJobs = {
+ gscProjectSync: defineJob({
+ payload: gscProjectPayload,
+ handler: async ({ payload, ctx }) => {
+ await ctx.services.gsc.runProjectSync(payload.projectId);
+ },
+ }),
+
+ gscProjectBackfill: defineJob({
+ payload: gscProjectPayload,
+ handler: async ({ payload, ctx }) => {
+ await ctx.services.gsc.runProjectBackfill(payload.projectId);
+ },
+ }),
+};
+
+/** This module's fragment of the `cron` queue's jobs. */
+export const gscCronJobs = {
+ // Fans out one `gscProjectSync` job per project with a connected GSC site.
+ // Matches V1 (apps/worker/src/jobs/gsc.ts's gscSyncAllJob).
+ gscSync: defineJob({
+ payload: z.null(),
+ handler: async ({ ctx }) => {
+ const connections = await ctx.services.gsc.listConnectionsForSync();
+
+ for (const { projectId } of connections) {
+ await ctx.queues.gsc.gscProjectSync.add({ projectId });
+ }
+ },
+ }),
+};
+
+/** This module's fragment of `CRON_SCHEDULES` — id and cadence unchanged. */
+export const gscCronSchedules: readonly SchedulerDefinition[] = [
+ { id: 'gscSync', schedule: { pattern: '0 3 * * *' } },
+];
diff --git a/packages/core/src/modules/gsc/gsc.routes.ts b/packages/core/src/modules/gsc/gsc.routes.ts
new file mode 100644
index 000000000..567c9331a
--- /dev/null
+++ b/packages/core/src/modules/gsc/gsc.routes.ts
@@ -0,0 +1,70 @@
+// The GSC OAuth callback (M5-002). V1's Fastify controller
+// (apps/api/src/controllers/gsc-oauth-callback.controller.ts) stays the LIVE
+// route (DELEGATE PATTERN) and delegates its token-exchange/upsert logic to
+// gsc.service.ts's `completeGscOAuthCallback` — the same function this route
+// calls. This route is not yet reachable: main.ts does not mount
+// `dashboardRoutes` until a real `AppDeps` exists (P3/P4/P8).
+//
+// NAMED GAP: per-route cookie signing is not wired yet (http/context.ts:147
+// — "app.ts lists the signed names (P3)"), so `ctx.cookies.get()` here reads
+// the raw, unsigned cookie value. V1's Fastify controller still verifies the
+// signature; this route only re-checks the state match, and should not be
+// treated as a signature-verified equivalent until that wiring lands.
+
+import { redirect } from 'elysia';
+import { z } from 'zod';
+import { defineRoutes } from '../../http/define';
+import { completeGscOAuthCallback } from './gsc.service';
+
+const gscCallbackQuery = z.object({
+ code: z.string(),
+ state: z.string(),
+});
+
+function dashboardUrl(): string {
+ return (
+ process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL || ''
+ );
+}
+
+function loginErrorRedirect(message: string) {
+ const url = new URL('/login', dashboardUrl() || 'http://localhost');
+ url.searchParams.set('error', message);
+ return redirect(url.toString());
+}
+
+export const gscRoutes = defineRoutes((app) =>
+ app.get(
+ '/gsc/callback',
+ async ({ query, ctx }) => {
+ const storedState = ctx.cookies.get('gsc_oauth_state');
+ const codeVerifier = ctx.cookies.get('gsc_code_verifier');
+ const projectId = ctx.cookies.get('gsc_project_id');
+
+ if (!(storedState && codeVerifier && projectId)) {
+ return loginErrorRedirect('Missing GSC OAuth cookies');
+ }
+
+ try {
+ const { organizationId } = await completeGscOAuthCallback({
+ code: query.code,
+ state: query.state,
+ storedState,
+ codeVerifier,
+ projectId,
+ });
+ return redirect(
+ `${dashboardUrl()}/${organizationId}/${projectId}/settings/gsc`
+ );
+ } catch (error) {
+ ctx.logger.error({ err: error }, 'GSC OAuth callback error');
+ const message =
+ error instanceof Error
+ ? error.message
+ : 'Failed to connect Google Search Console';
+ return loginErrorRedirect(message);
+ }
+ },
+ { query: gscCallbackQuery, detail: { hide: true } }
+ )
+);
diff --git a/packages/core/src/modules/gsc/gsc.rpc.test.ts b/packages/core/src/modules/gsc/gsc.rpc.test.ts
new file mode 100644
index 000000000..9df8a62e5
--- /dev/null
+++ b/packages/core/src/modules/gsc/gsc.rpc.test.ts
@@ -0,0 +1,73 @@
+// Only the "is anyone logged in" boundary is exercised here — no database.
+// The access-check + business logic ride on @openpanel/db (lazy-loaded, see
+// gsc.service.ts's header) and ctx.services.gsc; wiring this router
+// end-to-end against a real Postgres is P6's (protectedProcedure) job, not
+// this one's — see gsc.rpc.ts's header.
+
+import { expect, test } from 'bun:test';
+import { stubHttpCtx } from '../../../test/rpc-fixtures';
+import { makeTrpcContext } from '../../rpc/base';
+import type { CookieOptions } from '../../shared/cookie';
+import { gscRouter } from './gsc.rpc';
+
+const COOKIE_OPTIONS: CookieOptions = {
+ domain: '.openpanel.dev',
+ secure: true,
+ sameSite: 'lax',
+ httpOnly: true,
+ path: '/',
+};
+
+// EMPTY_SESSION's shape (packages/db/src/services/auth-session.service.ts) —
+// `TrpcContext.session` is never literally `null`, only its `userId` is.
+const EMPTY_SESSION = { session: null, user: null, userId: null };
+
+async function anonCaller() {
+ const { ctx } = stubHttpCtx({}, EMPTY_SESSION);
+ const trpcCtx = await makeTrpcContext(ctx, new Headers(), {
+ cookieOptions: COOKIE_OPTIONS,
+ });
+ return gscRouter.createCaller(trpcCtx);
+}
+
+test('getConnection rejects an unauthenticated caller before touching a project', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.getConnection({ projectId: 'proj_1' })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('initiateOAuth rejects an unauthenticated caller before minting state', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.initiateOAuth({ projectId: 'proj_1' })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('getSites rejects an unauthenticated caller before calling Google', async () => {
+ const caller = await anonCaller();
+ await expect(caller.getSites({ projectId: 'proj_1' })).rejects.toMatchObject({
+ code: 'UNAUTHORIZED',
+ });
+});
+
+test('selectSite rejects an unauthenticated caller before writing a connection', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.selectSite({ projectId: 'proj_1', siteUrl: 'https://example.com' })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('disconnect rejects an unauthenticated caller before deleting a connection', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.disconnect({ projectId: 'proj_1' })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
+
+test('getOverview rejects an unauthenticated caller before resolving a date range', async () => {
+ const caller = await anonCaller();
+ await expect(
+ caller.getOverview({ projectId: 'proj_1', range: '30d' })
+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });
+});
diff --git a/packages/core/src/modules/gsc/gsc.rpc.ts b/packages/core/src/modules/gsc/gsc.rpc.ts
new file mode 100644
index 000000000..27011ee17
--- /dev/null
+++ b/packages/core/src/modules/gsc/gsc.rpc.ts
@@ -0,0 +1,274 @@
+// Ported from packages/trpc/src/routers/gsc.ts (M5-002).
+//
+// V1's `protectedProcedure` — the logger/session-scope/rate-limit middleware
+// stack — lands in core with auth (rpc/base.ts: "those need the resolved
+// Session shape and the access rules, so they land with auth (P6)"). Until
+// then this router does its own minimal "is anyone logged in" check inline,
+// exactly like V1's `enforceUserIsAuthed`. V1 keeps serving the live route
+// through packages/trpc's own `protectedProcedure` (full stack included) and
+// delegates its handler bodies to `ctx.services.gsc` (DELEGATE PATTERN),
+// so nothing here is a live regression.
+//
+// The per-project access ladder itself IS shared: `./src/access.ts` binds
+// core's shared/access.ts ladder to @openpanel/db's real lookups, the same
+// way packages/trpc/src/access.ts does for V1.
+
+import { zRange, zTimeInterval } from '@openpanel/validation';
+import { z } from 'zod';
+import { createTRPCRouter, procedure } from '../../rpc/base';
+import { TRPCAccessError } from '../../rpc/errors';
+import { Arctic, googleGsc } from '../auth/auth.service';
+
+// Lazy, deliberately: ./src/access reaches @openpanel/db's real lookups,
+// which construct a real pino logger — with a transport worker thread — at
+// import time. See gsc.service.ts's header for the full reasoning.
+function loadAccessChecks() {
+ return import('./src/access');
+}
+
+const OAUTH_COOKIE_MAX_AGE_SECONDS = 60 * 10;
+
+const zGscDateInput = z.object({
+ projectId: z.string(),
+ range: zRange,
+ interval: zTimeInterval.optional().default('day'),
+ startDate: z.string().nullish(),
+ endDate: z.string().nullish(),
+});
+
+const GSC_OVERVIEW_INTERVALS = ['day', 'week', 'month'] as const;
+
+function toOverviewInterval(
+ interval: string
+): (typeof GSC_OVERVIEW_INTERVALS)[number] {
+ return (GSC_OVERVIEW_INTERVALS as readonly string[]).includes(interval)
+ ? (interval as (typeof GSC_OVERVIEW_INTERVALS)[number])
+ : 'day';
+}
+
+function requireLogin(userId: string | null | undefined): string {
+ if (!userId) {
+ throw new TRPCAccessError('Not authenticated');
+ }
+ return userId;
+}
+
+async function requireRead(userId: string, projectId: string) {
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({ userId, projectId, level: 'read' });
+}
+
+async function requireWrite(userId: string, projectId: string) {
+ const { requireProjectAccess } = await loadAccessChecks();
+ await requireProjectAccess({ userId, projectId, level: 'write' });
+}
+
+export const gscRouter = createTRPCRouter({
+ getConnection: procedure
+ .input(z.object({ projectId: z.string() }))
+ .query(async ({ input: { projectId }, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, projectId);
+ return ctx.services.gsc.getConnection(projectId);
+ }),
+
+ initiateOAuth: procedure
+ .input(z.object({ projectId: z.string() }))
+ .mutation(async ({ input: { projectId }, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireWrite(userId, projectId);
+
+ const state = Arctic.generateState();
+ const codeVerifier = Arctic.generateCodeVerifier();
+ const url = googleGsc.createAuthorizationURL(state, codeVerifier, [
+ 'https://www.googleapis.com/auth/webmasters.readonly',
+ ]);
+ url.searchParams.set('access_type', 'offline');
+ url.searchParams.set('prompt', 'consent');
+
+ const cookieOpts = { maxAge: OAUTH_COOKIE_MAX_AGE_SECONDS, signed: true };
+ ctx.setCookie('gsc_oauth_state', state, cookieOpts);
+ ctx.setCookie('gsc_code_verifier', codeVerifier, cookieOpts);
+ ctx.setCookie('gsc_project_id', projectId, cookieOpts);
+
+ return { url: url.toString() };
+ }),
+
+ getSites: procedure
+ .input(z.object({ projectId: z.string() }))
+ .query(async ({ input: { projectId }, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, projectId);
+ return ctx.services.gsc.listSites(projectId);
+ }),
+
+ selectSite: procedure
+ .input(z.object({ projectId: z.string(), siteUrl: z.string() }))
+ .mutation(async ({ input: { projectId, siteUrl }, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireWrite(userId, projectId);
+
+ await ctx.services.gsc.selectSite(projectId, siteUrl);
+ return { ok: true };
+ }),
+
+ disconnect: procedure
+ .input(z.object({ projectId: z.string() }))
+ .mutation(async ({ input: { projectId }, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireWrite(userId, projectId);
+
+ await ctx.services.gsc.disconnect(projectId);
+ return { ok: true };
+ }),
+
+ getOverview: procedure.input(zGscDateInput).query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, input.projectId);
+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(
+ input.projectId,
+ input
+ );
+ return ctx.services.gsc.getOverview(
+ input.projectId,
+ startDate,
+ endDate,
+ toOverviewInterval(input.interval)
+ );
+ }),
+
+ getPages: procedure
+ .input(
+ zGscDateInput.extend({
+ limit: z.number().min(1).max(10_000).optional().default(100),
+ })
+ )
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, input.projectId);
+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(
+ input.projectId,
+ input
+ );
+ return ctx.services.gsc.getPages(
+ input.projectId,
+ startDate,
+ endDate,
+ input.limit
+ );
+ }),
+
+ getPageDetails: procedure
+ .input(zGscDateInput.extend({ page: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, input.projectId);
+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(
+ input.projectId,
+ input
+ );
+ return ctx.services.gsc.getPageDetails(
+ input.projectId,
+ input.page,
+ startDate,
+ endDate
+ );
+ }),
+
+ getQueryDetails: procedure
+ .input(zGscDateInput.extend({ query: z.string() }))
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, input.projectId);
+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(
+ input.projectId,
+ input
+ );
+ return ctx.services.gsc.getQueryDetails(
+ input.projectId,
+ input.query,
+ startDate,
+ endDate
+ );
+ }),
+
+ getQueries: procedure
+ .input(
+ zGscDateInput.extend({
+ limit: z.number().min(1).max(1000).optional().default(100),
+ })
+ )
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, input.projectId);
+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(
+ input.projectId,
+ input
+ );
+ return ctx.services.gsc.getQueries(
+ input.projectId,
+ startDate,
+ endDate,
+ input.limit
+ );
+ }),
+
+ getSearchEngines: procedure
+ .input(zGscDateInput)
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, input.projectId);
+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(
+ input.projectId,
+ input
+ );
+ return ctx.services.gsc.getSearchEngines(
+ input.projectId,
+ startDate,
+ endDate
+ );
+ }),
+
+ getAiEngines: procedure.input(zGscDateInput).query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, input.projectId);
+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(
+ input.projectId,
+ input
+ );
+ return ctx.services.gsc.getAiEngines(input.projectId, startDate, endDate);
+ }),
+
+ getPreviousOverview: procedure
+ .input(zGscDateInput)
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, input.projectId);
+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(
+ input.projectId,
+ input
+ );
+ return ctx.services.gsc.getPreviousOverview(
+ input.projectId,
+ startDate,
+ endDate,
+ toOverviewInterval(input.interval)
+ );
+ }),
+
+ getCannibalization: procedure
+ .input(zGscDateInput)
+ .query(async ({ input, ctx }) => {
+ const userId = requireLogin(ctx.session.userId);
+ await requireRead(userId, input.projectId);
+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(
+ input.projectId,
+ input
+ );
+ return ctx.services.gsc.getCannibalization(
+ input.projectId,
+ startDate,
+ endDate
+ );
+ }),
+});
diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.ts
new file mode 100644
index 000000000..3b775027d
--- /dev/null
+++ b/packages/core/src/modules/gsc/gsc.service.test.ts
@@ -0,0 +1,406 @@
+// gsc.service.ts's db/ch access is lazy (`await import(...)` inside each
+// function — see the file's header), which is exactly what makes
+// `mock.module` work here with no import-time side effects to race: every
+// mock below is registered before the subject's first call, not before its
+// (side-effect-free) import.
+
+import { beforeAll, expect, mock, test } from 'bun:test';
+
+process.env.ENCRYPTION_KEY = 'a'.repeat(64);
+
+interface FakeGscConnection {
+ projectId: string;
+ siteUrl: string;
+ accessToken: string;
+ refreshToken: string;
+ accessTokenExpiresAt: Date | null;
+ lastSyncStatus: string | null;
+ lastSyncError: string | null;
+ backfillStatus: string | null;
+}
+
+const gscConnectionStore = new Map<string, FakeGscConnection>();
+
+const gscConnection = {
+ findUnique: mock(
+ async ({ where: { projectId } }: { where: { projectId: string } }) =>
+ gscConnectionStore.get(projectId) ?? null
+ ),
+ findUniqueOrThrow: mock(
+ async ({ where: { projectId } }: { where: { projectId: string } }) => {
+ const conn = gscConnectionStore.get(projectId);
+ if (!conn) {
+ throw new Error(`gscConnection ${projectId} not found`);
+ }
+ return conn;
+ }
+ ),
+ findMany: mock(async () =>
+ [...gscConnectionStore.values()]
+ .filter((c) => c.siteUrl !== '')
+ .map((c) => ({ projectId: c.projectId }))
+ ),
+ update: mock(
+ async ({
+ where: { projectId },
+ data,
+ }: {
+ where: { projectId: string };
+ data: Partial<FakeGscConnection>;
+ }) => {
+ const existing = gscConnectionStore.get(projectId);
+ const updated = { ...existing, ...data } as FakeGscConnection;
+ gscConnectionStore.set(projectId, updated);
+ return updated;
+ }
+ ),
+ upsert: mock(
+ async ({
+ where: { projectId },
+ create,
+ update,
+ }: {
+ where: { projectId: string };
+ create: FakeGscConnection;
+ update: Partial<FakeGscConnection>;
+ }) => {
+ const existing = gscConnectionStore.get(projectId);
+ const value = existing
+ ? ({ ...existing, ...update } as FakeGscConnection)
+ : create;
+ gscConnectionStore.set(projectId, value);
+ return value;
+ }
+ ),
+ deleteMany: mock(
+ async ({ where: { projectId } }: { where: { projectId: string } }) => {
+ const existed = gscConnectionStore.delete(projectId);
+ return { count: existed ? 1 : 0 };
+ }
+ ),
+};
+
+const project = {
+ findUnique: mock(async ({ where: { id } }: { where: { id: string } }) =>
+ id === 'proj_missing' ? null : { id, organizationId: 'org_1' }
+ ),
+};
+
+mock.module('@openpanel/db/src/prisma-client', () => ({
+ db: { gscConnection, project },
+}));
+
+const chQuery = mock(async () => [] as unknown[]);
+const originalCh = {
+ query: mock(async () => ({ json: async () => [] as unknown[] })),
+ insert: mock(async () => undefined),
+};
+// `ch` is unused here but included because `mock.module` replaces this
+// specifier process-wide (bun runs every test file in one shared module
+// registry without `--isolate` — see AGENTS.md) — insight.service.test.ts
+// mocks the same path, so both factories must be a superset of every
+// consumer's needs, whichever one ends up registered last.
+mock.module('@openpanel/db/src/clickhouse/client', () => ({
+ ch: {},
+ originalCh,
+ chQuery,
+ TABLE_NAMES: { sessions: 'sessions' },
+}));
+
+// Bypasses the Redis cache-aside entirely — `getGscCannibalization`'s own
+// logic is exercised directly, its caching is @openpanel/redis's concern.
+// `getRedisCache` is unused here but included for the same cross-file
+// mock.module reason as above.
+mock.module('@openpanel/redis', () => ({
+ cacheable: <T>(fn: T) => fn,
+ getRedisCache: () => ({
+ get: async () => null,
+ setex: async () => undefined,
+ }),
+}));
+
+const validateAuthorizationCode = mock(async () => ({
+ accessToken: () => 'google-access-token',
+ hasRefreshToken: () => true,
+ refreshToken: () => 'google-refresh-token',
+ accessTokenExpiresAt: () => new Date('2026-09-04T00:00:00.000Z'),
+}));
+mock.module('../auth/auth.service', () => ({
+ googleGsc: { validateAuthorizationCode },
+}));
+
+const getSettingsForProject = mock(async () => ({ timezone: 'UTC' }));
+mock.module('@openpanel/db/src/services/organization.service', () => ({
+ getSettingsForProject,
+}));
+
+const getChartStartEndDate = mock(() => ({
+ startDate: '2026-09-01T00:00:00.000Z',
+ endDate: '2026-09-03T00:00:00.000Z',
+}));
+mock.module('@openpanel/db/src/services/date.service', () => ({
+ getChartStartEndDate,
+}));
+
+function stubLogger(): import('../../logger').Logger {
+ const noop = () => undefined;
+ const logger: import('../../logger').Logger = {
+ fatal: noop,
+ error: noop,
+ warn: noop,
+ info: noop,
+ debug: noop,
+ trace: noop,
+ child: () => logger,
+ };
+ return logger;
+}
+
+let subject: typeof import('./gsc.service');
+beforeAll(async () => {
+ subject = await import('./gsc.service');
+});
+
+test('listGscConnectionsForSync returns only projects with a siteUrl', async () => {
+ gscConnectionStore.clear();
+ gscConnectionStore.set('p1', {
+ projectId: 'p1',
+ siteUrl: 'https://one.example.com',
+ accessToken: '',
+ refreshToken: '',
+ accessTokenExpiresAt: null,
+ lastSyncStatus: null,
+ lastSyncError: null,
+ backfillStatus: null,
+ });
+ gscConnectionStore.set('p2', {
+ projectId: 'p2',
+ siteUrl: '',
+ accessToken: '',
+ refreshToken: '',
+ accessTokenExpiresAt: null,
+ lastSyncStatus: null,
+ lastSyncError: null,
+ backfillStatus: null,
+ });
+
+ const result = await subject.listGscConnectionsForSync();
+ expect(result).toEqual([{ projectId: 'p1' }]);
+});
+
+test('getGscConnection returns null when no connection exists', async () => {
+ gscConnectionStore.clear();
+ const result = await subject.getGscConnection('missing');
+ expect(result).toBeNull();
+});
+
+test('selectGscSite throws TRPCNotFoundError when the connection is missing', async () => {
+ gscConnectionStore.clear();
+ await expect(
+ subject.selectGscSite('missing', 'https://example.com')
+ ).rejects.toMatchObject({ message: 'GSC connection not found' });
+});
+
+test('selectGscSite updates siteUrl and resets backfillStatus to pending', async () => {
+ gscConnectionStore.clear();
+ gscConnectionStore.set('p1', {
+ projectId: 'p1',
+ siteUrl: '',
+ accessToken: '',
+ refreshToken: '',
+ accessTokenExpiresAt: null,
+ lastSyncStatus: null,
+ lastSyncError: null,
+ backfillStatus: null,
+ });
+
+ await subject.selectGscSite('p1', 'https://example.com');
+
+ expect(gscConnectionStore.get('p1')).toMatchObject({
+ siteUrl: 'https://example.com',
+ backfillStatus: 'pending',
+ });
+});
+
+test('disconnectGscConnection removes the stored connection', async () => {
+ gscConnectionStore.clear();
+ gscConnectionStore.set('p1', {
+ projectId: 'p1',
+ siteUrl: 'https://example.com',
+ accessToken: '',
+ refreshToken: '',
+ accessTokenExpiresAt: null,
+ lastSyncStatus: null,
+ lastSyncError: null,
+ backfillStatus: null,
+ });
+
+ await subject.disconnectGscConnection('p1');
+ expect(gscConnectionStore.has('p1')).toBe(false);
+});
+
+test('runGscProjectSync skips silently when there is no connected site', async () => {
+ gscConnectionStore.clear();
+ await expect(
+ subject.runGscProjectSync('missing', stubLogger())
+ ).resolves.toBeUndefined();
+});
+
+test('runGscProjectSync records an error status when the token refresh has no credentials', async () => {
+ gscConnectionStore.clear();
+ gscConnectionStore.set('p1', {
+ projectId: 'p1',
+ siteUrl: 'https://example.com',
+ accessToken: 'enc:whatever',
+ refreshToken: 'enc:whatever',
+ accessTokenExpiresAt: null, // forces the refresh branch
+ lastSyncStatus: null,
+ lastSyncError: null,
+ backfillStatus: null,
+ });
+ const previousId = process.env.GOOGLE_CLIENT_ID;
+ const previousSecret = process.env.GOOGLE_CLIENT_SECRET;
+ delete process.env.GOOGLE_CLIENT_ID;
+ delete process.env.GOOGLE_CLIENT_SECRET;
+
+ await expect(subject.runGscProjectSync('p1', stubLogger())).rejects.toThrow();
+
+ expect(gscConnectionStore.get('p1')?.lastSyncStatus).toBe('error');
+
+ if (previousId !== undefined) {
+ process.env.GOOGLE_CLIENT_ID = previousId;
+ }
+ if (previousSecret !== undefined) {
+ process.env.GOOGLE_CLIENT_SECRET = previousSecret;
+ }
+});
+
+test('completeGscOAuthCallback rejects a state mismatch before calling Google', async () => {
+ validateAuthorizationCode.mockClear();
+ await expect(
+ subject.completeGscOAuthCallback({
+ code: 'code',
+ state: 'a',
+ storedState: 'b',
+ codeVerifier: 'verifier',
+ projectId: 'p1',
+ })
+ ).rejects.toThrow('GSC OAuth state mismatch');
+ expect(validateAuthorizationCode).not.toHaveBeenCalled();
+});
+
+test('completeGscOAuthCallback rejects an unknown project', async () => {
+ await expect(
+ subject.completeGscOAuthCallback({
+ code: 'code',
+ state: 'a',
+ storedState: 'a',
+ codeVerifier: 'verifier',
+ projectId: 'proj_missing',
+ })
+ ).rejects.toThrow('Project not found for GSC connection');
+});
+
+test('completeGscOAuthCallback upserts the connection and returns the organizationId', async () => {
+ gscConnectionStore.clear();
+ const result = await subject.completeGscOAuthCallback({
+ code: 'code',
+ state: 'a',
+ storedState: 'a',
+ codeVerifier: 'verifier',
+ projectId: 'p1',
+ });
+
+ expect(result).toEqual({ organizationId: 'org_1' });
+ expect(gscConnectionStore.get('p1')).toMatchObject({ projectId: 'p1' });
+});
+
+test('resolveGscDateRange resolves through the org timezone and chart date helper', async () => {
+ const result = await subject.resolveGscDateRange('p1', { range: '7d' });
+ expect(result).toEqual({ startDate: '2026-09-01', endDate: '2026-09-03' });
+ expect(getSettingsForProject).toHaveBeenCalledWith('p1');
+});
+
+test('getGscOverview returns the ClickHouse rows as-is', async () => {
+ const rows = [
+ { date: '2026-09-01', clicks: 1, impressions: 2, ctr: 0.5, position: 3 },
+ ];
+ originalCh.query.mockImplementationOnce(async () => ({
+ json: async () => rows,
+ }));
+
+ const result = await subject.getGscOverview('p1', '2026-09-01', '2026-09-03');
+ expect(result).toEqual(rows);
+});
+
+test('gscGetOverviewCore aggregates the daily rows into a summary', async () => {
+ originalCh.query.mockImplementationOnce(async () => ({
+ json: async () => [
+ {
+ date: '2026-09-01',
+ clicks: 10,
+ impressions: 100,
+ ctr: 0.1,
+ position: 5,
+ },
+ {
+ date: '2026-09-02',
+ clicks: 20,
+ impressions: 100,
+ ctr: 0.2,
+ position: 3,
+ },
+ ],
+ }));
+
+ const result = await subject.gscGetOverviewCore({
+ projectId: 'p1',
+ startDate: '2026-09-01',
+ endDate: '2026-09-02',
+ });
+
+ expect(result.summary).toEqual({
+ total_clicks: 30,
+ total_impressions: 200,
+ avg_ctr: 15,
+ avg_position: 4,
+ });
+});
+
+test('gscGetQueryOpportunitiesCore filters and scores mid-ranked, high-impression queries', async () => {
+ originalCh.query.mockImplementationOnce(async () => ({
+ json: async () => [
+ // survives the impressions prefilter, but position 1 is out of the
+ // opportunity band [4, 20] — excluded by computeOpportunities
+ {
+ query: 'too-high',
+ clicks: 5,
+ impressions: 500,
+ ctr: 0.05,
+ position: 1,
+ },
+ // excluded by the impressions prefilter itself (< 50)
+ { query: 'too-rare', clicks: 1, impressions: 10, ctr: 0.05, position: 8 },
+ // kept: mid-ranked, enough impressions
+ {
+ query: 'opportunity',
+ clicks: 5,
+ impressions: 200,
+ ctr: 0.02,
+ position: 8,
+ },
+ ],
+ }));
+
+ const result = await subject.gscGetQueryOpportunitiesCore({
+ projectId: 'p1',
+ startDate: '2026-09-01',
+ endDate: '2026-09-02',
+ });
+
+ // total_analyzed counts everything past the impressions prefilter, before
+ // computeOpportunities' own position/impressions band is applied.
+ expect(result.total_analyzed).toBe(2);
+ expect(result.opportunities).toHaveLength(1);
+ expect(result.opportunities[0]?.query).toBe('opportunity');
+});
diff --git a/packages/core/src/modules/gsc/gsc.service.ts b/packages/core/src/modules/gsc/gsc.service.ts
new file mode 100644
index 000000000..96d02426c
--- /dev/null
+++ b/packages/core/src/modules/gsc/gsc.service.ts
@@ -0,0 +1,1456 @@
+// Moved from packages/db/src/gsc.ts + packages/db/src/services/gsc.service.ts
+// (M5-002) — the GSC OAuth token lifecycle, the Search Console API client,
+// ClickHouse read/write, the AI-tool wrapper functions and the
+// gscProjectSync/gscProjectBackfill job bodies. packages/db/src/gsc.ts and
+// packages/db/src/services/gsc.service.ts become re-export shims of this
+// file (same shape as packages/db/src/encryption.ts since M4-006).
+//
+// db/ch access is LAZY (`load*` below), not a static top-level import — see
+// insight.service.ts's header for the full reasoning (jobs.registry.ts and
+// services.ts pull this module into the eager barrel chain nearly every core
+// test file reaches, and constructing @openpanel/db's clients at import time
+// would spawn a pino-pretty transport worker thread per test file).
+//
+// ClickHouse queries here still go through raw `originalCh`/`chQuery` calls,
+// not the `sql` tag: ADR-013 converts the analytics read path one query per
+// P7 task, and this module's queries haven't been converted yet.
+
+import { cacheable } from '@openpanel/redis';
+import { createLogger, type ILogger } from '../../clients/logger';
+import type { Logger } from '../../logger';
+import { TRPCNotFoundError } from '../../rpc/errors';
+import type { ServiceDeps } from '../../services';
+import { decrypt, encrypt } from '../../shared/encryption';
+import { googleGsc } from '../auth/auth.service';
+
+const BACKFILL_MONTHS = 6;
+const CHUNK_DAYS = 14;
+const CANNIBALIZATION_CACHE_TTL_SEC = 60 * 60 * 4;
+const GSC_ROW_LIMIT = 25_000;
+
+let _logger: ILogger | undefined;
+function getLogger(): ILogger {
+ _logger ??= createLogger({ name: 'core:gsc' });
+ return _logger;
+}
+
+function loadDb() {
+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);
+}
+
+function loadChClient() {
+ return import('@openpanel/db/src/clickhouse/client');
+}
+
+export interface GscSite {
+ siteUrl: string;
+ permissionLevel: string;
+}
+
+async function refreshGscToken(
+ refreshToken: string
+): Promise<{ accessToken: string; expiresAt: Date }> {
+ if (!(process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET)) {
+ throw new Error(
+ 'GOOGLE_CLIENT_ID or GOOGLE_CLIENT_SECRET is not set in this environment'
+ );
+ }
+
+ const params = new URLSearchParams({
+ client_id: process.env.GOOGLE_CLIENT_ID,
+ client_secret: process.env.GOOGLE_CLIENT_SECRET,
+ refresh_token: refreshToken,
+ grant_type: 'refresh_token',
+ });
+
+ const res = await fetch('https://oauth2.googleapis.com/token', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+ body: params.toString(),
+ });
+
+ if (!res.ok) {
+ const text = await res.text();
+ throw new Error(`Failed to refresh GSC token: ${text}`);
+ }
+
+ const data = (await res.json()) as {
+ access_token: string;
+ expires_in: number;
+ };
+ const expiresAt = new Date(Date.now() + data.expires_in * 1000);
+ return { accessToken: data.access_token, expiresAt };
+}
+
+export async function getGscAccessToken(projectId: string): Promise<string> {
+ const db = await loadDb();
+ const conn = await db.gscConnection.findUniqueOrThrow({
+ where: { projectId },
+ });
+
+ if (
+ conn.accessTokenExpiresAt &&
+ conn.accessTokenExpiresAt.getTime() > Date.now() + 60_000
+ ) {
+ getLogger().info(
+ { projectId, expiresAt: conn.accessTokenExpiresAt },
+ 'GSC using cached access token'
+ );
+ return decrypt(conn.accessToken);
+ }
+
+ getLogger().info(
+ {
+ projectId,
+ expiresAt: conn.accessTokenExpiresAt,
+ hasRefreshToken: !!conn.refreshToken,
+ },
+ 'GSC access token expired, attempting refresh'
+ );
+
+ try {
+ const { accessToken, expiresAt } = await refreshGscToken(
+ decrypt(conn.refreshToken)
+ );
+ await db.gscConnection.update({
+ where: { projectId },
+ data: {
+ accessToken: encrypt(accessToken),
+ accessTokenExpiresAt: expiresAt,
+ },
+ });
+ getLogger().info(
+ { projectId, expiresAt },
+ 'GSC token refreshed successfully'
+ );
+ return accessToken;
+ } catch (error) {
+ const errorMessage =
+ error instanceof Error ? error.message : 'Failed to refresh token';
+ getLogger().error(
+ { err: error, projectId, errorMessage },
+ 'GSC token refresh failed'
+ );
+ await db.gscConnection.update({
+ where: { projectId },
+ data: {
+ lastSyncStatus: 'token_expired',
+ lastSyncError: errorMessage,
+ },
+ });
+ throw new Error(
+ `GSC token refresh failed for project ${projectId}: ${errorMessage}`
+ );
+ }
+}
+
+export async function listGscSites(projectId: string): Promise<GscSite[]> {
+ const accessToken = await getGscAccessToken(projectId);
+ const res = await fetch('https://www.googleapis.com/webmasters/v3/sites', {
+ headers: { Authorization: `Bearer ${accessToken}` },
+ });
+
+ if (!res.ok) {
+ const text = await res.text();
+ throw new Error(`Failed to list GSC sites: ${text}`);
+ }
+
+ const data = (await res.json()) as {
+ siteEntry?: Array<{ siteUrl: string; permissionLevel: string }>;
+ };
+ return data.siteEntry ?? [];
+}
+
+interface GscApiRow {
+ keys: string[];
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+}
+
+interface GscDimensionFilter {
+ dimension: string;
+ operator: string;
+ expression: string;
+}
+
+interface GscFilterGroup {
+ filters: GscDimensionFilter[];
+}
+
+async function queryGscSearchAnalytics(
+ accessToken: string,
+ siteUrl: string,
+ startDate: string,
+ endDate: string,
+ dimensions: string[],
+ dimensionFilterGroups?: GscFilterGroup[]
+): Promise<GscApiRow[]> {
+ const encodedSiteUrl = encodeURIComponent(siteUrl);
+ const url = `https://www.googleapis.com/webmasters/v3/sites/${encodedSiteUrl}/searchAnalytics/query`;
+
+ const allRows: GscApiRow[] = [];
+ let startRow = 0;
+
+ while (true) {
+ const res = await fetch(url, {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ startDate,
+ endDate,
+ dimensions,
+ rowLimit: GSC_ROW_LIMIT,
+ startRow,
+ dataState: 'all',
+ ...(dimensionFilterGroups && { dimensionFilterGroups }),
+ }),
+ });
+
+ if (!res.ok) {
+ const text = await res.text();
+ throw new Error(
+ `GSC query failed for dimensions [${dimensions.join(',')}]: ${text}`
+ );
+ }
+
+ const data = (await res.json()) as { rows?: GscApiRow[] };
+ const rows = data.rows ?? [];
+ allRows.push(...rows);
+
+ if (rows.length < GSC_ROW_LIMIT) {
+ break;
+ }
+ startRow += GSC_ROW_LIMIT;
+ }
+
+ return allRows;
+}
+
+function formatDate(date: Date): string {
+ return date.toISOString().slice(0, 10);
+}
+
+function nowString(): string {
+ return new Date().toISOString().replace('T', ' ').replace('Z', '');
+}
+
+export async function syncGscData(
+ projectId: string,
+ startDate: Date,
+ endDate: Date
+): Promise<void> {
+ const db = await loadDb();
+ const conn = await db.gscConnection.findUniqueOrThrow({
+ where: { projectId },
+ });
+
+ if (!conn.siteUrl) {
+ throw new Error('No GSC site URL configured for this project');
+ }
+
+ const accessToken = await getGscAccessToken(projectId);
+ const start = formatDate(startDate);
+ const end = formatDate(endDate);
+ const syncedAt = nowString();
+ const { originalCh } = await loadChClient();
+
+ // 1. Daily totals — authoritative numbers for overview chart
+ const dailyRows = await queryGscSearchAnalytics(
+ accessToken,
+ conn.siteUrl,
+ start,
+ end,
+ ['date']
+ );
+
+ if (dailyRows.length > 0) {
+ await originalCh.insert({
+ table: 'gsc_daily',
+ values: dailyRows.map((row) => ({
+ project_id: projectId,
+ date: row.keys[0] ?? '',
+ clicks: row.clicks,
+ impressions: row.impressions,
+ ctr: row.ctr,
+ position: row.position,
+ synced_at: syncedAt,
+ })),
+ format: 'JSONEachRow',
+ });
+ }
+
+ // 2. Per-page breakdown
+ const pageRows = await queryGscSearchAnalytics(
+ accessToken,
+ conn.siteUrl,
+ start,
+ end,
+ ['date', 'page']
+ );
+
+ if (pageRows.length > 0) {
+ await originalCh.insert({
+ table: 'gsc_pages_daily',
+ values: pageRows.map((row) => ({
+ project_id: projectId,
+ date: row.keys[0] ?? '',
+ page: row.keys[1] ?? '',
+ clicks: row.clicks,
+ impressions: row.impressions,
+ ctr: row.ctr,
+ position: row.position,
+ synced_at: syncedAt,
+ })),
+ format: 'JSONEachRow',
+ });
+ }
+
+ // 3. Per-query breakdown
+ const queryRows = await queryGscSearchAnalytics(
+ accessToken,
+ conn.siteUrl,
+ start,
+ end,
+ ['date', 'query']
+ );
+
+ if (queryRows.length > 0) {
+ await originalCh.insert({
+ table: 'gsc_queries_daily',
+ values: queryRows.map((row) => ({
+ project_id: projectId,
+ date: row.keys[0] ?? '',
+ query: row.keys[1] ?? '',
+ clicks: row.clicks,
+ impressions: row.impressions,
+ ctr: row.ctr,
+ position: row.position,
+ synced_at: syncedAt,
+ })),
+ format: 'JSONEachRow',
+ });
+ }
+}
+
+export async function getGscOverview(
+ projectId: string,
+ startDate: string,
+ endDate: string,
+ interval: 'day' | 'week' | 'month' = 'day'
+): Promise<
+ Array<{
+ date: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ }>
+> {
+ const dateExpr =
+ interval === 'month'
+ ? 'toStartOfMonth(date)'
+ : interval === 'week'
+ ? 'toStartOfWeek(date)'
+ : 'date';
+
+ const { originalCh } = await loadChClient();
+ const result = await originalCh.query({
+ query: `
+ SELECT
+ ${dateExpr} as date,
+ sum(clicks) as clicks,
+ sum(impressions) as impressions,
+ avg(ctr) as ctr,
+ avg(position) as position
+ FROM gsc_daily
+ FINAL
+ WHERE project_id = {projectId: String}
+ AND date >= {startDate: String}
+ AND date <= {endDate: String}
+ GROUP BY date
+ ORDER BY date ASC
+ `,
+ query_params: { projectId, startDate, endDate },
+ format: 'JSONEachRow',
+ });
+ return result.json();
+}
+
+export async function getGscPages(
+ projectId: string,
+ startDate: string,
+ endDate: string,
+ limit = 100
+): Promise<
+ Array<{
+ page: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ }>
+> {
+ const { originalCh } = await loadChClient();
+ const result = await originalCh.query({
+ query: `
+ SELECT
+ page,
+ sum(clicks) as clicks,
+ sum(impressions) as impressions,
+ avg(ctr) as ctr,
+ avg(position) as position
+ FROM gsc_pages_daily
+ FINAL
+ WHERE project_id = {projectId: String}
+ AND date >= {startDate: String}
+ AND date <= {endDate: String}
+ GROUP BY page
+ ORDER BY clicks DESC
+ LIMIT {limit: UInt32}
+ `,
+ query_params: { projectId, startDate, endDate, limit },
+ format: 'JSONEachRow',
+ });
+ return result.json();
+}
+
+export interface GscCannibalizedQuery {
+ query: string;
+ totalImpressions: number;
+ totalClicks: number;
+ pages: Array<{
+ page: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ }>;
+}
+
+export const getGscCannibalization = cacheable(
+ async (
+ projectId: string,
+ startDate: string,
+ endDate: string
+ ): Promise<GscCannibalizedQuery[]> => {
+ const db = await loadDb();
+ const conn = await db.gscConnection.findUniqueOrThrow({
+ where: { projectId },
+ });
+ const accessToken = await getGscAccessToken(projectId);
+
+ const rows = await queryGscSearchAnalytics(
+ accessToken,
+ conn.siteUrl,
+ startDate,
+ endDate,
+ ['query', 'page']
+ );
+
+ const map = new Map<
+ string,
+ {
+ totalImpressions: number;
+ totalClicks: number;
+ pages: GscCannibalizedQuery['pages'];
+ }
+ >();
+
+ for (const row of rows) {
+ const query = row.keys[0] ?? '';
+ // Strip hash fragments — GSC records heading anchors (e.g. /page#section)
+ // as separate URLs but Google treats them as the same page
+ let page = row.keys[1] ?? '';
+ try {
+ const u = new URL(page);
+ u.hash = '';
+ page = u.toString();
+ } catch {
+ page = page.split('#')[0] ?? page;
+ }
+
+ const entry = map.get(query) ?? {
+ totalImpressions: 0,
+ totalClicks: 0,
+ pages: [],
+ };
+ entry.totalImpressions += row.impressions;
+ entry.totalClicks += row.clicks;
+ // Merge into existing page entry if already seen (from a different hash variant)
+ const existing = entry.pages.find((p) => p.page === page);
+ if (existing) {
+ const totalImpressions = existing.impressions + row.impressions;
+ if (totalImpressions > 0) {
+ existing.position =
+ (existing.position * existing.impressions +
+ row.position * row.impressions) /
+ totalImpressions;
+ }
+ existing.clicks += row.clicks;
+ existing.impressions += row.impressions;
+ existing.ctr =
+ existing.impressions > 0 ? existing.clicks / existing.impressions : 0;
+ } else {
+ entry.pages.push({
+ page,
+ clicks: row.clicks,
+ impressions: row.impressions,
+ ctr: row.ctr,
+ position: row.position,
+ });
+ }
+ map.set(query, entry);
+ }
+
+ return [...map.entries()]
+ .filter(([, v]) => v.pages.length >= 2 && v.totalImpressions >= 100)
+ .sort(([, a], [, b]) => b.totalImpressions - a.totalImpressions)
+ .slice(0, 50)
+ .map(([query, v]) => ({
+ query,
+ totalImpressions: v.totalImpressions,
+ totalClicks: v.totalClicks,
+ pages: v.pages.sort((a, b) =>
+ a.position !== b.position
+ ? a.position - b.position
+ : b.impressions - a.impressions
+ ),
+ }));
+ },
+ CANNIBALIZATION_CACHE_TTL_SEC
+);
+
+export async function getGscPageDetails(
+ projectId: string,
+ page: string,
+ startDate: string,
+ endDate: string
+): Promise<{
+ timeseries: Array<{
+ date: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ }>;
+ queries: Array<{
+ query: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ }>;
+}> {
+ const db = await loadDb();
+ const conn = await db.gscConnection.findUniqueOrThrow({
+ where: { projectId },
+ });
+ const accessToken = await getGscAccessToken(projectId);
+ const filterGroups: GscFilterGroup[] = [
+ { filters: [{ dimension: 'page', operator: 'equals', expression: page }] },
+ ];
+
+ const [timeseriesRows, queryRows] = await Promise.all([
+ queryGscSearchAnalytics(
+ accessToken,
+ conn.siteUrl,
+ startDate,
+ endDate,
+ ['date'],
+ filterGroups
+ ),
+ queryGscSearchAnalytics(
+ accessToken,
+ conn.siteUrl,
+ startDate,
+ endDate,
+ ['query'],
+ filterGroups
+ ),
+ ]);
+
+ return {
+ timeseries: timeseriesRows.map((row) => ({
+ date: row.keys[0] ?? '',
+ clicks: row.clicks,
+ impressions: row.impressions,
+ ctr: row.ctr,
+ position: row.position,
+ })),
+ queries: queryRows.map((row) => ({
+ query: row.keys[0] ?? '',
+ clicks: row.clicks,
+ impressions: row.impressions,
+ ctr: row.ctr,
+ position: row.position,
+ })),
+ };
+}
+
+export async function getGscQueryDetails(
+ projectId: string,
+ query: string,
+ startDate: string,
+ endDate: string
+): Promise<{
+ timeseries: Array<{
+ date: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ }>;
+ pages: Array<{
+ page: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ }>;
+}> {
+ const db = await loadDb();
+ const conn = await db.gscConnection.findUniqueOrThrow({
+ where: { projectId },
+ });
+ const accessToken = await getGscAccessToken(projectId);
+ const filterGroups: GscFilterGroup[] = [
+ {
+ filters: [{ dimension: 'query', operator: 'equals', expression: query }],
+ },
+ ];
+
+ const [timeseriesRows, pageRows] = await Promise.all([
+ queryGscSearchAnalytics(
+ accessToken,
+ conn.siteUrl,
+ startDate,
+ endDate,
+ ['date'],
+ filterGroups
+ ),
+ queryGscSearchAnalytics(
+ accessToken,
+ conn.siteUrl,
+ startDate,
+ endDate,
+ ['page'],
+ filterGroups
+ ),
+ ]);
+
+ return {
+ timeseries: timeseriesRows.map((row) => ({
+ date: row.keys[0] ?? '',
+ clicks: row.clicks,
+ impressions: row.impressions,
+ ctr: row.ctr,
+ position: row.position,
+ })),
+ pages: pageRows.map((row) => ({
+ page: row.keys[0] ?? '',
+ clicks: row.clicks,
+ impressions: row.impressions,
+ ctr: row.ctr,
+ position: row.position,
+ })),
+ };
+}
+
+export async function getGscQueries(
+ projectId: string,
+ startDate: string,
+ endDate: string,
+ limit = 100
+): Promise<
+ Array<{
+ query: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ }>
+> {
+ const { originalCh } = await loadChClient();
+ const result = await originalCh.query({
+ query: `
+ SELECT
+ query,
+ sum(clicks) as clicks,
+ sum(impressions) as impressions,
+ avg(ctr) as ctr,
+ avg(position) as position
+ FROM gsc_queries_daily
+ FINAL
+ WHERE project_id = {projectId: String}
+ AND date >= {startDate: String}
+ AND date <= {endDate: String}
+ GROUP BY query
+ ORDER BY clicks DESC
+ LIMIT {limit: UInt32}
+ `,
+ query_params: { projectId, startDate, endDate, limit },
+ format: 'JSONEachRow',
+ });
+ return result.json();
+}
+
+// ---------------------------------------------------------------------------
+// Connection CRUD — moved from packages/trpc/src/routers/gsc.ts's
+// getConnection/selectSite/disconnect handler bodies (M5-002).
+// ---------------------------------------------------------------------------
+
+export interface GscConnectionSummary {
+ id: string;
+ siteUrl: string;
+ lastSyncedAt: Date | null;
+ lastSyncStatus: string | null;
+ lastSyncError: string | null;
+ backfillStatus: string | null;
+ createdAt: Date;
+ updatedAt: Date;
+}
+
+export async function getGscConnection(
+ projectId: string
+): Promise<GscConnectionSummary | null> {
+ const db = await loadDb();
+ return db.gscConnection.findUnique({
+ where: { projectId },
+ select: {
+ id: true,
+ siteUrl: true,
+ lastSyncedAt: true,
+ lastSyncStatus: true,
+ lastSyncError: true,
+ backfillStatus: true,
+ createdAt: true,
+ updatedAt: true,
+ },
+ });
+}
+
+export async function selectGscSite(
+ projectId: string,
+ siteUrl: string
+): Promise<void> {
+ const db = await loadDb();
+ const conn = await db.gscConnection.findUnique({ where: { projectId } });
+ if (!conn) {
+ throw new TRPCNotFoundError('GSC connection not found');
+ }
+
+ await db.gscConnection.update({
+ where: { projectId },
+ data: { siteUrl, backfillStatus: 'pending' },
+ });
+}
+
+export async function disconnectGscConnection(
+ projectId: string
+): Promise<void> {
+ const db = await loadDb();
+ await db.gscConnection.deleteMany({ where: { projectId } });
+}
+
+// ---------------------------------------------------------------------------
+// Dashboard aggregate queries — moved from packages/trpc/src/routers/gsc.ts
+// (M5-002). The trpc router itself stays on V1 (DELEGATE PATTERN); this is
+// its date-resolution and search/AI-engine breakdown logic.
+// ---------------------------------------------------------------------------
+
+export interface GscDateRangeInput {
+ range: string;
+ startDate?: string | null;
+ endDate?: string | null;
+}
+
+export async function resolveGscDateRange(
+ projectId: string,
+ input: GscDateRangeInput
+): Promise<{ startDate: string; endDate: string }> {
+ const { getSettingsForProject } = await import(
+ '@openpanel/db/src/services/organization.service'
+ );
+ const { getChartStartEndDate } = await import(
+ '@openpanel/db/src/services/date.service'
+ );
+ const { timezone } = await getSettingsForProject(projectId);
+ const { startDate, endDate } = getChartStartEndDate(
+ {
+ range: input.range as never,
+ startDate: input.startDate,
+ endDate: input.endDate,
+ },
+ timezone
+ );
+ return {
+ startDate: startDate.slice(0, 10),
+ endDate: endDate.slice(0, 10),
+ };
+}
+
+/**
+ * ClickHouse stores the same AI referrer under several spellings: the parsed
+ * display name ('ChatGPT', 'Google Gemini'), the bare host ('chatgpt.com') and
+ * the full origin ('https://kagi.com'). Normalize before matching so every
+ * spelling lands on the same engine.
+ */
+const NORMALIZED_REFERRER_NAME =
+ "lower(regexp_replace(referrer_name, '^https?://(www[.])?', ''))";
+
+const AI_REFERRERS = [
+ {
+ canonical: 'chatgpt.com',
+ aliases: [
+ 'chatgpt',
+ 'chatgpt.com',
+ 'chat.openai.com',
+ 'openai',
+ 'openai.com',
+ ],
+ },
+ {
+ canonical: 'claude.ai',
+ aliases: ['claude', 'claude.ai', 'anthropic', 'anthropic.com'],
+ },
+ {
+ canonical: 'perplexity.ai',
+ aliases: ['perplexity', 'perplexity.ai'],
+ },
+ {
+ canonical: 'gemini.google.com',
+ aliases: [
+ 'gemini',
+ 'google gemini',
+ 'gemini.google.com',
+ 'bard.google.com',
+ ],
+ },
+ {
+ canonical: 'copilot.com',
+ aliases: [
+ 'copilot',
+ 'copilot.com',
+ 'copilot.microsoft.com',
+ 'microsoft copilot',
+ ],
+ },
+ { canonical: 'grok.com', aliases: ['grok', 'grok.com'] },
+ {
+ canonical: 'mistral.ai',
+ aliases: ['mistral', 'mistral.ai', 'chat.mistral.ai', 'le chat'],
+ },
+ {
+ canonical: 'kagi.com',
+ aliases: ['kagi', 'kagi.com', 'assistant.kagi.com'],
+ },
+] as const satisfies ReadonlyArray<{
+ canonical: string;
+ aliases: readonly string[];
+}>;
+
+const quoteList = (values: readonly string[]) =>
+ values.map((value) => `'${value}'`).join(', ');
+
+/** `norm` is the alias bound by AI_REFERRER_CTE below. */
+const AI_REFERRER_CTE = `WITH ${NORMALIZED_REFERRER_NAME} AS norm`;
+
+const AI_REFERRER_FILTER = `norm IN (${quoteList(
+ AI_REFERRERS.flatMap((engine) => engine.aliases)
+)})`;
+
+/** Collapses every alias onto one row per engine. */
+const AI_REFERRER_CANONICAL_NAME = `multiIf(${AI_REFERRERS.map(
+ (engine) => `norm IN (${quoteList(engine.aliases)}), '${engine.canonical}'`
+).join(', ')}, norm)`;
+
+/**
+ * Half-open windows so the last day of the range is included and the previous
+ * window ends exactly where the current one starts (no gap, no overlap).
+ */
+function getComparisonWindows(startDate: string, endDate: string) {
+ const start = new Date(`${startDate}T00:00:00Z`);
+ const endExclusive = new Date(`${endDate}T00:00:00Z`);
+ endExclusive.setUTCDate(endExclusive.getUTCDate() + 1);
+ const previousStart = new Date(
+ start.getTime() - (endExclusive.getTime() - start.getTime())
+ );
+ const fmt = (date: Date) => date.toISOString().slice(0, 19).replace('T', ' ');
+ return {
+ current: { start: fmt(start), end: fmt(endExclusive) },
+ previous: { start: fmt(previousStart), end: fmt(start) },
+ };
+}
+
+export async function getGscSearchEngines(
+ projectId: string,
+ startDate: string,
+ endDate: string
+) {
+ const windows = getComparisonWindows(startDate, endDate);
+ const { chQuery, TABLE_NAMES } = await loadChClient();
+
+ const where = (window: { start: string; end: string }) =>
+ `project_id = '${projectId}'
+ AND referrer_type = 'search'
+ AND created_at >= '${window.start}'
+ AND created_at < '${window.end}'`;
+
+ const [engines, [currentResult], [prevResult]] = await Promise.all([
+ chQuery<{ name: string; sessions: number }>(
+ `SELECT
+ referrer_name as name,
+ count(*) as sessions
+ FROM ${TABLE_NAMES.sessions}
+ WHERE ${where(windows.current)}
+ GROUP BY name
+ ORDER BY sessions DESC
+ LIMIT 10`
+ ),
+ chQuery<{ sessions: number }>(
+ `SELECT count(*) as sessions
+ FROM ${TABLE_NAMES.sessions}
+ WHERE ${where(windows.current)}`
+ ),
+ chQuery<{ sessions: number }>(
+ `SELECT count(*) as sessions
+ FROM ${TABLE_NAMES.sessions}
+ WHERE ${where(windows.previous)}`
+ ),
+ ]);
+
+ return {
+ engines,
+ // Counted separately from `engines`, which is capped at the top 10, so
+ // the total compares like-for-like with the previous period.
+ total: currentResult?.sessions ?? 0,
+ previousTotal: prevResult?.sessions ?? 0,
+ };
+}
+
+export async function getGscAiEngines(
+ projectId: string,
+ startDate: string,
+ endDate: string
+) {
+ const windows = getComparisonWindows(startDate, endDate);
+ const { chQuery, TABLE_NAMES } = await loadChClient();
+
+ // Matched by name — will switch to referrer_type = 'ai' once available.
+ const where = (window: { start: string; end: string }) =>
+ `project_id = '${projectId}'
+ AND ${AI_REFERRER_FILTER}
+ AND created_at >= '${window.start}'
+ AND created_at < '${window.end}'`;
+
+ const [engines, [prevResult]] = await Promise.all([
+ chQuery<{ name: string; sessions: number }>(
+ `${AI_REFERRER_CTE}
+ SELECT ${AI_REFERRER_CANONICAL_NAME} as name, count(*) as sessions
+ FROM ${TABLE_NAMES.sessions}
+ WHERE ${where(windows.current)}
+ GROUP BY name
+ ORDER BY sessions DESC`
+ ),
+ chQuery<{ sessions: number }>(
+ `${AI_REFERRER_CTE}
+ SELECT count(*) as sessions
+ FROM ${TABLE_NAMES.sessions}
+ WHERE ${where(windows.previous)}`
+ ),
+ ]);
+
+ return {
+ engines,
+ // The filter is a fixed alias set, so the grouped rows are the whole
+ // population — no LIMIT, and the total matches the previous period.
+ total: engines.reduce((sum, engine) => sum + engine.sessions, 0),
+ previousTotal: prevResult?.sessions ?? 0,
+ };
+}
+
+export async function getGscPreviousOverview(
+ projectId: string,
+ startDate: string,
+ endDate: string,
+ interval: 'day' | 'week' | 'month' = 'day'
+) {
+ const startMs = new Date(startDate).getTime();
+ const duration = new Date(endDate).getTime() - startMs;
+ const prevEnd = new Date(startMs - 1);
+ const prevStart = new Date(prevEnd.getTime() - duration);
+ const fmt = (d: Date) => d.toISOString().slice(0, 10);
+
+ return getGscOverview(projectId, fmt(prevStart), fmt(prevEnd), interval);
+}
+
+// ---------------------------------------------------------------------------
+// AI-tool wrappers — moved from packages/db/src/services/gsc.service.ts.
+// Consumed by apps/api's assistant SEO tools and the /insights REST surface
+// (both still reach @openpanel/db, which re-exports these unchanged).
+// ---------------------------------------------------------------------------
+
+export interface GscQueryOpportunity {
+ query: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ opportunity_score: number;
+ reason: string;
+}
+
+const CTR_BENCHMARKS: Record<string, number> = {
+ '1': 0.28,
+ '2': 0.15,
+ '3': 0.11,
+ '4-6': 0.065,
+ '7-10': 0.035,
+ '11-20': 0.012,
+};
+
+function getCtrBenchmark(position: number): number {
+ if (position <= 1) {
+ return CTR_BENCHMARKS['1'] ?? 0.28;
+ }
+ if (position <= 2) {
+ return CTR_BENCHMARKS['2'] ?? 0.15;
+ }
+ if (position <= 3) {
+ return CTR_BENCHMARKS['3'] ?? 0.11;
+ }
+ if (position <= 6) {
+ return CTR_BENCHMARKS['4-6'] ?? 0.065;
+ }
+ if (position <= 10) {
+ return CTR_BENCHMARKS['7-10'] ?? 0.035;
+ }
+ return CTR_BENCHMARKS['11-20'] ?? 0.012;
+}
+
+function computeOpportunities(
+ queries: Array<{
+ query: string;
+ clicks: number;
+ impressions: number;
+ ctr: number;
+ position: number;
+ }>
+): GscQueryOpportunity[] {
+ return queries
+ .filter((q) => q.position >= 4 && q.position <= 20 && q.impressions >= 50)
+ .map((q) => {
+ const benchmark = getCtrBenchmark(q.position);
+ const ctrGap = Math.max(0, benchmark - q.ctr);
+ const opportunity_score =
+ Math.round(q.impressions * (1 / q.position) * (1 + ctrGap) * 100) / 100;
+
+ let reason: string;
+ if (q.position <= 6) {
+ reason = `Position ${q.position.toFixed(1)} — one rank improvement could significantly boost clicks`;
+ } else if (q.ctr < benchmark * 0.5) {
+ reason = `CTR (${(q.ctr * 100).toFixed(1)}%) is well below expected ${(benchmark * 100).toFixed(1)}% — title/meta optimization may help`;
+ } else {
+ reason = `Position ${q.position.toFixed(1)} with ${q.impressions} impressions — push to page 1 for major gains`;
+ }
+
+ return {
+ query: q.query,
+ clicks: q.clicks,
+ impressions: q.impressions,
+ ctr: Math.round(q.ctr * 10_000) / 100,
+ position: Math.round(q.position * 10) / 10,
+ opportunity_score,
+ reason,
+ };
+ })
+ .sort((a, b) => b.opportunity_score - a.opportunity_score)
+ .slice(0, 50);
+}
+
+export async function gscGetOverviewCore(input: {
+ projectId: string;
+ startDate: string;
+ endDate: string;
+ interval?: 'day' | 'week' | 'month';
+}) {
+ const data = await getGscOverview(
+ input.projectId,
+ input.startDate,
+ input.endDate,
+ input.interval ?? 'day'
+ );
+ return {
+ data,
+ summary: {
+ total_clicks: data.reduce((s, r) => s + r.clicks, 0),
+ total_impressions: data.reduce((s, r) => s + r.impressions, 0),
+ avg_ctr:
+ data.length > 0
+ ? Math.round(
+ (data.reduce((s, r) => s + r.ctr, 0) / data.length) * 10_000
+ ) / 100
+ : 0,
+ avg_position:
+ data.length > 0
+ ? Math.round(
+ (data.reduce((s, r) => s + r.position, 0) / data.length) * 10
+ ) / 10
+ : 0,
+ },
+ };
+}
+
+export async function gscGetTopPagesCore(input: {
+ projectId: string;
+ startDate: string;
+ endDate: string;
+ limit?: number;
+}) {
+ return getGscPages(
+ input.projectId,
+ input.startDate,
+ input.endDate,
+ input.limit ?? 100
+ );
+}
+
+export async function gscGetPageDetailsCore(input: {
+ projectId: string;
+ startDate: string;
+ endDate: string;
+ page: string;
+}) {
+ return getGscPageDetails(
+ input.projectId,
+ input.page,
+ input.startDate,
+ input.endDate
+ );
+}
+
+export async function gscGetTopQueriesCore(input: {
+ projectId: string;
+ startDate: string;
+ endDate: string;
+ limit?: number;
+}) {
+ return getGscQueries(
+ input.projectId,
+ input.startDate,
+ input.endDate,
+ input.limit ?? 100
+ );
+}
+
+export async function gscGetQueryOpportunitiesCore(input: {
+ projectId: string;
+ startDate: string;
+ endDate: string;
+ minImpressions?: number;
+}) {
+ const queries = await getGscQueries(
+ input.projectId,
+ input.startDate,
+ input.endDate,
+ 5000
+ );
+ const filtered = queries.filter(
+ (q) => q.impressions >= (input.minImpressions ?? 50)
+ );
+ const opportunities = computeOpportunities(filtered);
+ return {
+ opportunities,
+ total_analyzed: filtered.length,
+ min_impressions: input.minImpressions ?? 50,
+ };
+}
+
+export async function gscGetQueryDetailsCore(input: {
+ projectId: string;
+ startDate: string;
+ endDate: string;
+ query: string;
+}) {
+ return getGscQueryDetails(
+ input.projectId,
+ input.query,
+ input.startDate,
+ input.endDate
+ );
+}
+
+export async function gscGetCannibalizationCore(input: {
+ projectId: string;
+ startDate: string;
+ endDate: string;
+}) {
+ return getGscCannibalization(input.projectId, input.startDate, input.endDate);
+}
+
+// ---------------------------------------------------------------------------
+// Job bodies — moved from apps/worker/src/jobs/gsc.ts (M5-002). Each fully
+// owns its status-update bookkeeping so the worker file is a bare dispatch.
+// ---------------------------------------------------------------------------
+
+/** The `gscSync` cron fan-out: every project with a connected GSC site. */
+export async function listGscConnectionsForSync(): Promise<
+ { projectId: string }[]
+> {
+ const db = await loadDb();
+ return db.gscConnection.findMany({
+ where: { siteUrl: { not: '' } },
+ select: { projectId: true },
+ });
+}
+
+/** The `gscProjectSync` job body: rolling 3-day window (GSC data arrives late). */
+export async function runGscProjectSync(
+ projectId: string,
+ logger: Logger = getLogger()
+): Promise<void> {
+ const db = await loadDb();
+ const conn = await db.gscConnection.findUnique({ where: { projectId } });
+ if (!conn?.siteUrl) {
+ logger.warn({ projectId }, 'GSC sync skipped: no connection or siteUrl');
+ return;
+ }
+
+ try {
+ const endDate = new Date();
+ endDate.setDate(endDate.getDate() - 1); // yesterday
+ const startDate = new Date(endDate);
+ startDate.setDate(startDate.getDate() - 2); // 3 days total
+
+ await syncGscData(projectId, startDate, endDate);
+
+ await db.gscConnection.update({
+ where: { projectId },
+ data: {
+ lastSyncedAt: new Date(),
+ lastSyncStatus: 'success',
+ lastSyncError: null,
+ },
+ });
+ logger.info({ projectId }, 'GSC sync completed');
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ await db.gscConnection.update({
+ where: { projectId },
+ data: {
+ lastSyncedAt: new Date(),
+ lastSyncStatus: 'error',
+ lastSyncError: message,
+ },
+ });
+ logger.error({ err: error, projectId }, 'GSC sync failed');
+ throw error;
+ }
+}
+
+/** The `gscProjectBackfill` job body: chunked to avoid timeouts and API limits. */
+export async function runGscProjectBackfill(
+ projectId: string,
+ logger: Logger = getLogger()
+): Promise<void> {
+ const db = await loadDb();
+ const conn = await db.gscConnection.findUnique({ where: { projectId } });
+ if (!conn?.siteUrl) {
+ logger.warn(
+ { projectId },
+ 'GSC backfill skipped: no connection or siteUrl'
+ );
+ return;
+ }
+
+ await db.gscConnection.update({
+ where: { projectId },
+ data: { backfillStatus: 'running' },
+ });
+
+ try {
+ const endDate = new Date();
+ endDate.setDate(endDate.getDate() - 1); // yesterday
+
+ const startDate = new Date(endDate);
+ startDate.setMonth(startDate.getMonth() - BACKFILL_MONTHS);
+
+ // Process in chunks to avoid timeouts and respect API limits
+ let chunkEnd = new Date(endDate);
+ while (chunkEnd > startDate) {
+ const chunkStart = new Date(chunkEnd);
+ chunkStart.setDate(chunkStart.getDate() - CHUNK_DAYS + 1);
+ if (chunkStart < startDate) {
+ chunkStart.setTime(startDate.getTime());
+ }
+
+ logger.info(
+ {
+ projectId,
+ from: chunkStart.toISOString().slice(0, 10),
+ to: chunkEnd.toISOString().slice(0, 10),
+ },
+ 'GSC backfill chunk'
+ );
+
+ await syncGscData(projectId, chunkStart, chunkEnd);
+
+ chunkEnd = new Date(chunkStart);
+ chunkEnd.setDate(chunkEnd.getDate() - 1);
+ }
+
+ await db.gscConnection.update({
+ where: { projectId },
+ data: {
+ backfillStatus: 'completed',
+ lastSyncedAt: new Date(),
+ lastSyncStatus: 'success',
+ lastSyncError: null,
+ },
+ });
+ logger.info({ projectId }, 'GSC backfill completed');
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ await db.gscConnection.update({
+ where: { projectId },
+ data: {
+ backfillStatus: 'failed',
+ lastSyncStatus: 'error',
+ lastSyncError: message,
+ },
+ });
+ logger.error({ err: error, projectId }, 'GSC backfill failed');
+ throw error;
+ }
+}
+
+// ---------------------------------------------------------------------------
+// OAuth callback — moved from apps/api/src/controllers/gsc-oauth-callback.
+// controller.ts's business logic (M5-002). Cookie reading/signing and the
+// redirect stay with each transport (Fastify today, Elysia's gsc.routes.ts
+// once it is live) — this is the part that is transport-agnostic.
+// ---------------------------------------------------------------------------
+
+export interface GscOAuthCallbackInput {
+ code: string;
+ state: string;
+ storedState: string;
+ codeVerifier: string;
+ projectId: string;
+}
+
+export interface GscOAuthCallbackResult {
+ organizationId: string;
+}
+
+export async function completeGscOAuthCallback(
+ input: GscOAuthCallbackInput
+): Promise<GscOAuthCallbackResult> {
+ if (input.state !== input.storedState) {
+ throw new Error('GSC OAuth state mismatch');
+ }
+
+ const tokens = await googleGsc.validateAuthorizationCode(
+ input.code,
+ input.codeVerifier
+ );
+
+ const accessToken = tokens.accessToken();
+ const refreshToken = tokens.hasRefreshToken() ? tokens.refreshToken() : null;
+ const accessTokenExpiresAt = tokens.accessTokenExpiresAt();
+
+ if (!refreshToken) {
+ throw new Error('No refresh token returned from Google GSC OAuth');
+ }
+
+ const db = await loadDb();
+ const project = await db.project.findUnique({
+ where: { id: input.projectId },
+ select: { id: true, organizationId: true },
+ });
+
+ if (!project) {
+ throw new Error('Project not found for GSC connection');
+ }
+
+ await db.gscConnection.upsert({
+ where: { projectId: input.projectId },
+ create: {
+ projectId: input.projectId,
+ accessToken: encrypt(accessToken),
+ refreshToken: encrypt(refreshToken),
+ accessTokenExpiresAt,
+ siteUrl: '',
+ },
+ update: {
+ accessToken: encrypt(accessToken),
+ refreshToken: encrypt(refreshToken),
+ accessTokenExpiresAt,
+ lastSyncStatus: null,
+ lastSyncError: null,
+ },
+ });
+
+ return { organizationId: project.organizationId };
+}
+
+// ---------------------------------------------------------------------------
+// `ctx.services.gsc` binding.
+// ---------------------------------------------------------------------------
+
+export interface GscService {
+ getConnection(projectId: string): Promise<GscConnectionSummary | null>;
+ listSites(projectId: string): Promise<GscSite[]>;
+ /** Also enqueues the `gscProjectBackfill` job — same as V1's router. */
+ selectSite(projectId: string, siteUrl: string): Promise<void>;
+ disconnect(projectId: string): Promise<void>;
+ resolveDateRange(
+ projectId: string,
+ input: GscDateRangeInput
+ ): ReturnType<typeof resolveGscDateRange>;
+ getOverview: typeof getGscOverview;
+ getPreviousOverview: typeof getGscPreviousOverview;
+ getPages: typeof getGscPages;
+ getPageDetails: typeof getGscPageDetails;
+ getQueryDetails: typeof getGscQueryDetails;
+ getQueries: typeof getGscQueries;
+ getSearchEngines: typeof getGscSearchEngines;
+ getAiEngines: typeof getGscAiEngines;
+ getCannibalization: typeof getGscCannibalization;
+ listConnectionsForSync(): Promise<{ projectId: string }[]>;
+ runProjectSync(projectId: string): Promise<void>;
+ runProjectBackfill(projectId: string): Promise<void>;
+}
+
+/** `ctx.services.gsc` — a thin binding of the functions above to a Ctx's logger/queues. */
+export function createGscService(deps: ServiceDeps): GscService {
+ const logger = deps.logger.child({ module: 'gsc' });
+
+ return {
+ getConnection: getGscConnection,
+ listSites: listGscSites,
+ selectSite: async (projectId, siteUrl) => {
+ await selectGscSite(projectId, siteUrl);
+ await deps.queues.gsc.gscProjectBackfill.add({ projectId });
+ },
+ disconnect: disconnectGscConnection,
+ resolveDateRange: resolveGscDateRange,
+ getOverview: getGscOverview,
+ getPreviousOverview: getGscPreviousOverview,
+ getPages: getGscPages,
+ getPageDetails: getGscPageDetails,
+ getQueryDetails: getGscQueryDetails,
+ getQueries: getGscQueries,
+ getSearchEngines: getGscSearchEngines,
+ getAiEngines: getGscAiEngines,
+ getCannibalization: getGscCannibalization,
+ listConnectionsForSync: listGscConnectionsForSync,
+ runProjectSync: (projectId) => runGscProjectSync(projectId, logger),
+ runProjectBackfill: (projectId) => runGscProjectBackfill(projectId, logger),
+ };
+}
diff --git a/packages/core/src/modules/gsc/src/access.ts b/packages/core/src/modules/gsc/src/access.ts
new file mode 100644
index 000000000..de4a4168e
--- /dev/null
+++ b/packages/core/src/modules/gsc/src/access.ts
@@ -0,0 +1,21 @@
+// Binds core's shared/access.ts ladder to @openpanel/db's real lookups, for
+// this module's own router — the same binding packages/trpc/src/access.ts
+// does for V1, self-contained here because the general core -> db binding
+// itself moves with auth (P6), not this wave.
+
+import {
+ canWriteProject,
+ getOrganizationAccess,
+ getProjectAccess,
+} from '@openpanel/db/src/services/access.service';
+import { getProjectById } from '@openpanel/db/src/services/project.service';
+import { createAccessChecks } from '../../../shared/access';
+
+const checks = createAccessChecks({
+ getProjectAccess,
+ canWriteProject,
+ getOrganizationAccess,
+ getProjectById,
+});
+
+export const requireProjectAccess = checks.requireProjectAccess;
diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.ts
index a8c9034a1..e02e69597 100644
--- a/packages/core/src/modules/insight/insight.jobs.test.ts
+++ b/packages/core/src/modules/insight/insight.jobs.test.ts
@@ -39,6 +39,7 @@ function stubJobCtx(
const services: Services = {
auth: {} as Services['auth'],
insight: insight as InsightService,
+ gsc: {} as Services['gsc'],
};
return {
db: {},
@@ -62,8 +63,11 @@ test('the insight jobs registry declares insightsProject on the insights queue',
});
test('the insight cron fragment is spread into the cron queue', () => {
- expect(Object.keys(queues.cron.jobs).sort()).toEqual(
- ['insightCleanup', 'insightsDaily', 'weeklyDigest'].sort()
+ // Subset, not exact equality: other modules (e.g. gsc, M5-002) spread
+ // their own fragments into the same cron queue. jobs.registry.test.ts owns
+ // the exhaustive membership check.
+ expect(Object.keys(queues.cron.jobs)).toEqual(
+ expect.arrayContaining(['insightCleanup', 'insightsDaily', 'weeklyDigest'])
);
});
@@ -113,6 +117,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId de
{ projectId: 'p2', date },
],
} as InsightService,
+ gsc: {} as Services['gsc'],
},
requestId: 'req_1',
job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'insightsDaily' },
diff --git a/packages/core/src/modules/insight/insight.service.test.ts b/packages/core/src/modules/insight/insight.service.test.ts
index 663aa7f37..2fb12340a 100644
--- a/packages/core/src/modules/insight/insight.service.test.ts
+++ b/packages/core/src/modules/insight/insight.service.test.ts
@@ -22,7 +22,17 @@ mock.module('@openpanel/db/src/prisma-client', () => ({
db: { $executeRaw },
}));
-mock.module('@openpanel/db/src/clickhouse/client', () => ({ ch: {} }));
+// `originalCh`/`chQuery`/`TABLE_NAMES` are unused here but included because
+// `mock.module` replaces this specifier process-wide (bun runs every test
+// file in one shared module registry without `--isolate` — see AGENTS.md) —
+// gsc.service.test.ts mocks the same path, so both factories must be a
+// superset of every consumer's needs, whichever one ends up registered last.
+mock.module('@openpanel/db/src/clickhouse/client', () => ({
+ ch: {},
+ originalCh: { query: mock(async () => ({ json: async () => [] })), insert: mock(async () => undefined) },
+ chQuery: mock(async () => []),
+ TABLE_NAMES: { sessions: 'sessions' },
+}));
const spikesQuery = mock(async () => [
{ anchorDate: '2026-09-01', spikes: [] },
@@ -58,7 +68,12 @@ const getRedisCache = mock(() => ({
return Promise.resolve();
}),
}));
-mock.module('@openpanel/redis', () => ({ getRedisCache }));
+// `cacheable` is unused here but included for the same cross-file
+// mock.module reason as the clickhouse/client mock above.
+mock.module('@openpanel/redis', () => ({
+ getRedisCache,
+ cacheable: <T>(fn: T) => fn,
+}));
let subject: typeof import('./insight.service');
beforeAll(async () => {
diff --git a/packages/core/src/rest.routes.test.ts b/packages/core/src/rest.routes.test.ts
index 851596492..a86620605 100644
--- a/packages/core/src/rest.routes.test.ts
+++ b/packages/core/src/rest.routes.test.ts
@@ -21,7 +21,7 @@ test('opsRoutes answers a liveness probe through requestContext', async () => {
expect(await response.json()).toEqual({ live: true });
});
-test('publicApiRoutes and dashboardRoutes compose with no modules yet', async () => {
+test('publicApiRoutes and dashboardRoutes compose, dashboardRoutes carrying gsc', async () => {
const { deps } = stubAppDeps();
const app = new Elysia()
diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.ts
index 89df30d08..6a1cd109d 100644
--- a/packages/core/src/rest.routes.ts
+++ b/packages/core/src/rest.routes.ts
@@ -8,6 +8,7 @@ import { openapi } from '@elysiajs/openapi';
import { Elysia } from 'elysia';
import type { AppDeps } from './context';
import { registry } from './metrics';
+import { gscRoutes } from './modules/gsc/gsc.routes';
import { healthRoutes } from './modules/health/health.routes';
// ADR-002/ADR-003 pin: @elysiajs/openapi at the 1.4.15 fallback (2.0 is
@@ -19,16 +20,19 @@ const OPENAPI_SPEC_PATH = '/openapi.json';
// endpoint (ADR-003: "the transform that hides /metrics from the spec").
const OPENAPI_EXCLUDED_PATHS = ['/metrics'];
-// No module lands on either surface yet — the 35 modules' HTTP halves land
-// with their waves (P5-P8). Both still take `deps` now so a module addition
+// Both surfaces still take `deps` even where unused so a module addition
// never changes the factory's signature, only its body. Return types are
// left inferred, as `defineRoutes`'s `RouteApp` is: `.use()`'d plugins widen
// Elysia's type parameters in a way the bare `Elysia` type cannot express.
export const publicApiRoutes = (_deps: AppDeps) =>
new Elysia({ name: 'core/public-api-routes' });
-export const dashboardRoutes = (_deps: AppDeps) =>
- new Elysia({ name: 'core/dashboard-routes' });
+// gsc's callback is the first module to land here (M5-002). It is not yet
+// reachable in production: main.ts only mounts `dashboardRoutes` once a real
+// `AppDeps` exists (P3/P4/P8) — see gsc.routes.ts's header for the named gap
+// that keeps it inert until then.
+export const dashboardRoutes = (deps: AppDeps) =>
+ new Elysia({ name: 'core/dashboard-routes' }).use(gscRoutes(deps));
// healthz/metrics/misc share V1's ops surface (http/context.ts's
// UNLOGGED_PATH_PREFIXES) — unauthenticated, uncorsed, unlogged. /metrics and
diff --git a/packages/core/src/rpc.router.ts b/packages/core/src/rpc.router.ts
index 317665686..4c2a3ddb5 100644
--- a/packages/core/src/rpc.router.ts
+++ b/packages/core/src/rpc.router.ts
@@ -4,6 +4,7 @@
// (ADR-007 decision 19). Only `health` exists so far; the 28 real routers
// land with their module waves (P5-P8).
+import { gscRouter } from './modules/gsc/gsc.rpc';
import { healthRouter } from './modules/health/health.rpc';
import { insightRouter } from './modules/insight/insight.rpc';
import { createTRPCRouter } from './rpc/base';
@@ -11,6 +12,7 @@ import { createTRPCRouter } from './rpc/base';
export const appRouter = createTRPCRouter({
health: healthRouter,
insight: insightRouter,
+ gsc: gscRouter,
});
export type AppRouter = typeof appRouter;
diff --git a/packages/core/src/services.ts b/packages/core/src/services.ts
index 2c049de16..5284c1dd1 100644
--- a/packages/core/src/services.ts
+++ b/packages/core/src/services.ts
@@ -7,6 +7,7 @@ import {
type AuthService,
createAuthService,
} from './modules/auth/auth.service';
+import { createGscService, type GscService } from './modules/gsc/gsc.service';
import {
createInsightService,
type InsightService,
@@ -30,6 +31,7 @@ export type ServiceDeps = Pick<
export interface Services {
auth: AuthService;
insight: InsightService;
+ gsc: GscService;
}
export function createServices(deps: ServiceDeps): Services {
@@ -39,5 +41,6 @@ export function createServices(deps: ServiceDeps): Services {
return {
auth: createAuthService(deps),
insight: createInsightService(deps),
+ gsc: createGscService(deps),
};
}
diff --git a/packages/db/src/gsc.ts b/packages/db/src/gsc.ts
index c23a98a86..3adbd1268 100644
--- a/packages/db/src/gsc.ts
+++ b/packages/db/src/gsc.ts
@@ -1,653 +1,15 @@
-import { cacheable } from '@openpanel/redis';
-import { originalCh } from './clickhouse/client';
-import { decrypt, encrypt } from './encryption';
-import { createLogger } from './logger';
-import { db } from './prisma-client';
-
-const logger = createLogger({ name: 'db:gsc' });
-
-export interface GscSite {
- siteUrl: string;
- permissionLevel: string;
-}
-
-async function refreshGscToken(
- refreshToken: string
-): Promise<{ accessToken: string; expiresAt: Date }> {
- if (!(process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET)) {
- throw new Error(
- 'GOOGLE_CLIENT_ID or GOOGLE_CLIENT_SECRET is not set in this environment'
- );
- }
-
- const params = new URLSearchParams({
- client_id: process.env.GOOGLE_CLIENT_ID,
- client_secret: process.env.GOOGLE_CLIENT_SECRET,
- refresh_token: refreshToken,
- grant_type: 'refresh_token',
- });
-
- const res = await fetch('https://oauth2.googleapis.com/token', {
- method: 'POST',
- headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
- body: params.toString(),
- });
-
- if (!res.ok) {
- const text = await res.text();
- throw new Error(`Failed to refresh GSC token: ${text}`);
- }
-
- const data = (await res.json()) as {
- access_token: string;
- expires_in: number;
- };
- const expiresAt = new Date(Date.now() + data.expires_in * 1000);
- return { accessToken: data.access_token, expiresAt };
-}
-
-export async function getGscAccessToken(projectId: string): Promise<string> {
- const conn = await db.gscConnection.findUniqueOrThrow({
- where: { projectId },
- });
-
- if (
- conn.accessTokenExpiresAt &&
- conn.accessTokenExpiresAt.getTime() > Date.now() + 60_000
- ) {
- logger.info(
- { projectId, expiresAt: conn.accessTokenExpiresAt },
- 'GSC using cached access token'
- );
- return decrypt(conn.accessToken);
- }
-
- logger.info(
- {
- projectId,
- expiresAt: conn.accessTokenExpiresAt,
- hasRefreshToken: !!conn.refreshToken,
- },
- 'GSC access token expired, attempting refresh'
- );
-
- try {
- const { accessToken, expiresAt } = await refreshGscToken(
- decrypt(conn.refreshToken)
- );
- await db.gscConnection.update({
- where: { projectId },
- data: {
- accessToken: encrypt(accessToken),
- accessTokenExpiresAt: expiresAt,
- },
- });
- logger.info({ projectId, expiresAt }, 'GSC token refreshed successfully');
- return accessToken;
- } catch (error) {
- const errorMessage =
- error instanceof Error ? error.message : 'Failed to refresh token';
- logger.error(
- { err: error, projectId, errorMessage },
- 'GSC token refresh failed'
- );
- await db.gscConnection.update({
- where: { projectId },
- data: {
- lastSyncStatus: 'token_expired',
- lastSyncError: errorMessage,
- },
- });
- throw new Error(
- `GSC token refresh failed for project ${projectId}: ${errorMessage}`
- );
- }
-}
-
-export async function listGscSites(projectId: string): Promise<GscSite[]> {
- const accessToken = await getGscAccessToken(projectId);
- const res = await fetch('https://www.googleapis.com/webmasters/v3/sites', {
- headers: { Authorization: `Bearer ${accessToken}` },
- });
-
- if (!res.ok) {
- const text = await res.text();
- throw new Error(`Failed to list GSC sites: ${text}`);
- }
-
- const data = (await res.json()) as {
- siteEntry?: Array<{ siteUrl: string; permissionLevel: string }>;
- };
- return data.siteEntry ?? [];
-}
-
-interface GscApiRow {
- keys: string[];
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
-}
-
-interface GscDimensionFilter {
- dimension: string;
- operator: string;
- expression: string;
-}
-
-interface GscFilterGroup {
- filters: GscDimensionFilter[];
-}
-
-async function queryGscSearchAnalytics(
- accessToken: string,
- siteUrl: string,
- startDate: string,
- endDate: string,
- dimensions: string[],
- dimensionFilterGroups?: GscFilterGroup[]
-): Promise<GscApiRow[]> {
- const encodedSiteUrl = encodeURIComponent(siteUrl);
- const url = `https://www.googleapis.com/webmasters/v3/sites/${encodedSiteUrl}/searchAnalytics/query`;
-
- const allRows: GscApiRow[] = [];
- let startRow = 0;
- const rowLimit = 25_000;
-
- while (true) {
- const res = await fetch(url, {
- method: 'POST',
- headers: {
- Authorization: `Bearer ${accessToken}`,
- 'Content-Type': 'application/json',
- },
- body: JSON.stringify({
- startDate,
- endDate,
- dimensions,
- rowLimit,
- startRow,
- dataState: 'all',
- ...(dimensionFilterGroups && { dimensionFilterGroups }),
- }),
- });
-
- if (!res.ok) {
- const text = await res.text();
- throw new Error(
- `GSC query failed for dimensions [${dimensions.join(',')}]: ${text}`
- );
- }
-
- const data = (await res.json()) as { rows?: GscApiRow[] };
- const rows = data.rows ?? [];
- allRows.push(...rows);
-
- if (rows.length < rowLimit) {
- break;
- }
- startRow += rowLimit;
- }
-
- return allRows;
-}
-
-function formatDate(date: Date): string {
- return date.toISOString().slice(0, 10);
-}
-
-function nowString(): string {
- return new Date().toISOString().replace('T', ' ').replace('Z', '');
-}
-
-export async function syncGscData(
- projectId: string,
- startDate: Date,
- endDate: Date
-): Promise<void> {
- const conn = await db.gscConnection.findUniqueOrThrow({
- where: { projectId },
- });
-
- if (!conn.siteUrl) {
- throw new Error('No GSC site URL configured for this project');
- }
-
- const accessToken = await getGscAccessToken(projectId);
- const start = formatDate(startDate);
- const end = formatDate(endDate);
- const syncedAt = nowString();
-
- // 1. Daily totals — authoritative numbers for overview chart
- const dailyRows = await queryGscSearchAnalytics(
- accessToken,
- conn.siteUrl,
- start,
- end,
- ['date']
- );
-
- if (dailyRows.length > 0) {
- await originalCh.insert({
- table: 'gsc_daily',
- values: dailyRows.map((row) => ({
- project_id: projectId,
- date: row.keys[0] ?? '',
- clicks: row.clicks,
- impressions: row.impressions,
- ctr: row.ctr,
- position: row.position,
- synced_at: syncedAt,
- })),
- format: 'JSONEachRow',
- });
- }
-
- // 2. Per-page breakdown
- const pageRows = await queryGscSearchAnalytics(
- accessToken,
- conn.siteUrl,
- start,
- end,
- ['date', 'page']
- );
-
- if (pageRows.length > 0) {
- await originalCh.insert({
- table: 'gsc_pages_daily',
- values: pageRows.map((row) => ({
- project_id: projectId,
- date: row.keys[0] ?? '',
- page: row.keys[1] ?? '',
- clicks: row.clicks,
- impressions: row.impressions,
- ctr: row.ctr,
- position: row.position,
- synced_at: syncedAt,
- })),
- format: 'JSONEachRow',
- });
- }
-
- // 3. Per-query breakdown
- const queryRows = await queryGscSearchAnalytics(
- accessToken,
- conn.siteUrl,
- start,
- end,
- ['date', 'query']
- );
-
- if (queryRows.length > 0) {
- await originalCh.insert({
- table: 'gsc_queries_daily',
- values: queryRows.map((row) => ({
- project_id: projectId,
- date: row.keys[0] ?? '',
- query: row.keys[1] ?? '',
- clicks: row.clicks,
- impressions: row.impressions,
- ctr: row.ctr,
- position: row.position,
- synced_at: syncedAt,
- })),
- format: 'JSONEachRow',
- });
- }
-}
-
-export async function getGscOverview(
- projectId: string,
- startDate: string,
- endDate: string,
- interval: 'day' | 'week' | 'month' = 'day'
-): Promise<
- Array<{
- date: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- }>
-> {
- const dateExpr =
- interval === 'month'
- ? 'toStartOfMonth(date)'
- : interval === 'week'
- ? 'toStartOfWeek(date)'
- : 'date';
-
- const result = await originalCh.query({
- query: `
- SELECT
- ${dateExpr} as date,
- sum(clicks) as clicks,
- sum(impressions) as impressions,
- avg(ctr) as ctr,
- avg(position) as position
- FROM gsc_daily
- FINAL
- WHERE project_id = {projectId: String}
- AND date >= {startDate: String}
- AND date <= {endDate: String}
- GROUP BY date
- ORDER BY date ASC
- `,
- query_params: { projectId, startDate, endDate },
- format: 'JSONEachRow',
- });
- return result.json();
-}
-
-export async function getGscPages(
- projectId: string,
- startDate: string,
- endDate: string,
- limit = 100
-): Promise<
- Array<{
- page: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- }>
-> {
- const result = await originalCh.query({
- query: `
- SELECT
- page,
- sum(clicks) as clicks,
- sum(impressions) as impressions,
- avg(ctr) as ctr,
- avg(position) as position
- FROM gsc_pages_daily
- FINAL
- WHERE project_id = {projectId: String}
- AND date >= {startDate: String}
- AND date <= {endDate: String}
- GROUP BY page
- ORDER BY clicks DESC
- LIMIT {limit: UInt32}
- `,
- query_params: { projectId, startDate, endDate, limit },
- format: 'JSONEachRow',
- });
- return result.json();
-}
-
-export interface GscCannibalizedQuery {
- query: string;
- totalImpressions: number;
- totalClicks: number;
- pages: Array<{
- page: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- }>;
-}
-
-export const getGscCannibalization = cacheable(
- async (
- projectId: string,
- startDate: string,
- endDate: string
- ): Promise<GscCannibalizedQuery[]> => {
- const conn = await db.gscConnection.findUniqueOrThrow({
- where: { projectId },
- });
- const accessToken = await getGscAccessToken(projectId);
-
- const rows = await queryGscSearchAnalytics(
- accessToken,
- conn.siteUrl,
- startDate,
- endDate,
- ['query', 'page']
- );
-
- const map = new Map<
- string,
- {
- totalImpressions: number;
- totalClicks: number;
- pages: GscCannibalizedQuery['pages'];
- }
- >();
-
- for (const row of rows) {
- const query = row.keys[0] ?? '';
- // Strip hash fragments — GSC records heading anchors (e.g. /page#section)
- // as separate URLs but Google treats them as the same page
- let page = row.keys[1] ?? '';
- try {
- const u = new URL(page);
- u.hash = '';
- page = u.toString();
- } catch {
- page = page.split('#')[0] ?? page;
- }
-
- const entry = map.get(query) ?? {
- totalImpressions: 0,
- totalClicks: 0,
- pages: [],
- };
- entry.totalImpressions += row.impressions;
- entry.totalClicks += row.clicks;
- // Merge into existing page entry if already seen (from a different hash variant)
- const existing = entry.pages.find((p) => p.page === page);
- if (existing) {
- const totalImpressions = existing.impressions + row.impressions;
- if (totalImpressions > 0) {
- existing.position =
- (existing.position * existing.impressions +
- row.position * row.impressions) /
- totalImpressions;
- }
- existing.clicks += row.clicks;
- existing.impressions += row.impressions;
- existing.ctr =
- existing.impressions > 0 ? existing.clicks / existing.impressions : 0;
- } else {
- entry.pages.push({
- page,
- clicks: row.clicks,
- impressions: row.impressions,
- ctr: row.ctr,
- position: row.position,
- });
- }
- map.set(query, entry);
- }
-
- return [...map.entries()]
- .filter(([, v]) => v.pages.length >= 2 && v.totalImpressions >= 100)
- .sort(([, a], [, b]) => b.totalImpressions - a.totalImpressions)
- .slice(0, 50)
- .map(([query, v]) => ({
- query,
- totalImpressions: v.totalImpressions,
- totalClicks: v.totalClicks,
- pages: v.pages.sort((a, b) =>
- a.position !== b.position
- ? a.position - b.position
- : b.impressions - a.impressions
- ),
- }));
- },
- 60 * 60 * 4
-);
-
-export async function getGscPageDetails(
- projectId: string,
- page: string,
- startDate: string,
- endDate: string
-): Promise<{
- timeseries: Array<{
- date: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- }>;
- queries: Array<{
- query: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- }>;
-}> {
- const conn = await db.gscConnection.findUniqueOrThrow({
- where: { projectId },
- });
- const accessToken = await getGscAccessToken(projectId);
- const filterGroups: GscFilterGroup[] = [
- { filters: [{ dimension: 'page', operator: 'equals', expression: page }] },
- ];
-
- const [timeseriesRows, queryRows] = await Promise.all([
- queryGscSearchAnalytics(
- accessToken,
- conn.siteUrl,
- startDate,
- endDate,
- ['date'],
- filterGroups
- ),
- queryGscSearchAnalytics(
- accessToken,
- conn.siteUrl,
- startDate,
- endDate,
- ['query'],
- filterGroups
- ),
- ]);
-
- return {
- timeseries: timeseriesRows.map((row) => ({
- date: row.keys[0] ?? '',
- clicks: row.clicks,
- impressions: row.impressions,
- ctr: row.ctr,
- position: row.position,
- })),
- queries: queryRows.map((row) => ({
- query: row.keys[0] ?? '',
- clicks: row.clicks,
- impressions: row.impressions,
- ctr: row.ctr,
- position: row.position,
- })),
- };
-}
-
-export async function getGscQueryDetails(
- projectId: string,
- query: string,
- startDate: string,
- endDate: string
-): Promise<{
- timeseries: Array<{
- date: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- }>;
- pages: Array<{
- page: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- }>;
-}> {
- const conn = await db.gscConnection.findUniqueOrThrow({
- where: { projectId },
- });
- const accessToken = await getGscAccessToken(projectId);
- const filterGroups: GscFilterGroup[] = [
- {
- filters: [{ dimension: 'query', operator: 'equals', expression: query }],
- },
- ];
-
- const [timeseriesRows, pageRows] = await Promise.all([
- queryGscSearchAnalytics(
- accessToken,
- conn.siteUrl,
- startDate,
- endDate,
- ['date'],
- filterGroups
- ),
- queryGscSearchAnalytics(
- accessToken,
- conn.siteUrl,
- startDate,
- endDate,
- ['page'],
- filterGroups
- ),
- ]);
-
- return {
- timeseries: timeseriesRows.map((row) => ({
- date: row.keys[0] ?? '',
- clicks: row.clicks,
- impressions: row.impressions,
- ctr: row.ctr,
- position: row.position,
- })),
- pages: pageRows.map((row) => ({
- page: row.keys[0] ?? '',
- clicks: row.clicks,
- impressions: row.impressions,
- ctr: row.ctr,
- position: row.position,
- })),
- };
-}
-
-export async function getGscQueries(
- projectId: string,
- startDate: string,
- endDate: string,
- limit = 100
-): Promise<
- Array<{
- query: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- }>
-> {
- const result = await originalCh.query({
- query: `
- SELECT
- query,
- sum(clicks) as clicks,
- sum(impressions) as impressions,
- avg(ctr) as ctr,
- avg(position) as position
- FROM gsc_queries_daily
- FINAL
- WHERE project_id = {projectId: String}
- AND date >= {startDate: String}
- AND date <= {endDate: String}
- GROUP BY query
- ORDER BY clicks DESC
- LIMIT {limit: UInt32}
- `,
- query_params: { projectId, startDate, endDate, limit },
- format: 'JSONEachRow',
- });
- return result.json();
-}
+// The GSC OAuth token lifecycle, the Search Console API client and its
+// ClickHouse read/write live in @openpanel/core now (M5-002). Re-exported
+// here for existing `@openpanel/db` importers (packages/mcp's gsc tools) —
+// same shape as packages/db/src/encryption.ts since M4-006.
+export type { GscCannibalizedQuery, GscSite } from '@openpanel/core';
+export {
+ getGscCannibalization,
+ getGscOverview,
+ getGscPageDetails,
+ getGscPages,
+ getGscQueries,
+ getGscQueryDetails,
+ listGscSites,
+ syncGscData,
+} from '@openpanel/core';
diff --git a/packages/db/src/services/gsc.service.ts b/packages/db/src/services/gsc.service.ts
index 8314ef361..75b8e28f4 100644
--- a/packages/db/src/services/gsc.service.ts
+++ b/packages/db/src/services/gsc.service.ts
@@ -1,194 +1,14 @@
-import { getGscCannibalization, getGscOverview, getGscPageDetails, getGscPages, getGscQueryDetails, getGscQueries } from '../gsc';
-
-export interface GscQueryOpportunity {
- query: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- opportunity_score: number;
- reason: string;
-}
-
-function computeOpportunities(
- queries: Array<{
- query: string;
- clicks: number;
- impressions: number;
- ctr: number;
- position: number;
- }>,
-): GscQueryOpportunity[] {
- const ctrBenchmarks: Record<string, number> = {
- '1': 0.28,
- '2': 0.15,
- '3': 0.11,
- '4-6': 0.065,
- '7-10': 0.035,
- '11-20': 0.012,
- };
-
- function getBenchmark(position: number): number {
- if (position <= 1) return ctrBenchmarks['1'] ?? 0.28;
- if (position <= 2) return ctrBenchmarks['2'] ?? 0.15;
- if (position <= 3) return ctrBenchmarks['3'] ?? 0.11;
- if (position <= 6) return ctrBenchmarks['4-6'] ?? 0.065;
- if (position <= 10) return ctrBenchmarks['7-10'] ?? 0.035;
- return ctrBenchmarks['11-20'] ?? 0.012;
- }
-
- return queries
- .filter((q) => q.position >= 4 && q.position <= 20 && q.impressions >= 50)
- .map((q) => {
- const benchmark = getBenchmark(q.position);
- const ctrGap = Math.max(0, benchmark - q.ctr);
- const opportunity_score =
- Math.round(q.impressions * (1 / q.position) * (1 + ctrGap) * 100) /
- 100;
-
- let reason: string;
- if (q.position <= 6) {
- reason = `Position ${q.position.toFixed(1)} — one rank improvement could significantly boost clicks`;
- } else if (q.ctr < benchmark * 0.5) {
- reason = `CTR (${(q.ctr * 100).toFixed(1)}%) is well below expected ${(benchmark * 100).toFixed(1)}% — title/meta optimization may help`;
- } else {
- reason = `Position ${q.position.toFixed(1)} with ${q.impressions} impressions — push to page 1 for major gains`;
- }
-
- return {
- query: q.query,
- clicks: q.clicks,
- impressions: q.impressions,
- ctr: Math.round(q.ctr * 10000) / 100,
- position: Math.round(q.position * 10) / 10,
- opportunity_score,
- reason,
- };
- })
- .sort((a, b) => b.opportunity_score - a.opportunity_score)
- .slice(0, 50);
-}
-
-export async function gscGetOverviewCore(input: {
- projectId: string;
- startDate: string;
- endDate: string;
- interval?: 'day' | 'week' | 'month';
-}) {
- const data = await getGscOverview(
- input.projectId,
- input.startDate,
- input.endDate,
- input.interval ?? 'day',
- );
- return {
- data,
- summary: {
- total_clicks: data.reduce((s, r) => s + r.clicks, 0),
- total_impressions: data.reduce((s, r) => s + r.impressions, 0),
- avg_ctr:
- data.length > 0
- ? Math.round(
- (data.reduce((s, r) => s + r.ctr, 0) / data.length) * 10000,
- ) / 100
- : 0,
- avg_position:
- data.length > 0
- ? Math.round(
- (data.reduce((s, r) => s + r.position, 0) / data.length) * 10,
- ) / 10
- : 0,
- },
- };
-}
-
-export async function gscGetTopPagesCore(input: {
- projectId: string;
- startDate: string;
- endDate: string;
- limit?: number;
-}) {
- return getGscPages(
- input.projectId,
- input.startDate,
- input.endDate,
- input.limit ?? 100,
- );
-}
-
-export async function gscGetPageDetailsCore(input: {
- projectId: string;
- startDate: string;
- endDate: string;
- page: string;
-}) {
- return getGscPageDetails(
- input.projectId,
- input.page,
- input.startDate,
- input.endDate,
- );
-}
-
-export async function gscGetTopQueriesCore(input: {
- projectId: string;
- startDate: string;
- endDate: string;
- limit?: number;
-}) {
- return getGscQueries(
- input.projectId,
- input.startDate,
- input.endDate,
- input.limit ?? 100,
- );
-}
-
-export async function gscGetQueryOpportunitiesCore(input: {
- projectId: string;
- startDate: string;
- endDate: string;
- minImpressions?: number;
-}) {
- const queries = await getGscQueries(
- input.projectId,
- input.startDate,
- input.endDate,
- 5000,
- );
- const filtered = queries.filter(
- (q) => q.impressions >= (input.minImpressions ?? 50),
- );
- const opportunities = computeOpportunities(filtered);
- return {
- opportunities,
- total_analyzed: filtered.length,
- min_impressions: input.minImpressions ?? 50,
- };
-}
-
-export async function gscGetQueryDetailsCore(input: {
- projectId: string;
- startDate: string;
- endDate: string;
- query: string;
-}) {
- return getGscQueryDetails(
- input.projectId,
- input.query,
- input.startDate,
- input.endDate,
- );
-}
-
-export async function gscGetCannibalizationCore(input: {
- projectId: string;
- startDate: string;
- endDate: string;
-}) {
- return getGscCannibalization(
- input.projectId,
- input.startDate,
- input.endDate,
- );
-}
+// The AI-tool wrapper functions live in @openpanel/core now (M5-002).
+// Re-exported here for existing `@openpanel/db` importers (the assistant's
+// SEO tools, the /insights REST controller) — same shape as
+// packages/db/src/encryption.ts since M4-006.
+export type { GscQueryOpportunity } from '@openpanel/core';
+export {
+ gscGetCannibalizationCore,
+ gscGetOverviewCore,
+ gscGetPageDetailsCore,
+ gscGetQueryDetailsCore,
+ gscGetQueryOpportunitiesCore,
+ gscGetTopPagesCore,
+ gscGetTopQueriesCore,
+} from '@openpanel/core';
diff --git a/packages/trpc/src/routers/gsc.ts b/packages/trpc/src/routers/gsc.ts
index d1a732aff..d86bd6df9 100644
--- a/packages/trpc/src/routers/gsc.ts
+++ b/packages/trpc/src/routers/gsc.ts
@@ -1,25 +1,38 @@
-import { Arctic, googleGsc } from '@openpanel/core';
+// Dissolved into @openpanel/core's gsc module (M5-002): the OAuth token
+// lifecycle, the Search Console client, the ClickHouse read path and the
+// AI/search-engine breakdown logic moved to
+// packages/core/src/modules/gsc/gsc.service.ts. This router stays
+// (DELEGATE PATTERN): it keeps V1's protectedProcedure stack (session/access/
+// logger/rate-limit middleware) and its own gscConnection CRUD untouched, and
+// delegates every read/write of GSC data to the core service.
import {
- chQuery,
- db,
- getChartStartEndDate,
+ Arctic,
+ disconnectGscConnection,
+ getGscAiEngines,
getGscCannibalization,
+ getGscConnection,
getGscOverview,
getGscPageDetails,
getGscPages,
+ getGscPreviousOverview,
getGscQueries,
getGscQueryDetails,
- getSettingsForProject,
+ getGscSearchEngines,
+ googleGsc,
listGscSites,
- TABLE_NAMES,
-} from '@openpanel/db';
+ resolveGscDateRange,
+ selectGscSite,
+} from '@openpanel/core';
import { gscQueue } from '@openpanel/queue';
import { zRange, zTimeInterval } from '@openpanel/validation';
import { z } from 'zod';
import { getProjectAccess, requireProjectAccess } from '../access';
-import { TRPCForbiddenError, TRPCNotFoundError } from '../errors';
+import { TRPCForbiddenError } from '../errors';
import { createTRPCRouter, protectedProcedure } from '../trpc';
+const OAUTH_COOKIE_MAX_AGE_SECONDS = 60 * 10;
+const GSC_OVERVIEW_INTERVALS = ['day', 'week', 'month'] as const;
+
const zGscDateInput = z.object({
projectId: z.string(),
range: zRange,
@@ -28,142 +41,27 @@ const zGscDateInput = z.object({
endDate: z.string().nullish(),
});
-async function resolveDates(
- projectId: string,
- input: { range: string; startDate?: string | null; endDate?: string | null }
-) {
- const { timezone } = await getSettingsForProject(projectId);
- const { startDate, endDate } = getChartStartEndDate(
- {
- range: input.range as any,
- startDate: input.startDate,
- endDate: input.endDate,
- },
- timezone
- );
- return {
- startDate: startDate.slice(0, 10),
- endDate: endDate.slice(0, 10),
- };
+function toOverviewInterval(
+ interval: string
+): (typeof GSC_OVERVIEW_INTERVALS)[number] {
+ return (GSC_OVERVIEW_INTERVALS as readonly string[]).includes(interval)
+ ? (interval as (typeof GSC_OVERVIEW_INTERVALS)[number])
+ : 'day';
}
-/**
- * ClickHouse stores the same AI referrer under several spellings: the parsed
- * display name ('ChatGPT', 'Google Gemini'), the bare host ('chatgpt.com') and
- * the full origin ('https://kagi.com'). Normalize before matching so every
- * spelling lands on the same engine.
- */
-const NORMALIZED_REFERRER_NAME =
- "lower(regexp_replace(referrer_name, '^https?://(www[.])?', ''))";
-
-const AI_REFERRERS = [
- {
- canonical: 'chatgpt.com',
- aliases: [
- 'chatgpt',
- 'chatgpt.com',
- 'chat.openai.com',
- 'openai',
- 'openai.com',
- ],
- },
- {
- canonical: 'claude.ai',
- aliases: ['claude', 'claude.ai', 'anthropic', 'anthropic.com'],
- },
- {
- canonical: 'perplexity.ai',
- aliases: ['perplexity', 'perplexity.ai'],
- },
- {
- canonical: 'gemini.google.com',
- aliases: [
- 'gemini',
- 'google gemini',
- 'gemini.google.com',
- 'bard.google.com',
- ],
- },
- {
- canonical: 'copilot.com',
- aliases: [
- 'copilot',
- 'copilot.com',
- 'copilot.microsoft.com',
- 'microsoft copilot',
- ],
- },
- { canonical: 'grok.com', aliases: ['grok', 'grok.com'] },
- {
- canonical: 'mistral.ai',
- aliases: ['mistral', 'mistral.ai', 'chat.mistral.ai', 'le chat'],
- },
- {
- canonical: 'kagi.com',
- aliases: ['kagi', 'kagi.com', 'assistant.kagi.com'],
- },
-] as const satisfies ReadonlyArray<{
- canonical: string;
- aliases: readonly string[];
-}>;
-
-const quoteList = (values: readonly string[]) =>
- values.map((value) => `'${value}'`).join(', ');
-
-/** `norm` is the alias bound by AI_REFERRER_CTE below. */
-const AI_REFERRER_CTE = `WITH ${NORMALIZED_REFERRER_NAME} AS norm`;
-
-const AI_REFERRER_FILTER = `norm IN (${quoteList(
- AI_REFERRERS.flatMap((engine) => engine.aliases)
-)})`;
-
-/** Collapses every alias onto one row per engine. */
-const AI_REFERRER_CANONICAL_NAME = `multiIf(${AI_REFERRERS.map(
- (engine) => `norm IN (${quoteList(engine.aliases)}), '${engine.canonical}'`
-).join(', ')}, norm)`;
-
-/**
- * Half-open windows so the last day of the range is included and the previous
- * window ends exactly where the current one starts (no gap, no overlap).
- */
-function getComparisonWindows(startDate: string, endDate: string) {
- const start = new Date(`${startDate}T00:00:00Z`);
- const endExclusive = new Date(`${endDate}T00:00:00Z`);
- endExclusive.setUTCDate(endExclusive.getUTCDate() + 1);
- const previousStart = new Date(
- start.getTime() - (endExclusive.getTime() - start.getTime())
- );
- const fmt = (date: Date) => date.toISOString().slice(0, 19).replace('T', ' ');
- return {
- current: { start: fmt(start), end: fmt(endExclusive) },
- previous: { start: fmt(previousStart), end: fmt(start) },
- };
+async function requireRead(userId: string, projectId: string) {
+ const access = await getProjectAccess({ userId, projectId });
+ if (!access) {
+ throw new TRPCForbiddenError('You do not have access to this project');
+ }
}
export const gscRouter = createTRPCRouter({
getConnection: protectedProcedure
.input(z.object({ projectId: z.string() }))
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- return db.gscConnection.findUnique({
- where: { projectId: input.projectId },
- select: {
- id: true,
- siteUrl: true,
- lastSyncedAt: true,
- lastSyncStatus: true,
- lastSyncError: true,
- backfillStatus: true,
- createdAt: true,
- updatedAt: true,
- },
- });
+ await requireRead(ctx.session.userId, input.projectId);
+ return getGscConnection(input.projectId);
}),
initiateOAuth: protectedProcedure
@@ -183,7 +81,7 @@ export const gscRouter = createTRPCRouter({
url.searchParams.set('access_type', 'offline');
url.searchParams.set('prompt', 'consent');
- const cookieOpts = { maxAge: 60 * 10, signed: true };
+ const cookieOpts = { maxAge: OAUTH_COOKIE_MAX_AGE_SECONDS, signed: true };
ctx.setCookie('gsc_oauth_state', state, cookieOpts);
ctx.setCookie('gsc_code_verifier', codeVerifier, cookieOpts);
ctx.setCookie('gsc_project_id', input.projectId, cookieOpts);
@@ -194,13 +92,7 @@ export const gscRouter = createTRPCRouter({
getSites: protectedProcedure
.input(z.object({ projectId: z.string() }))
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
+ await requireRead(ctx.session.userId, input.projectId);
return listGscSites(input.projectId);
}),
@@ -213,20 +105,7 @@ export const gscRouter = createTRPCRouter({
level: 'write',
});
- const conn = await db.gscConnection.findUnique({
- where: { projectId: input.projectId },
- });
- if (!conn) {
- throw new TRPCNotFoundError('GSC connection not found');
- }
-
- await db.gscConnection.update({
- where: { projectId: input.projectId },
- data: {
- siteUrl: input.siteUrl,
- backfillStatus: 'pending',
- },
- });
+ await selectGscSite(input.projectId, input.siteUrl);
await gscQueue.add('gscProjectBackfill', {
type: 'gscProjectBackfill',
@@ -245,28 +124,24 @@ export const gscRouter = createTRPCRouter({
level: 'write',
});
- await db.gscConnection.deleteMany({
- where: { projectId: input.projectId },
- });
-
+ await disconnectGscConnection(input.projectId);
return { ok: true };
}),
getOverview: protectedProcedure
.input(zGscDateInput)
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- const { startDate, endDate } = await resolveDates(input.projectId, input);
- const interval = ['day', 'week', 'month'].includes(input.interval)
- ? (input.interval as 'day' | 'week' | 'month')
- : 'day';
- return getGscOverview(input.projectId, startDate, endDate, interval);
+ await requireRead(ctx.session.userId, input.projectId);
+ const { startDate, endDate } = await resolveGscDateRange(
+ input.projectId,
+ input
+ );
+ return getGscOverview(
+ input.projectId,
+ startDate,
+ endDate,
+ toOverviewInterval(input.interval)
+ );
}),
getPages: protectedProcedure
@@ -276,42 +151,33 @@ export const gscRouter = createTRPCRouter({
})
)
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- const { startDate, endDate } = await resolveDates(input.projectId, input);
+ await requireRead(ctx.session.userId, input.projectId);
+ const { startDate, endDate } = await resolveGscDateRange(
+ input.projectId,
+ input
+ );
return getGscPages(input.projectId, startDate, endDate, input.limit);
}),
getPageDetails: protectedProcedure
.input(zGscDateInput.extend({ page: z.string() }))
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- const { startDate, endDate } = await resolveDates(input.projectId, input);
+ await requireRead(ctx.session.userId, input.projectId);
+ const { startDate, endDate } = await resolveGscDateRange(
+ input.projectId,
+ input
+ );
return getGscPageDetails(input.projectId, input.page, startDate, endDate);
}),
getQueryDetails: protectedProcedure
.input(zGscDateInput.extend({ query: z.string() }))
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- const { startDate, endDate } = await resolveDates(input.projectId, input);
+ await requireRead(ctx.session.userId, input.projectId);
+ const { startDate, endDate } = await resolveGscDateRange(
+ input.projectId,
+ input
+ );
return getGscQueryDetails(
input.projectId,
input.query,
@@ -327,157 +193,60 @@ export const gscRouter = createTRPCRouter({
})
)
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- const { startDate, endDate } = await resolveDates(input.projectId, input);
+ await requireRead(ctx.session.userId, input.projectId);
+ const { startDate, endDate } = await resolveGscDateRange(
+ input.projectId,
+ input
+ );
return getGscQueries(input.projectId, startDate, endDate, input.limit);
}),
getSearchEngines: protectedProcedure
.input(zGscDateInput)
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- const { startDate, endDate } = await resolveDates(input.projectId, input);
- const windows = getComparisonWindows(startDate, endDate);
-
- const where = (window: { start: string; end: string }) =>
- `project_id = '${input.projectId}'
- AND referrer_type = 'search'
- AND created_at >= '${window.start}'
- AND created_at < '${window.end}'`;
-
- const [engines, [currentResult], [prevResult]] = await Promise.all([
- chQuery<{ name: string; sessions: number }>(
- `SELECT
- referrer_name as name,
- count(*) as sessions
- FROM ${TABLE_NAMES.sessions}
- WHERE ${where(windows.current)}
- GROUP BY name
- ORDER BY sessions DESC
- LIMIT 10`
- ),
- chQuery<{ sessions: number }>(
- `SELECT count(*) as sessions
- FROM ${TABLE_NAMES.sessions}
- WHERE ${where(windows.current)}`
- ),
- chQuery<{ sessions: number }>(
- `SELECT count(*) as sessions
- FROM ${TABLE_NAMES.sessions}
- WHERE ${where(windows.previous)}`
- ),
- ]);
-
- return {
- engines,
- // Counted separately from `engines`, which is capped at the top 10, so
- // the total compares like-for-like with the previous period.
- total: currentResult?.sessions ?? 0,
- previousTotal: prevResult?.sessions ?? 0,
- };
+ await requireRead(ctx.session.userId, input.projectId);
+ const { startDate, endDate } = await resolveGscDateRange(
+ input.projectId,
+ input
+ );
+ return getGscSearchEngines(input.projectId, startDate, endDate);
}),
getAiEngines: protectedProcedure
.input(zGscDateInput)
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- const { startDate, endDate } = await resolveDates(input.projectId, input);
- const windows = getComparisonWindows(startDate, endDate);
-
- // Matched by name — will switch to referrer_type = 'ai' once available.
- const where = (window: { start: string; end: string }) =>
- `project_id = '${input.projectId}'
- AND ${AI_REFERRER_FILTER}
- AND created_at >= '${window.start}'
- AND created_at < '${window.end}'`;
-
- const [engines, [prevResult]] = await Promise.all([
- chQuery<{ name: string; sessions: number }>(
- `${AI_REFERRER_CTE}
- SELECT ${AI_REFERRER_CANONICAL_NAME} as name, count(*) as sessions
- FROM ${TABLE_NAMES.sessions}
- WHERE ${where(windows.current)}
- GROUP BY name
- ORDER BY sessions DESC`
- ),
- chQuery<{ sessions: number }>(
- `${AI_REFERRER_CTE}
- SELECT count(*) as sessions
- FROM ${TABLE_NAMES.sessions}
- WHERE ${where(windows.previous)}`
- ),
- ]);
-
- return {
- engines,
- // The filter is a fixed alias set, so the grouped rows are the whole
- // population — no LIMIT, and the total matches the previous period.
- total: engines.reduce((sum, engine) => sum + engine.sessions, 0),
- previousTotal: prevResult?.sessions ?? 0,
- };
+ await requireRead(ctx.session.userId, input.projectId);
+ const { startDate, endDate } = await resolveGscDateRange(
+ input.projectId,
+ input
+ );
+ return getGscAiEngines(input.projectId, startDate, endDate);
}),
getPreviousOverview: protectedProcedure
.input(zGscDateInput)
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- const { startDate, endDate } = await resolveDates(input.projectId, input);
-
- const startMs = new Date(startDate).getTime();
- const duration = new Date(endDate).getTime() - startMs;
- const prevEnd = new Date(startMs - 1);
- const prevStart = new Date(prevEnd.getTime() - duration);
- const fmt = (d: Date) => d.toISOString().slice(0, 10);
-
- const interval = (['day', 'week', 'month'] as const).includes(
- input.interval as 'day' | 'week' | 'month'
- )
- ? (input.interval as 'day' | 'week' | 'month')
- : 'day';
-
- return getGscOverview(
+ await requireRead(ctx.session.userId, input.projectId);
+ const { startDate, endDate } = await resolveGscDateRange(
input.projectId,
- fmt(prevStart),
- fmt(prevEnd),
- interval
+ input
+ );
+ return getGscPreviousOverview(
+ input.projectId,
+ startDate,
+ endDate,
+ toOverviewInterval(input.interval)
);
}),
getCannibalization: protectedProcedure
.input(zGscDateInput)
.query(async ({ input, ctx }) => {
- const access = await getProjectAccess({
- projectId: input.projectId,
- userId: ctx.session.userId,
- });
- if (!access) {
- throw new TRPCForbiddenError('You do not have access to this project');
- }
- const { startDate, endDate } = await resolveDates(input.projectId, input);
+ await requireRead(ctx.session.userId, input.projectId);
+ const { startDate, endDate } = await resolveGscDateRange(
+ input.projectId,
+ input
+ );
return getGscCannibalization(input.projectId, startDate, endDate);
}),
});