OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M5-002): gsc module: service + db client + trpc + /gsc callback + gsc queue + gscSync cron into core
ralphb49c55f2ec62
diff --git a/apps/api/src/controllers/gsc-oauth-callback.controller.ts b/apps/api/src/controllers/gsc-oauth-callback.controller.tsindex 24cbb502f..0b50ecd58 100644--- a/apps/api/src/controllers/gsc-oauth-callback.controller.ts+++ b/apps/api/src/controllers/gsc-oauth-callback.controller.ts@@ -1,5 +1,9 @@-import { googleGsc } from '@openpanel/core';-import { db, encrypt } from '@openpanel/db';+// Dissolved into @openpanel/core's gsc module (M5-002): state verification,+// the Google token exchange and the gscConnection upsert moved to+// packages/core/src/modules/gsc/gsc.service.ts#completeGscOAuthCallback.+// This controller stays (DELEGATE PATTERN) — it keeps Fastify's signed-cookie+// verification and the redirect shape, and delegates everything else.+import { completeGscOAuthCallback } from '@openpanel/core';import type { FastifyReply, FastifyRequest } from 'fastify';import { z } from 'zod';import { LogError } from '@/utils/errors';@@ -83,65 +87,30 @@ export async function gscGoogleCallback(const codeVerifierStr = codeVerifierResult?.value;const projectIdStr = projectIdResult?.value;- if (state !== stateStr) {- throw new LogError('GSC OAuth state mismatch', {- hasState: true,- hasStoredState: true,- stateMismatch: true,- });- }-- const tokens = await googleGsc.validateAuthorizationCode(- code,- codeVerifierStr- );-- const accessToken = tokens.accessToken();- const refreshToken = tokens.hasRefreshToken()- ? tokens.refreshToken()- : null;- const accessTokenExpiresAt = tokens.accessTokenExpiresAt();-- if (!refreshToken) {- throw new LogError('No refresh token returned from Google GSC OAuth');- }-- const project = await db.project.findUnique({- where: { id: projectIdStr },- select: { id: true, organizationId: true },- });-- if (!project) {- throw new LogError('Project not found for GSC connection', {+ let organizationId: string;+ try {+ ({ organizationId } = await completeGscOAuthCallback({+ code,+ state,+ storedState: stateStr,+ codeVerifier: codeVerifierStr,projectId: projectIdStr,- });+ }));+ } catch (error) {+ // Re-wrapped as LogError so redirectWithError shows the same+ // per-failure message it always has, sourced now from+ // gsc.service.ts#completeGscOAuthCallback instead of inline.+ const message = error instanceof Error ? error.message : String(error);+ throw new LogError(message, { projectId: projectIdStr });}- await db.gscConnection.upsert({- where: { projectId: projectIdStr },- create: {- projectId: projectIdStr,- accessToken: encrypt(accessToken),- refreshToken: encrypt(refreshToken),- accessTokenExpiresAt,- siteUrl: '',- },- update: {- accessToken: encrypt(accessToken),- refreshToken: encrypt(refreshToken),- accessTokenExpiresAt,- lastSyncStatus: null,- lastSyncError: null,- },- });-reply.clearCookie('gsc_oauth_state');reply.clearCookie('gsc_code_verifier');reply.clearCookie('gsc_project_id');const dashboardUrl =process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL!;- const redirectUrl = `${dashboardUrl}/${project.organizationId}/${projectIdStr}/settings/gsc`;+ const redirectUrl = `${dashboardUrl}/${organizationId}/${projectIdStr}/settings/gsc`;return reply.redirect(redirectUrl);} catch (error) {req.log.error({ err: error }, 'GSC OAuth callback error');diff --git a/apps/worker/src/jobs/gsc.ts b/apps/worker/src/jobs/gsc.tsindex 37c9f7070..9480be5db 100644--- a/apps/worker/src/jobs/gsc.ts+++ b/apps/worker/src/jobs/gsc.ts@@ -1,145 +1,36 @@-import { db, syncGscData } from '@openpanel/db';+// Dissolved into @openpanel/core's gsc module (M5-002): the sync/backfill job+// bodies and their status-update bookkeeping moved to+// packages/core/src/modules/gsc/gsc.service.ts. This file stays (DELEGATE+// PATTERN) — it is the BullMQ job body V1's worker registers+// (boot-workers.ts) and the `cron.ts` dispatcher's `gscSync` case, both thin+// wrappers around the core functions.+import {+ listGscConnectionsForSync,+ runGscProjectBackfill,+ runGscProjectSync,+} from '@openpanel/core';import type { GscQueuePayload } from '@openpanel/queue';import { gscQueue } from '@openpanel/queue';import type { Job } from 'bullmq';import { logger } from '../utils/logger';-const BACKFILL_MONTHS = 6;-const CHUNK_DAYS = 14;-export function gscJob(job: Job<GscQueuePayload>) {switch (job.data.type) {case 'gscProjectSync':- return gscProjectSyncJob(job.data.payload.projectId);+ return runGscProjectSync(job.data.payload.projectId);case 'gscProjectBackfill':- return gscProjectBackfillJob(job.data.payload.projectId);+ return runGscProjectBackfill(job.data.payload.projectId);default:throw new Error('Unknown GSC job type');}}-async function gscProjectSyncJob(projectId: string) {- const conn = await db.gscConnection.findUnique({ where: { projectId } });- if (!conn?.siteUrl) {- logger.warn({ projectId }, 'GSC sync skipped: no connection or siteUrl');- return;- }-- try {- // Sync rolling 3-day window (GSC data can arrive late)- const endDate = new Date();- endDate.setDate(endDate.getDate() - 1); // yesterday- const startDate = new Date(endDate);- startDate.setDate(startDate.getDate() - 2); // 3 days total-- await syncGscData(projectId, startDate, endDate);-- await db.gscConnection.update({- where: { projectId },- data: {- lastSyncedAt: new Date(),- lastSyncStatus: 'success',- lastSyncError: null,- },- });- logger.info({ projectId }, 'GSC sync completed');- } catch (error) {- const message = error instanceof Error ? error.message : String(error);- await db.gscConnection.update({- where: { projectId },- data: {- lastSyncedAt: new Date(),- lastSyncStatus: 'error',- lastSyncError: message,- },- });- logger.error({ err: error, projectId }, 'GSC sync failed');- throw error;- }-}--async function gscProjectBackfillJob(projectId: string) {- const conn = await db.gscConnection.findUnique({ where: { projectId } });- if (!conn?.siteUrl) {- logger.warn(- { projectId },- 'GSC backfill skipped: no connection or siteUrl',- );- return;- }-- await db.gscConnection.update({- where: { projectId },- data: { backfillStatus: 'running' },- });-- try {- const endDate = new Date();- endDate.setDate(endDate.getDate() - 1); // yesterday-- const startDate = new Date(endDate);- startDate.setMonth(startDate.getMonth() - BACKFILL_MONTHS);-- // Process in chunks to avoid timeouts and respect API limits- let chunkEnd = new Date(endDate);- while (chunkEnd > startDate) {- const chunkStart = new Date(chunkEnd);- chunkStart.setDate(chunkStart.getDate() - CHUNK_DAYS + 1);- if (chunkStart < startDate) {- chunkStart.setTime(startDate.getTime());- }-- logger.info(- {- projectId,- from: chunkStart.toISOString().slice(0, 10),- to: chunkEnd.toISOString().slice(0, 10),- },- 'GSC backfill chunk',- );-- await syncGscData(projectId, chunkStart, chunkEnd);-- chunkEnd = new Date(chunkStart);- chunkEnd.setDate(chunkEnd.getDate() - 1);- }-- await db.gscConnection.update({- where: { projectId },- data: {- backfillStatus: 'completed',- lastSyncedAt: new Date(),- lastSyncStatus: 'success',- lastSyncError: null,- },- });- logger.info({ projectId }, 'GSC backfill completed');- } catch (error) {- const message = error instanceof Error ? error.message : String(error);- await db.gscConnection.update({- where: { projectId },- data: {- backfillStatus: 'failed',- lastSyncStatus: 'error',- lastSyncError: message,- },- });- logger.error({ err: error, projectId }, 'GSC backfill failed');- throw error;- }-}-export async function gscSyncAllJob() {- const connections = await db.gscConnection.findMany({- where: {- siteUrl: { not: '' },- },- select: { projectId: true },- });+ const connections = await listGscConnectionsForSync();logger.info({ count: connections.length },- 'GSC nightly sync: enqueuing projects',+ 'GSC nightly sync: enqueuing projects');for (const conn of connections) {diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex dd4992e9b..e8819ced7 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -145,6 +145,49 @@ export {verifyPasswordHash,verifyTotpCode,} from './modules/auth/auth.service';+// Dissolved from @openpanel/db's src/gsc.ts + services/gsc.service.ts+// (M5-002) — apps/worker's gsc job file, apps/api's gsc OAuth callback+// controller and packages/trpc's gsc router call these directly, the same+// way V1 reaches every other dissolved service here. packages/db/src/gsc.ts+// and packages/db/src/services/gsc.service.ts re-export the subset MCP's gsc+// tools and the assistant's SEO tools still reach via `@openpanel/db`.+export type {+ GscCannibalizedQuery,+ GscConnectionSummary,+ GscDateRangeInput,+ GscOAuthCallbackInput,+ GscOAuthCallbackResult,+ GscQueryOpportunity,+ GscSite,+} from './modules/gsc/gsc.service';+export {+ completeGscOAuthCallback,+ disconnectGscConnection,+ getGscAiEngines,+ getGscCannibalization,+ getGscConnection,+ getGscOverview,+ getGscPageDetails,+ getGscPages,+ getGscPreviousOverview,+ getGscQueries,+ getGscQueryDetails,+ getGscSearchEngines,+ gscGetCannibalizationCore,+ gscGetOverviewCore,+ gscGetPageDetailsCore,+ gscGetQueryDetailsCore,+ gscGetQueryOpportunitiesCore,+ gscGetTopPagesCore,+ gscGetTopQueriesCore,+ listGscConnectionsForSync,+ listGscSites,+ resolveGscDateRange,+ runGscProjectBackfill,+ runGscProjectSync,+ selectGscSite,+ syncGscData,+} from './modules/gsc/gsc.service';// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service// (M5-001) — apps/worker's insight job files and packages/trpc's insight// router call these directly, the same way V1 reaches every other dissolveddiff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.tsindex 5c3f79545..f0c8be344 100644--- a/packages/core/src/jobs.registry.test.ts+++ b/packages/core/src/jobs.registry.test.ts@@ -82,7 +82,7 @@ test('the insights queue carries the insight module job', () => {// job's `type` straight onto its name, so these three ids double as job names.test('the cron queue carries the insight module cron fragment', () => {expect(Object.keys(queues.cron.jobs).sort()).toEqual(- ['insightCleanup', 'insightsDaily', 'weeklyDigest'].sort()+ ['gscSync', 'insightCleanup', 'insightsDaily', 'weeklyDigest'].sort());for (const name of ['insightsDaily', 'insightCleanup', 'weeklyDigest']) {expect(@@ -90,3 +90,24 @@ test('the cron queue carries the insight module cron fragment', () => {).toMatchObject({ queue: 'cron', name });}});++// M5-002: the gsc module's own queue job, plus its cron fan-out fragment.+// legacyCompat.gsc already discriminates on these exact names (jobs/compat.ts),+// so this pins the two in agreement.+test('the gsc queue carries the gsc module jobs', () => {+ expect(Object.keys(queues.gsc.jobs).sort()).toEqual(+ ['gscProjectBackfill', 'gscProjectSync'].sort()+ );+ for (const name of ['gscProjectSync', 'gscProjectBackfill']) {+ expect(queues.gsc.jobs[name as keyof typeof queues.gsc.jobs]).toMatchObject(+ { queue: 'gsc', name }+ );+ }+});++test('the cron queue carries the gsc module cron fragment', () => {+ expect(queues.cron.jobs.gscSync).toMatchObject({+ queue: 'cron',+ name: 'gscSync',+ });+});diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.tsindex 5e3c41877..79f898545 100644--- a/packages/core/src/jobs.registry.ts+++ b/packages/core/src/jobs.registry.ts@@ -14,6 +14,7 @@ import { legacyCompat } from './jobs/compat';import type { Producers } from './jobs/define';import { defineQueue } from './jobs/define';import type { ProducerHandle } from './jobs/producers';+import { gscCronJobs, gscQueueJobs } from './modules/gsc/gsc.jobs';import {insightCronJobs,insightQueueJobs,@@ -53,6 +54,7 @@ const cron = defineQueue('cron', {// Every module's cron fragment spreads in here (ADR-005: "no cron module").jobs: {...insightCronJobs,+ ...gscCronJobs,},});@@ -83,7 +85,9 @@ const gsc = defineQueue('gsc', {defaults: { removeOnComplete: 50, removeOnFail: 100 },worker: { concurrency: CONCURRENCY.gsc },compat: legacyCompat.gsc,- jobs: {},+ jobs: {+ ...gscQueueJobs,+ },});const cohortCompute = defineQueue('cohortCompute', {diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.tsindex 81c291543..b94cfc668 100644--- a/packages/core/src/jobs/schedulers.ts+++ b/packages/core/src/jobs/schedulers.ts@@ -5,6 +5,7 @@// locking of our own needed.import type { Logger } from '../logger';+import { gscCronSchedules } from '../modules/gsc/gsc.jobs';import { insightCronSchedules } from '../modules/insight/insight.jobs';import { wrap } from './envelope';@@ -63,7 +64,9 @@ export const CRON_SCHEDULES: readonly SchedulerDefinition[] = [{ id: 'flushReplay', schedule: { every: 10_000 } },{ id: 'flushGroups', schedule: { every: 10_000 } },{ id: 'onboarding', schedule: { pattern: '0 * * * *' } },- { id: 'gscSync', schedule: { pattern: '0 3 * * *' } },+ // gscSync — owned by the gsc module, declared next to its jobs+ // (modules/gsc/gsc.jobs.ts).+ ...gscCronSchedules,{ id: 'cohortRefresh', schedule: { pattern: '*/30 * * * *' } },{ id: 'sessionReaper', schedule: { every: 5 * MINUTE_MS } },// Daily 04:00 UTC — backstop for cleanup leaks.diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.tsnew file mode 100644index 000000000..36ceeaccb--- /dev/null+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts@@ -0,0 +1,151 @@+// Job/cron-fragment wiring — no ClickHouse/Postgres touched. The service+// methods themselves are exercised in gsc.service.test.ts.++import { expect, test } from 'bun:test';+import type { JobCtx } from '../../context';+import { createRecordingProducers } from '../../jobs/testing';+import { queues } from '../../jobs.registry';+import type { Logger } from '../../logger';+import type { Services } from '../../services';+import { gscCronJobs, gscCronSchedules, gscQueueJobs } from './gsc.jobs';+import type { GscService } from './gsc.service';++function stubLogger(): Logger {+ const noop = () => undefined;+ const logger: Logger = {+ fatal: noop,+ error: noop,+ warn: noop,+ info: noop,+ debug: noop,+ trace: noop,+ child: () => logger,+ };+ return logger;+}++function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {+ const services: Services = {+ auth: {} as Services['auth'],+ insight: {} as Services['insight'],+ gsc: gsc as GscService,+ };+ return {+ db: {},+ ch: {},+ redis: {},+ clients: {},+ buffers: {},+ logger: stubLogger(),+ queues: createRecordingProducers(queues).queues,+ services,+ requestId: 'req_1',+ job: { id: 'job_1', attempt: 0, queue: 'gsc', name: jobName },+ };+}++test('the gsc jobs registry declares gscProjectSync and gscProjectBackfill on the gsc queue', () => {+ expect(queues.gsc.jobs.gscProjectSync).toMatchObject({+ queue: 'gsc',+ name: 'gscProjectSync',+ });+ expect(queues.gsc.jobs.gscProjectBackfill).toMatchObject({+ queue: 'gsc',+ name: 'gscProjectBackfill',+ });+});++test('the gsc cron fragment is spread into the cron queue', () => {+ expect(queues.cron.jobs.gscSync).toBeDefined();+});++test('gscProjectSync validates its payload', () => {+ expect(() => gscQueueJobs.gscProjectSync.payload.parse({})).toThrow();+ expect(+ gscQueueJobs.gscProjectSync.payload.parse({ projectId: 'p1' })+ ).toEqual({ projectId: 'p1' });+});++test('gscProjectBackfill validates its payload', () => {+ expect(() => gscQueueJobs.gscProjectBackfill.payload.parse({})).toThrow();+ expect(+ gscQueueJobs.gscProjectBackfill.payload.parse({ projectId: 'p1' })+ ).toEqual({ projectId: 'p1' });+});++test('gscProjectSync delegates to runProjectSync', async () => {+ const calls: unknown[] = [];+ const ctx = stubJobCtx({+ runProjectSync: async (projectId) => {+ calls.push(projectId);+ },+ });++ await gscQueueJobs.gscProjectSync.handler({+ payload: { projectId: 'p1' },+ ctx,+ });++ expect(calls).toEqual(['p1']);+});++test('gscProjectBackfill delegates to runProjectBackfill', async () => {+ const calls: unknown[] = [];+ const ctx = stubJobCtx({+ runProjectBackfill: async (projectId) => {+ calls.push(projectId);+ },+ });++ await gscQueueJobs.gscProjectBackfill.handler({+ payload: { projectId: 'p1' },+ ctx,+ });++ expect(calls).toEqual(['p1']);+});++test('gscSync fans out one gscProjectSync enqueue per connected project', async () => {+ const producers = createRecordingProducers(queues);+ const ctx: JobCtx = {+ db: {},+ ch: {},+ redis: {},+ clients: {},+ buffers: {},+ logger: stubLogger(),+ queues: producers.queues,+ services: {+ auth: {} as Services['auth'],+ insight: {} as Services['insight'],+ gsc: {+ listConnectionsForSync: async () => [+ { projectId: 'p1' },+ { projectId: 'p2' },+ ],+ } as GscService,+ },+ requestId: 'req_1',+ job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'gscSync' },+ };++ await gscCronJobs.gscSync.handler({ payload: null, ctx });++ expect(producers.recorded).toHaveLength(2);+ expect(producers.recorded.map((r) => r.job)).toEqual([+ 'gscProjectSync',+ 'gscProjectSync',+ ]);+ expect(producers.recorded.map((r) => r.payload)).toEqual([+ { projectId: 'p1' },+ { projectId: 'p2' },+ ]);+});++// Byte-identity with the id/cadence schedulers.test.ts's golden snapshot pins+// (apps/worker/src/boot-cron.ts).+test('the gsc cron fragment carries V1 id and cadence unchanged', () => {+ expect(gscCronSchedules).toEqual([+ { id: 'gscSync', schedule: { pattern: '0 3 * * *' } },+ ]);+});diff --git a/packages/core/src/modules/gsc/gsc.jobs.ts b/packages/core/src/modules/gsc/gsc.jobs.tsnew file mode 100644index 000000000..bb7ee11cb--- /dev/null+++ b/packages/core/src/modules/gsc/gsc.jobs.ts@@ -0,0 +1,52 @@+// Ported from apps/worker/src/jobs/gsc.ts + boot-cron.ts's gscSync schedule+// (M5-002).+//+// `gscProjectSync` / `gscProjectBackfill` are this module's own queue (`gsc`+// in the registry — ADR-005's registry key). `gscSync` is a cron fragment:+// declared here, spread into the ONE `cron` queue by jobs.registry.ts and+// into `CRON_SCHEDULES` by jobs/schedulers.ts. Scheduler id and cadence are+// V1's, unchanged (apps/worker/src/boot-cron.ts).++import { z } from 'zod';+import { defineJob } from '../../jobs/define';+import type { SchedulerDefinition } from '../../jobs/schedulers';++const gscProjectPayload = z.object({ projectId: z.string() });++/** The `gsc` queue's own jobs. */+export const gscQueueJobs = {+ gscProjectSync: defineJob({+ payload: gscProjectPayload,+ handler: async ({ payload, ctx }) => {+ await ctx.services.gsc.runProjectSync(payload.projectId);+ },+ }),++ gscProjectBackfill: defineJob({+ payload: gscProjectPayload,+ handler: async ({ payload, ctx }) => {+ await ctx.services.gsc.runProjectBackfill(payload.projectId);+ },+ }),+};++/** This module's fragment of the `cron` queue's jobs. */+export const gscCronJobs = {+ // Fans out one `gscProjectSync` job per project with a connected GSC site.+ // Matches V1 (apps/worker/src/jobs/gsc.ts's gscSyncAllJob).+ gscSync: defineJob({+ payload: z.null(),+ handler: async ({ ctx }) => {+ const connections = await ctx.services.gsc.listConnectionsForSync();++ for (const { projectId } of connections) {+ await ctx.queues.gsc.gscProjectSync.add({ projectId });+ }+ },+ }),+};++/** This module's fragment of `CRON_SCHEDULES` — id and cadence unchanged. */+export const gscCronSchedules: readonly SchedulerDefinition[] = [+ { id: 'gscSync', schedule: { pattern: '0 3 * * *' } },+];diff --git a/packages/core/src/modules/gsc/gsc.routes.ts b/packages/core/src/modules/gsc/gsc.routes.tsnew file mode 100644index 000000000..567c9331a--- /dev/null+++ b/packages/core/src/modules/gsc/gsc.routes.ts@@ -0,0 +1,70 @@+// The GSC OAuth callback (M5-002). V1's Fastify controller+// (apps/api/src/controllers/gsc-oauth-callback.controller.ts) stays the LIVE+// route (DELEGATE PATTERN) and delegates its token-exchange/upsert logic to+// gsc.service.ts's `completeGscOAuthCallback` — the same function this route+// calls. This route is not yet reachable: main.ts does not mount+// `dashboardRoutes` until a real `AppDeps` exists (P3/P4/P8).+//+// NAMED GAP: per-route cookie signing is not wired yet (http/context.ts:147+// — "app.ts lists the signed names (P3)"), so `ctx.cookies.get()` here reads+// the raw, unsigned cookie value. V1's Fastify controller still verifies the+// signature; this route only re-checks the state match, and should not be+// treated as a signature-verified equivalent until that wiring lands.++import { redirect } from 'elysia';+import { z } from 'zod';+import { defineRoutes } from '../../http/define';+import { completeGscOAuthCallback } from './gsc.service';++const gscCallbackQuery = z.object({+ code: z.string(),+ state: z.string(),+});++function dashboardUrl(): string {+ return (+ process.env.DASHBOARD_URL || process.env.NEXT_PUBLIC_DASHBOARD_URL || ''+ );+}++function loginErrorRedirect(message: string) {+ const url = new URL('/login', dashboardUrl() || 'http://localhost');+ url.searchParams.set('error', message);+ return redirect(url.toString());+}++export const gscRoutes = defineRoutes((app) =>+ app.get(+ '/gsc/callback',+ async ({ query, ctx }) => {+ const storedState = ctx.cookies.get('gsc_oauth_state');+ const codeVerifier = ctx.cookies.get('gsc_code_verifier');+ const projectId = ctx.cookies.get('gsc_project_id');++ if (!(storedState && codeVerifier && projectId)) {+ return loginErrorRedirect('Missing GSC OAuth cookies');+ }++ try {+ const { organizationId } = await completeGscOAuthCallback({+ code: query.code,+ state: query.state,+ storedState,+ codeVerifier,+ projectId,+ });+ return redirect(+ `${dashboardUrl()}/${organizationId}/${projectId}/settings/gsc`+ );+ } catch (error) {+ ctx.logger.error({ err: error }, 'GSC OAuth callback error');+ const message =+ error instanceof Error+ ? error.message+ : 'Failed to connect Google Search Console';+ return loginErrorRedirect(message);+ }+ },+ { query: gscCallbackQuery, detail: { hide: true } }+ )+);diff --git a/packages/core/src/modules/gsc/gsc.rpc.test.ts b/packages/core/src/modules/gsc/gsc.rpc.test.tsnew file mode 100644index 000000000..9df8a62e5--- /dev/null+++ b/packages/core/src/modules/gsc/gsc.rpc.test.ts@@ -0,0 +1,73 @@+// Only the "is anyone logged in" boundary is exercised here — no database.+// The access-check + business logic ride on @openpanel/db (lazy-loaded, see+// gsc.service.ts's header) and ctx.services.gsc; wiring this router+// end-to-end against a real Postgres is P6's (protectedProcedure) job, not+// this one's — see gsc.rpc.ts's header.++import { expect, test } from 'bun:test';+import { stubHttpCtx } from '../../../test/rpc-fixtures';+import { makeTrpcContext } from '../../rpc/base';+import type { CookieOptions } from '../../shared/cookie';+import { gscRouter } from './gsc.rpc';++const COOKIE_OPTIONS: CookieOptions = {+ domain: '.openpanel.dev',+ secure: true,+ sameSite: 'lax',+ httpOnly: true,+ path: '/',+};++// EMPTY_SESSION's shape (packages/db/src/services/auth-session.service.ts) —+// `TrpcContext.session` is never literally `null`, only its `userId` is.+const EMPTY_SESSION = { session: null, user: null, userId: null };++async function anonCaller() {+ const { ctx } = stubHttpCtx({}, EMPTY_SESSION);+ const trpcCtx = await makeTrpcContext(ctx, new Headers(), {+ cookieOptions: COOKIE_OPTIONS,+ });+ return gscRouter.createCaller(trpcCtx);+}++test('getConnection rejects an unauthenticated caller before touching a project', async () => {+ const caller = await anonCaller();+ await expect(+ caller.getConnection({ projectId: 'proj_1' })+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });+});++test('initiateOAuth rejects an unauthenticated caller before minting state', async () => {+ const caller = await anonCaller();+ await expect(+ caller.initiateOAuth({ projectId: 'proj_1' })+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });+});++test('getSites rejects an unauthenticated caller before calling Google', async () => {+ const caller = await anonCaller();+ await expect(caller.getSites({ projectId: 'proj_1' })).rejects.toMatchObject({+ code: 'UNAUTHORIZED',+ });+});++test('selectSite rejects an unauthenticated caller before writing a connection', async () => {+ const caller = await anonCaller();+ await expect(+ caller.selectSite({ projectId: 'proj_1', siteUrl: 'https://example.com' })+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });+});++test('disconnect rejects an unauthenticated caller before deleting a connection', async () => {+ const caller = await anonCaller();+ await expect(+ caller.disconnect({ projectId: 'proj_1' })+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });+});++test('getOverview rejects an unauthenticated caller before resolving a date range', async () => {+ const caller = await anonCaller();+ await expect(+ caller.getOverview({ projectId: 'proj_1', range: '30d' })+ ).rejects.toMatchObject({ code: 'UNAUTHORIZED' });+});diff --git a/packages/core/src/modules/gsc/gsc.rpc.ts b/packages/core/src/modules/gsc/gsc.rpc.tsnew file mode 100644index 000000000..27011ee17--- /dev/null+++ b/packages/core/src/modules/gsc/gsc.rpc.ts@@ -0,0 +1,274 @@+// Ported from packages/trpc/src/routers/gsc.ts (M5-002).+//+// V1's `protectedProcedure` — the logger/session-scope/rate-limit middleware+// stack — lands in core with auth (rpc/base.ts: "those need the resolved+// Session shape and the access rules, so they land with auth (P6)"). Until+// then this router does its own minimal "is anyone logged in" check inline,+// exactly like V1's `enforceUserIsAuthed`. V1 keeps serving the live route+// through packages/trpc's own `protectedProcedure` (full stack included) and+// delegates its handler bodies to `ctx.services.gsc` (DELEGATE PATTERN),+// so nothing here is a live regression.+//+// The per-project access ladder itself IS shared: `./src/access.ts` binds+// core's shared/access.ts ladder to @openpanel/db's real lookups, the same+// way packages/trpc/src/access.ts does for V1.++import { zRange, zTimeInterval } from '@openpanel/validation';+import { z } from 'zod';+import { createTRPCRouter, procedure } from '../../rpc/base';+import { TRPCAccessError } from '../../rpc/errors';+import { Arctic, googleGsc } from '../auth/auth.service';++// Lazy, deliberately: ./src/access reaches @openpanel/db's real lookups,+// which construct a real pino logger — with a transport worker thread — at+// import time. See gsc.service.ts's header for the full reasoning.+function loadAccessChecks() {+ return import('./src/access');+}++const OAUTH_COOKIE_MAX_AGE_SECONDS = 60 * 10;++const zGscDateInput = z.object({+ projectId: z.string(),+ range: zRange,+ interval: zTimeInterval.optional().default('day'),+ startDate: z.string().nullish(),+ endDate: z.string().nullish(),+});++const GSC_OVERVIEW_INTERVALS = ['day', 'week', 'month'] as const;++function toOverviewInterval(+ interval: string+): (typeof GSC_OVERVIEW_INTERVALS)[number] {+ return (GSC_OVERVIEW_INTERVALS as readonly string[]).includes(interval)+ ? (interval as (typeof GSC_OVERVIEW_INTERVALS)[number])+ : 'day';+}++function requireLogin(userId: string | null | undefined): string {+ if (!userId) {+ throw new TRPCAccessError('Not authenticated');+ }+ return userId;+}++async function requireRead(userId: string, projectId: string) {+ const { requireProjectAccess } = await loadAccessChecks();+ await requireProjectAccess({ userId, projectId, level: 'read' });+}++async function requireWrite(userId: string, projectId: string) {+ const { requireProjectAccess } = await loadAccessChecks();+ await requireProjectAccess({ userId, projectId, level: 'write' });+}++export const gscRouter = createTRPCRouter({+ getConnection: procedure+ .input(z.object({ projectId: z.string() }))+ .query(async ({ input: { projectId }, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, projectId);+ return ctx.services.gsc.getConnection(projectId);+ }),++ initiateOAuth: procedure+ .input(z.object({ projectId: z.string() }))+ .mutation(async ({ input: { projectId }, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireWrite(userId, projectId);++ const state = Arctic.generateState();+ const codeVerifier = Arctic.generateCodeVerifier();+ const url = googleGsc.createAuthorizationURL(state, codeVerifier, [+ 'https://www.googleapis.com/auth/webmasters.readonly',+ ]);+ url.searchParams.set('access_type', 'offline');+ url.searchParams.set('prompt', 'consent');++ const cookieOpts = { maxAge: OAUTH_COOKIE_MAX_AGE_SECONDS, signed: true };+ ctx.setCookie('gsc_oauth_state', state, cookieOpts);+ ctx.setCookie('gsc_code_verifier', codeVerifier, cookieOpts);+ ctx.setCookie('gsc_project_id', projectId, cookieOpts);++ return { url: url.toString() };+ }),++ getSites: procedure+ .input(z.object({ projectId: z.string() }))+ .query(async ({ input: { projectId }, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, projectId);+ return ctx.services.gsc.listSites(projectId);+ }),++ selectSite: procedure+ .input(z.object({ projectId: z.string(), siteUrl: z.string() }))+ .mutation(async ({ input: { projectId, siteUrl }, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireWrite(userId, projectId);++ await ctx.services.gsc.selectSite(projectId, siteUrl);+ return { ok: true };+ }),++ disconnect: procedure+ .input(z.object({ projectId: z.string() }))+ .mutation(async ({ input: { projectId }, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireWrite(userId, projectId);++ await ctx.services.gsc.disconnect(projectId);+ return { ok: true };+ }),++ getOverview: procedure.input(zGscDateInput).query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, input.projectId);+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(+ input.projectId,+ input+ );+ return ctx.services.gsc.getOverview(+ input.projectId,+ startDate,+ endDate,+ toOverviewInterval(input.interval)+ );+ }),++ getPages: procedure+ .input(+ zGscDateInput.extend({+ limit: z.number().min(1).max(10_000).optional().default(100),+ })+ )+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, input.projectId);+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(+ input.projectId,+ input+ );+ return ctx.services.gsc.getPages(+ input.projectId,+ startDate,+ endDate,+ input.limit+ );+ }),++ getPageDetails: procedure+ .input(zGscDateInput.extend({ page: z.string() }))+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, input.projectId);+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(+ input.projectId,+ input+ );+ return ctx.services.gsc.getPageDetails(+ input.projectId,+ input.page,+ startDate,+ endDate+ );+ }),++ getQueryDetails: procedure+ .input(zGscDateInput.extend({ query: z.string() }))+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, input.projectId);+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(+ input.projectId,+ input+ );+ return ctx.services.gsc.getQueryDetails(+ input.projectId,+ input.query,+ startDate,+ endDate+ );+ }),++ getQueries: procedure+ .input(+ zGscDateInput.extend({+ limit: z.number().min(1).max(1000).optional().default(100),+ })+ )+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, input.projectId);+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(+ input.projectId,+ input+ );+ return ctx.services.gsc.getQueries(+ input.projectId,+ startDate,+ endDate,+ input.limit+ );+ }),++ getSearchEngines: procedure+ .input(zGscDateInput)+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, input.projectId);+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(+ input.projectId,+ input+ );+ return ctx.services.gsc.getSearchEngines(+ input.projectId,+ startDate,+ endDate+ );+ }),++ getAiEngines: procedure.input(zGscDateInput).query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, input.projectId);+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(+ input.projectId,+ input+ );+ return ctx.services.gsc.getAiEngines(input.projectId, startDate, endDate);+ }),++ getPreviousOverview: procedure+ .input(zGscDateInput)+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, input.projectId);+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(+ input.projectId,+ input+ );+ return ctx.services.gsc.getPreviousOverview(+ input.projectId,+ startDate,+ endDate,+ toOverviewInterval(input.interval)+ );+ }),++ getCannibalization: procedure+ .input(zGscDateInput)+ .query(async ({ input, ctx }) => {+ const userId = requireLogin(ctx.session.userId);+ await requireRead(userId, input.projectId);+ const { startDate, endDate } = await ctx.services.gsc.resolveDateRange(+ input.projectId,+ input+ );+ return ctx.services.gsc.getCannibalization(+ input.projectId,+ startDate,+ endDate+ );+ }),+});diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.tsnew file mode 100644index 000000000..3b775027d--- /dev/null+++ b/packages/core/src/modules/gsc/gsc.service.test.ts@@ -0,0 +1,406 @@+// gsc.service.ts's db/ch access is lazy (`await import(...)` inside each+// function — see the file's header), which is exactly what makes+// `mock.module` work here with no import-time side effects to race: every+// mock below is registered before the subject's first call, not before its+// (side-effect-free) import.++import { beforeAll, expect, mock, test } from 'bun:test';++process.env.ENCRYPTION_KEY = 'a'.repeat(64);++interface FakeGscConnection {+ projectId: string;+ siteUrl: string;+ accessToken: string;+ refreshToken: string;+ accessTokenExpiresAt: Date | null;+ lastSyncStatus: string | null;+ lastSyncError: string | null;+ backfillStatus: string | null;+}++const gscConnectionStore = new Map<string, FakeGscConnection>();++const gscConnection = {+ findUnique: mock(+ async ({ where: { projectId } }: { where: { projectId: string } }) =>+ gscConnectionStore.get(projectId) ?? null+ ),+ findUniqueOrThrow: mock(+ async ({ where: { projectId } }: { where: { projectId: string } }) => {+ const conn = gscConnectionStore.get(projectId);+ if (!conn) {+ throw new Error(`gscConnection ${projectId} not found`);+ }+ return conn;+ }+ ),+ findMany: mock(async () =>+ [...gscConnectionStore.values()]+ .filter((c) => c.siteUrl !== '')+ .map((c) => ({ projectId: c.projectId }))+ ),+ update: mock(+ async ({+ where: { projectId },+ data,+ }: {+ where: { projectId: string };+ data: Partial<FakeGscConnection>;+ }) => {+ const existing = gscConnectionStore.get(projectId);+ const updated = { ...existing, ...data } as FakeGscConnection;+ gscConnectionStore.set(projectId, updated);+ return updated;+ }+ ),+ upsert: mock(+ async ({+ where: { projectId },+ create,+ update,+ }: {+ where: { projectId: string };+ create: FakeGscConnection;+ update: Partial<FakeGscConnection>;+ }) => {+ const existing = gscConnectionStore.get(projectId);+ const value = existing+ ? ({ ...existing, ...update } as FakeGscConnection)+ : create;+ gscConnectionStore.set(projectId, value);+ return value;+ }+ ),+ deleteMany: mock(+ async ({ where: { projectId } }: { where: { projectId: string } }) => {+ const existed = gscConnectionStore.delete(projectId);+ return { count: existed ? 1 : 0 };+ }+ ),+};++const project = {+ findUnique: mock(async ({ where: { id } }: { where: { id: string } }) =>+ id === 'proj_missing' ? null : { id, organizationId: 'org_1' }+ ),+};++mock.module('@openpanel/db/src/prisma-client', () => ({+ db: { gscConnection, project },+}));++const chQuery = mock(async () => [] as unknown[]);+const originalCh = {+ query: mock(async () => ({ json: async () => [] as unknown[] })),+ insert: mock(async () => undefined),+};+// `ch` is unused here but included because `mock.module` replaces this+// specifier process-wide (bun runs every test file in one shared module+// registry without `--isolate` — see AGENTS.md) — insight.service.test.ts+// mocks the same path, so both factories must be a superset of every+// consumer's needs, whichever one ends up registered last.+mock.module('@openpanel/db/src/clickhouse/client', () => ({+ ch: {},+ originalCh,+ chQuery,+ TABLE_NAMES: { sessions: 'sessions' },+}));++// Bypasses the Redis cache-aside entirely — `getGscCannibalization`'s own+// logic is exercised directly, its caching is @openpanel/redis's concern.+// `getRedisCache` is unused here but included for the same cross-file+// mock.module reason as above.+mock.module('@openpanel/redis', () => ({+ cacheable: <T>(fn: T) => fn,+ getRedisCache: () => ({+ get: async () => null,+ setex: async () => undefined,+ }),+}));++const validateAuthorizationCode = mock(async () => ({+ accessToken: () => 'google-access-token',+ hasRefreshToken: () => true,+ refreshToken: () => 'google-refresh-token',+ accessTokenExpiresAt: () => new Date('2026-09-04T00:00:00.000Z'),+}));+mock.module('../auth/auth.service', () => ({+ googleGsc: { validateAuthorizationCode },+}));++const getSettingsForProject = mock(async () => ({ timezone: 'UTC' }));+mock.module('@openpanel/db/src/services/organization.service', () => ({+ getSettingsForProject,+}));++const getChartStartEndDate = mock(() => ({+ startDate: '2026-09-01T00:00:00.000Z',+ endDate: '2026-09-03T00:00:00.000Z',+}));+mock.module('@openpanel/db/src/services/date.service', () => ({+ getChartStartEndDate,+}));++function stubLogger(): import('../../logger').Logger {+ const noop = () => undefined;+ const logger: import('../../logger').Logger = {+ fatal: noop,+ error: noop,+ warn: noop,+ info: noop,+ debug: noop,+ trace: noop,+ child: () => logger,+ };+ return logger;+}++let subject: typeof import('./gsc.service');+beforeAll(async () => {+ subject = await import('./gsc.service');+});++test('listGscConnectionsForSync returns only projects with a siteUrl', async () => {+ gscConnectionStore.clear();+ gscConnectionStore.set('p1', {+ projectId: 'p1',+ siteUrl: 'https://one.example.com',+ accessToken: '',+ refreshToken: '',+ accessTokenExpiresAt: null,+ lastSyncStatus: null,+ lastSyncError: null,+ backfillStatus: null,+ });+ gscConnectionStore.set('p2', {+ projectId: 'p2',+ siteUrl: '',+ accessToken: '',+ refreshToken: '',+ accessTokenExpiresAt: null,+ lastSyncStatus: null,+ lastSyncError: null,+ backfillStatus: null,+ });++ const result = await subject.listGscConnectionsForSync();+ expect(result).toEqual([{ projectId: 'p1' }]);+});++test('getGscConnection returns null when no connection exists', async () => {+ gscConnectionStore.clear();+ const result = await subject.getGscConnection('missing');+ expect(result).toBeNull();+});++test('selectGscSite throws TRPCNotFoundError when the connection is missing', async () => {+ gscConnectionStore.clear();+ await expect(+ subject.selectGscSite('missing', 'https://example.com')+ ).rejects.toMatchObject({ message: 'GSC connection not found' });+});++test('selectGscSite updates siteUrl and resets backfillStatus to pending', async () => {+ gscConnectionStore.clear();+ gscConnectionStore.set('p1', {+ projectId: 'p1',+ siteUrl: '',+ accessToken: '',+ refreshToken: '',+ accessTokenExpiresAt: null,+ lastSyncStatus: null,+ lastSyncError: null,+ backfillStatus: null,+ });++ await subject.selectGscSite('p1', 'https://example.com');++ expect(gscConnectionStore.get('p1')).toMatchObject({+ siteUrl: 'https://example.com',+ backfillStatus: 'pending',+ });+});++test('disconnectGscConnection removes the stored connection', async () => {+ gscConnectionStore.clear();+ gscConnectionStore.set('p1', {+ projectId: 'p1',+ siteUrl: 'https://example.com',+ accessToken: '',+ refreshToken: '',+ accessTokenExpiresAt: null,+ lastSyncStatus: null,+ lastSyncError: null,+ backfillStatus: null,+ });++ await subject.disconnectGscConnection('p1');+ expect(gscConnectionStore.has('p1')).toBe(false);+});++test('runGscProjectSync skips silently when there is no connected site', async () => {+ gscConnectionStore.clear();+ await expect(+ subject.runGscProjectSync('missing', stubLogger())+ ).resolves.toBeUndefined();+});++test('runGscProjectSync records an error status when the token refresh has no credentials', async () => {+ gscConnectionStore.clear();+ gscConnectionStore.set('p1', {+ projectId: 'p1',+ siteUrl: 'https://example.com',+ accessToken: 'enc:whatever',+ refreshToken: 'enc:whatever',+ accessTokenExpiresAt: null, // forces the refresh branch+ lastSyncStatus: null,+ lastSyncError: null,+ backfillStatus: null,+ });+ const previousId = process.env.GOOGLE_CLIENT_ID;+ const previousSecret = process.env.GOOGLE_CLIENT_SECRET;+ delete process.env.GOOGLE_CLIENT_ID;+ delete process.env.GOOGLE_CLIENT_SECRET;++ await expect(subject.runGscProjectSync('p1', stubLogger())).rejects.toThrow();++ expect(gscConnectionStore.get('p1')?.lastSyncStatus).toBe('error');++ if (previousId !== undefined) {+ process.env.GOOGLE_CLIENT_ID = previousId;+ }+ if (previousSecret !== undefined) {+ process.env.GOOGLE_CLIENT_SECRET = previousSecret;+ }+});++test('completeGscOAuthCallback rejects a state mismatch before calling Google', async () => {+ validateAuthorizationCode.mockClear();+ await expect(+ subject.completeGscOAuthCallback({+ code: 'code',+ state: 'a',+ storedState: 'b',+ codeVerifier: 'verifier',+ projectId: 'p1',+ })+ ).rejects.toThrow('GSC OAuth state mismatch');+ expect(validateAuthorizationCode).not.toHaveBeenCalled();+});++test('completeGscOAuthCallback rejects an unknown project', async () => {+ await expect(+ subject.completeGscOAuthCallback({+ code: 'code',+ state: 'a',+ storedState: 'a',+ codeVerifier: 'verifier',+ projectId: 'proj_missing',+ })+ ).rejects.toThrow('Project not found for GSC connection');+});++test('completeGscOAuthCallback upserts the connection and returns the organizationId', async () => {+ gscConnectionStore.clear();+ const result = await subject.completeGscOAuthCallback({+ code: 'code',+ state: 'a',+ storedState: 'a',+ codeVerifier: 'verifier',+ projectId: 'p1',+ });++ expect(result).toEqual({ organizationId: 'org_1' });+ expect(gscConnectionStore.get('p1')).toMatchObject({ projectId: 'p1' });+});++test('resolveGscDateRange resolves through the org timezone and chart date helper', async () => {+ const result = await subject.resolveGscDateRange('p1', { range: '7d' });+ expect(result).toEqual({ startDate: '2026-09-01', endDate: '2026-09-03' });+ expect(getSettingsForProject).toHaveBeenCalledWith('p1');+});++test('getGscOverview returns the ClickHouse rows as-is', async () => {+ const rows = [+ { date: '2026-09-01', clicks: 1, impressions: 2, ctr: 0.5, position: 3 },+ ];+ originalCh.query.mockImplementationOnce(async () => ({+ json: async () => rows,+ }));++ const result = await subject.getGscOverview('p1', '2026-09-01', '2026-09-03');+ expect(result).toEqual(rows);+});++test('gscGetOverviewCore aggregates the daily rows into a summary', async () => {+ originalCh.query.mockImplementationOnce(async () => ({+ json: async () => [+ {+ date: '2026-09-01',+ clicks: 10,+ impressions: 100,+ ctr: 0.1,+ position: 5,+ },+ {+ date: '2026-09-02',+ clicks: 20,+ impressions: 100,+ ctr: 0.2,+ position: 3,+ },+ ],+ }));++ const result = await subject.gscGetOverviewCore({+ projectId: 'p1',+ startDate: '2026-09-01',+ endDate: '2026-09-02',+ });++ expect(result.summary).toEqual({+ total_clicks: 30,+ total_impressions: 200,+ avg_ctr: 15,+ avg_position: 4,+ });+});++test('gscGetQueryOpportunitiesCore filters and scores mid-ranked, high-impression queries', async () => {+ originalCh.query.mockImplementationOnce(async () => ({+ json: async () => [+ // survives the impressions prefilter, but position 1 is out of the+ // opportunity band [4, 20] — excluded by computeOpportunities+ {+ query: 'too-high',+ clicks: 5,+ impressions: 500,+ ctr: 0.05,+ position: 1,+ },+ // excluded by the impressions prefilter itself (< 50)+ { query: 'too-rare', clicks: 1, impressions: 10, ctr: 0.05, position: 8 },+ // kept: mid-ranked, enough impressions+ {+ query: 'opportunity',+ clicks: 5,+ impressions: 200,+ ctr: 0.02,+ position: 8,+ },+ ],+ }));++ const result = await subject.gscGetQueryOpportunitiesCore({+ projectId: 'p1',+ startDate: '2026-09-01',+ endDate: '2026-09-02',+ });++ // total_analyzed counts everything past the impressions prefilter, before+ // computeOpportunities' own position/impressions band is applied.+ expect(result.total_analyzed).toBe(2);+ expect(result.opportunities).toHaveLength(1);+ expect(result.opportunities[0]?.query).toBe('opportunity');+});diff --git a/packages/core/src/modules/gsc/gsc.service.ts b/packages/core/src/modules/gsc/gsc.service.tsnew file mode 100644index 000000000..96d02426c--- /dev/null+++ b/packages/core/src/modules/gsc/gsc.service.ts@@ -0,0 +1,1456 @@+// Moved from packages/db/src/gsc.ts + packages/db/src/services/gsc.service.ts+// (M5-002) — the GSC OAuth token lifecycle, the Search Console API client,+// ClickHouse read/write, the AI-tool wrapper functions and the+// gscProjectSync/gscProjectBackfill job bodies. packages/db/src/gsc.ts and+// packages/db/src/services/gsc.service.ts become re-export shims of this+// file (same shape as packages/db/src/encryption.ts since M4-006).+//+// db/ch access is LAZY (`load*` below), not a static top-level import — see+// insight.service.ts's header for the full reasoning (jobs.registry.ts and+// services.ts pull this module into the eager barrel chain nearly every core+// test file reaches, and constructing @openpanel/db's clients at import time+// would spawn a pino-pretty transport worker thread per test file).+//+// ClickHouse queries here still go through raw `originalCh`/`chQuery` calls,+// not the `sql` tag: ADR-013 converts the analytics read path one query per+// P7 task, and this module's queries haven't been converted yet.++import { cacheable } from '@openpanel/redis';+import { createLogger, type ILogger } from '../../clients/logger';+import type { Logger } from '../../logger';+import { TRPCNotFoundError } from '../../rpc/errors';+import type { ServiceDeps } from '../../services';+import { decrypt, encrypt } from '../../shared/encryption';+import { googleGsc } from '../auth/auth.service';++const BACKFILL_MONTHS = 6;+const CHUNK_DAYS = 14;+const CANNIBALIZATION_CACHE_TTL_SEC = 60 * 60 * 4;+const GSC_ROW_LIMIT = 25_000;++let _logger: ILogger | undefined;+function getLogger(): ILogger {+ _logger ??= createLogger({ name: 'core:gsc' });+ return _logger;+}++function loadDb() {+ return import('@openpanel/db/src/prisma-client').then((m) => m.db);+}++function loadChClient() {+ return import('@openpanel/db/src/clickhouse/client');+}++export interface GscSite {+ siteUrl: string;+ permissionLevel: string;+}++async function refreshGscToken(+ refreshToken: string+): Promise<{ accessToken: string; expiresAt: Date }> {+ if (!(process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET)) {+ throw new Error(+ 'GOOGLE_CLIENT_ID or GOOGLE_CLIENT_SECRET is not set in this environment'+ );+ }++ const params = new URLSearchParams({+ client_id: process.env.GOOGLE_CLIENT_ID,+ client_secret: process.env.GOOGLE_CLIENT_SECRET,+ refresh_token: refreshToken,+ grant_type: 'refresh_token',+ });++ const res = await fetch('https://oauth2.googleapis.com/token', {+ method: 'POST',+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },+ body: params.toString(),+ });++ if (!res.ok) {+ const text = await res.text();+ throw new Error(`Failed to refresh GSC token: ${text}`);+ }++ const data = (await res.json()) as {+ access_token: string;+ expires_in: number;+ };+ const expiresAt = new Date(Date.now() + data.expires_in * 1000);+ return { accessToken: data.access_token, expiresAt };+}++export async function getGscAccessToken(projectId: string): Promise<string> {+ const db = await loadDb();+ const conn = await db.gscConnection.findUniqueOrThrow({+ where: { projectId },+ });++ if (+ conn.accessTokenExpiresAt &&+ conn.accessTokenExpiresAt.getTime() > Date.now() + 60_000+ ) {+ getLogger().info(+ { projectId, expiresAt: conn.accessTokenExpiresAt },+ 'GSC using cached access token'+ );+ return decrypt(conn.accessToken);+ }++ getLogger().info(+ {+ projectId,+ expiresAt: conn.accessTokenExpiresAt,+ hasRefreshToken: !!conn.refreshToken,+ },+ 'GSC access token expired, attempting refresh'+ );++ try {+ const { accessToken, expiresAt } = await refreshGscToken(+ decrypt(conn.refreshToken)+ );+ await db.gscConnection.update({+ where: { projectId },+ data: {+ accessToken: encrypt(accessToken),+ accessTokenExpiresAt: expiresAt,+ },+ });+ getLogger().info(+ { projectId, expiresAt },+ 'GSC token refreshed successfully'+ );+ return accessToken;+ } catch (error) {+ const errorMessage =+ error instanceof Error ? error.message : 'Failed to refresh token';+ getLogger().error(+ { err: error, projectId, errorMessage },+ 'GSC token refresh failed'+ );+ await db.gscConnection.update({+ where: { projectId },+ data: {+ lastSyncStatus: 'token_expired',+ lastSyncError: errorMessage,+ },+ });+ throw new Error(+ `GSC token refresh failed for project ${projectId}: ${errorMessage}`+ );+ }+}++export async function listGscSites(projectId: string): Promise<GscSite[]> {+ const accessToken = await getGscAccessToken(projectId);+ const res = await fetch('https://www.googleapis.com/webmasters/v3/sites', {+ headers: { Authorization: `Bearer ${accessToken}` },+ });++ if (!res.ok) {+ const text = await res.text();+ throw new Error(`Failed to list GSC sites: ${text}`);+ }++ const data = (await res.json()) as {+ siteEntry?: Array<{ siteUrl: string; permissionLevel: string }>;+ };+ return data.siteEntry ?? [];+}++interface GscApiRow {+ keys: string[];+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+}++interface GscDimensionFilter {+ dimension: string;+ operator: string;+ expression: string;+}++interface GscFilterGroup {+ filters: GscDimensionFilter[];+}++async function queryGscSearchAnalytics(+ accessToken: string,+ siteUrl: string,+ startDate: string,+ endDate: string,+ dimensions: string[],+ dimensionFilterGroups?: GscFilterGroup[]+): Promise<GscApiRow[]> {+ const encodedSiteUrl = encodeURIComponent(siteUrl);+ const url = `https://www.googleapis.com/webmasters/v3/sites/${encodedSiteUrl}/searchAnalytics/query`;++ const allRows: GscApiRow[] = [];+ let startRow = 0;++ while (true) {+ const res = await fetch(url, {+ method: 'POST',+ headers: {+ Authorization: `Bearer ${accessToken}`,+ 'Content-Type': 'application/json',+ },+ body: JSON.stringify({+ startDate,+ endDate,+ dimensions,+ rowLimit: GSC_ROW_LIMIT,+ startRow,+ dataState: 'all',+ ...(dimensionFilterGroups && { dimensionFilterGroups }),+ }),+ });++ if (!res.ok) {+ const text = await res.text();+ throw new Error(+ `GSC query failed for dimensions [${dimensions.join(',')}]: ${text}`+ );+ }++ const data = (await res.json()) as { rows?: GscApiRow[] };+ const rows = data.rows ?? [];+ allRows.push(...rows);++ if (rows.length < GSC_ROW_LIMIT) {+ break;+ }+ startRow += GSC_ROW_LIMIT;+ }++ return allRows;+}++function formatDate(date: Date): string {+ return date.toISOString().slice(0, 10);+}++function nowString(): string {+ return new Date().toISOString().replace('T', ' ').replace('Z', '');+}++export async function syncGscData(+ projectId: string,+ startDate: Date,+ endDate: Date+): Promise<void> {+ const db = await loadDb();+ const conn = await db.gscConnection.findUniqueOrThrow({+ where: { projectId },+ });++ if (!conn.siteUrl) {+ throw new Error('No GSC site URL configured for this project');+ }++ const accessToken = await getGscAccessToken(projectId);+ const start = formatDate(startDate);+ const end = formatDate(endDate);+ const syncedAt = nowString();+ const { originalCh } = await loadChClient();++ // 1. Daily totals — authoritative numbers for overview chart+ const dailyRows = await queryGscSearchAnalytics(+ accessToken,+ conn.siteUrl,+ start,+ end,+ ['date']+ );++ if (dailyRows.length > 0) {+ await originalCh.insert({+ table: 'gsc_daily',+ values: dailyRows.map((row) => ({+ project_id: projectId,+ date: row.keys[0] ?? '',+ clicks: row.clicks,+ impressions: row.impressions,+ ctr: row.ctr,+ position: row.position,+ synced_at: syncedAt,+ })),+ format: 'JSONEachRow',+ });+ }++ // 2. Per-page breakdown+ const pageRows = await queryGscSearchAnalytics(+ accessToken,+ conn.siteUrl,+ start,+ end,+ ['date', 'page']+ );++ if (pageRows.length > 0) {+ await originalCh.insert({+ table: 'gsc_pages_daily',+ values: pageRows.map((row) => ({+ project_id: projectId,+ date: row.keys[0] ?? '',+ page: row.keys[1] ?? '',+ clicks: row.clicks,+ impressions: row.impressions,+ ctr: row.ctr,+ position: row.position,+ synced_at: syncedAt,+ })),+ format: 'JSONEachRow',+ });+ }++ // 3. Per-query breakdown+ const queryRows = await queryGscSearchAnalytics(+ accessToken,+ conn.siteUrl,+ start,+ end,+ ['date', 'query']+ );++ if (queryRows.length > 0) {+ await originalCh.insert({+ table: 'gsc_queries_daily',+ values: queryRows.map((row) => ({+ project_id: projectId,+ date: row.keys[0] ?? '',+ query: row.keys[1] ?? '',+ clicks: row.clicks,+ impressions: row.impressions,+ ctr: row.ctr,+ position: row.position,+ synced_at: syncedAt,+ })),+ format: 'JSONEachRow',+ });+ }+}++export async function getGscOverview(+ projectId: string,+ startDate: string,+ endDate: string,+ interval: 'day' | 'week' | 'month' = 'day'+): Promise<+ Array<{+ date: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ }>+> {+ const dateExpr =+ interval === 'month'+ ? 'toStartOfMonth(date)'+ : interval === 'week'+ ? 'toStartOfWeek(date)'+ : 'date';++ const { originalCh } = await loadChClient();+ const result = await originalCh.query({+ query: `+ SELECT+ ${dateExpr} as date,+ sum(clicks) as clicks,+ sum(impressions) as impressions,+ avg(ctr) as ctr,+ avg(position) as position+ FROM gsc_daily+ FINAL+ WHERE project_id = {projectId: String}+ AND date >= {startDate: String}+ AND date <= {endDate: String}+ GROUP BY date+ ORDER BY date ASC+ `,+ query_params: { projectId, startDate, endDate },+ format: 'JSONEachRow',+ });+ return result.json();+}++export async function getGscPages(+ projectId: string,+ startDate: string,+ endDate: string,+ limit = 100+): Promise<+ Array<{+ page: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ }>+> {+ const { originalCh } = await loadChClient();+ const result = await originalCh.query({+ query: `+ SELECT+ page,+ sum(clicks) as clicks,+ sum(impressions) as impressions,+ avg(ctr) as ctr,+ avg(position) as position+ FROM gsc_pages_daily+ FINAL+ WHERE project_id = {projectId: String}+ AND date >= {startDate: String}+ AND date <= {endDate: String}+ GROUP BY page+ ORDER BY clicks DESC+ LIMIT {limit: UInt32}+ `,+ query_params: { projectId, startDate, endDate, limit },+ format: 'JSONEachRow',+ });+ return result.json();+}++export interface GscCannibalizedQuery {+ query: string;+ totalImpressions: number;+ totalClicks: number;+ pages: Array<{+ page: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ }>;+}++export const getGscCannibalization = cacheable(+ async (+ projectId: string,+ startDate: string,+ endDate: string+ ): Promise<GscCannibalizedQuery[]> => {+ const db = await loadDb();+ const conn = await db.gscConnection.findUniqueOrThrow({+ where: { projectId },+ });+ const accessToken = await getGscAccessToken(projectId);++ const rows = await queryGscSearchAnalytics(+ accessToken,+ conn.siteUrl,+ startDate,+ endDate,+ ['query', 'page']+ );++ const map = new Map<+ string,+ {+ totalImpressions: number;+ totalClicks: number;+ pages: GscCannibalizedQuery['pages'];+ }+ >();++ for (const row of rows) {+ const query = row.keys[0] ?? '';+ // Strip hash fragments — GSC records heading anchors (e.g. /page#section)+ // as separate URLs but Google treats them as the same page+ let page = row.keys[1] ?? '';+ try {+ const u = new URL(page);+ u.hash = '';+ page = u.toString();+ } catch {+ page = page.split('#')[0] ?? page;+ }++ const entry = map.get(query) ?? {+ totalImpressions: 0,+ totalClicks: 0,+ pages: [],+ };+ entry.totalImpressions += row.impressions;+ entry.totalClicks += row.clicks;+ // Merge into existing page entry if already seen (from a different hash variant)+ const existing = entry.pages.find((p) => p.page === page);+ if (existing) {+ const totalImpressions = existing.impressions + row.impressions;+ if (totalImpressions > 0) {+ existing.position =+ (existing.position * existing.impressions ++ row.position * row.impressions) /+ totalImpressions;+ }+ existing.clicks += row.clicks;+ existing.impressions += row.impressions;+ existing.ctr =+ existing.impressions > 0 ? existing.clicks / existing.impressions : 0;+ } else {+ entry.pages.push({+ page,+ clicks: row.clicks,+ impressions: row.impressions,+ ctr: row.ctr,+ position: row.position,+ });+ }+ map.set(query, entry);+ }++ return [...map.entries()]+ .filter(([, v]) => v.pages.length >= 2 && v.totalImpressions >= 100)+ .sort(([, a], [, b]) => b.totalImpressions - a.totalImpressions)+ .slice(0, 50)+ .map(([query, v]) => ({+ query,+ totalImpressions: v.totalImpressions,+ totalClicks: v.totalClicks,+ pages: v.pages.sort((a, b) =>+ a.position !== b.position+ ? a.position - b.position+ : b.impressions - a.impressions+ ),+ }));+ },+ CANNIBALIZATION_CACHE_TTL_SEC+);++export async function getGscPageDetails(+ projectId: string,+ page: string,+ startDate: string,+ endDate: string+): Promise<{+ timeseries: Array<{+ date: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ }>;+ queries: Array<{+ query: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ }>;+}> {+ const db = await loadDb();+ const conn = await db.gscConnection.findUniqueOrThrow({+ where: { projectId },+ });+ const accessToken = await getGscAccessToken(projectId);+ const filterGroups: GscFilterGroup[] = [+ { filters: [{ dimension: 'page', operator: 'equals', expression: page }] },+ ];++ const [timeseriesRows, queryRows] = await Promise.all([+ queryGscSearchAnalytics(+ accessToken,+ conn.siteUrl,+ startDate,+ endDate,+ ['date'],+ filterGroups+ ),+ queryGscSearchAnalytics(+ accessToken,+ conn.siteUrl,+ startDate,+ endDate,+ ['query'],+ filterGroups+ ),+ ]);++ return {+ timeseries: timeseriesRows.map((row) => ({+ date: row.keys[0] ?? '',+ clicks: row.clicks,+ impressions: row.impressions,+ ctr: row.ctr,+ position: row.position,+ })),+ queries: queryRows.map((row) => ({+ query: row.keys[0] ?? '',+ clicks: row.clicks,+ impressions: row.impressions,+ ctr: row.ctr,+ position: row.position,+ })),+ };+}++export async function getGscQueryDetails(+ projectId: string,+ query: string,+ startDate: string,+ endDate: string+): Promise<{+ timeseries: Array<{+ date: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ }>;+ pages: Array<{+ page: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ }>;+}> {+ const db = await loadDb();+ const conn = await db.gscConnection.findUniqueOrThrow({+ where: { projectId },+ });+ const accessToken = await getGscAccessToken(projectId);+ const filterGroups: GscFilterGroup[] = [+ {+ filters: [{ dimension: 'query', operator: 'equals', expression: query }],+ },+ ];++ const [timeseriesRows, pageRows] = await Promise.all([+ queryGscSearchAnalytics(+ accessToken,+ conn.siteUrl,+ startDate,+ endDate,+ ['date'],+ filterGroups+ ),+ queryGscSearchAnalytics(+ accessToken,+ conn.siteUrl,+ startDate,+ endDate,+ ['page'],+ filterGroups+ ),+ ]);++ return {+ timeseries: timeseriesRows.map((row) => ({+ date: row.keys[0] ?? '',+ clicks: row.clicks,+ impressions: row.impressions,+ ctr: row.ctr,+ position: row.position,+ })),+ pages: pageRows.map((row) => ({+ page: row.keys[0] ?? '',+ clicks: row.clicks,+ impressions: row.impressions,+ ctr: row.ctr,+ position: row.position,+ })),+ };+}++export async function getGscQueries(+ projectId: string,+ startDate: string,+ endDate: string,+ limit = 100+): Promise<+ Array<{+ query: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ }>+> {+ const { originalCh } = await loadChClient();+ const result = await originalCh.query({+ query: `+ SELECT+ query,+ sum(clicks) as clicks,+ sum(impressions) as impressions,+ avg(ctr) as ctr,+ avg(position) as position+ FROM gsc_queries_daily+ FINAL+ WHERE project_id = {projectId: String}+ AND date >= {startDate: String}+ AND date <= {endDate: String}+ GROUP BY query+ ORDER BY clicks DESC+ LIMIT {limit: UInt32}+ `,+ query_params: { projectId, startDate, endDate, limit },+ format: 'JSONEachRow',+ });+ return result.json();+}++// ---------------------------------------------------------------------------+// Connection CRUD — moved from packages/trpc/src/routers/gsc.ts's+// getConnection/selectSite/disconnect handler bodies (M5-002).+// ---------------------------------------------------------------------------++export interface GscConnectionSummary {+ id: string;+ siteUrl: string;+ lastSyncedAt: Date | null;+ lastSyncStatus: string | null;+ lastSyncError: string | null;+ backfillStatus: string | null;+ createdAt: Date;+ updatedAt: Date;+}++export async function getGscConnection(+ projectId: string+): Promise<GscConnectionSummary | null> {+ const db = await loadDb();+ return db.gscConnection.findUnique({+ where: { projectId },+ select: {+ id: true,+ siteUrl: true,+ lastSyncedAt: true,+ lastSyncStatus: true,+ lastSyncError: true,+ backfillStatus: true,+ createdAt: true,+ updatedAt: true,+ },+ });+}++export async function selectGscSite(+ projectId: string,+ siteUrl: string+): Promise<void> {+ const db = await loadDb();+ const conn = await db.gscConnection.findUnique({ where: { projectId } });+ if (!conn) {+ throw new TRPCNotFoundError('GSC connection not found');+ }++ await db.gscConnection.update({+ where: { projectId },+ data: { siteUrl, backfillStatus: 'pending' },+ });+}++export async function disconnectGscConnection(+ projectId: string+): Promise<void> {+ const db = await loadDb();+ await db.gscConnection.deleteMany({ where: { projectId } });+}++// ---------------------------------------------------------------------------+// Dashboard aggregate queries — moved from packages/trpc/src/routers/gsc.ts+// (M5-002). The trpc router itself stays on V1 (DELEGATE PATTERN); this is+// its date-resolution and search/AI-engine breakdown logic.+// ---------------------------------------------------------------------------++export interface GscDateRangeInput {+ range: string;+ startDate?: string | null;+ endDate?: string | null;+}++export async function resolveGscDateRange(+ projectId: string,+ input: GscDateRangeInput+): Promise<{ startDate: string; endDate: string }> {+ const { getSettingsForProject } = await import(+ '@openpanel/db/src/services/organization.service'+ );+ const { getChartStartEndDate } = await import(+ '@openpanel/db/src/services/date.service'+ );+ const { timezone } = await getSettingsForProject(projectId);+ const { startDate, endDate } = getChartStartEndDate(+ {+ range: input.range as never,+ startDate: input.startDate,+ endDate: input.endDate,+ },+ timezone+ );+ return {+ startDate: startDate.slice(0, 10),+ endDate: endDate.slice(0, 10),+ };+}++/**+ * ClickHouse stores the same AI referrer under several spellings: the parsed+ * display name ('ChatGPT', 'Google Gemini'), the bare host ('chatgpt.com') and+ * the full origin ('https://kagi.com'). Normalize before matching so every+ * spelling lands on the same engine.+ */+const NORMALIZED_REFERRER_NAME =+ "lower(regexp_replace(referrer_name, '^https?://(www[.])?', ''))";++const AI_REFERRERS = [+ {+ canonical: 'chatgpt.com',+ aliases: [+ 'chatgpt',+ 'chatgpt.com',+ 'chat.openai.com',+ 'openai',+ 'openai.com',+ ],+ },+ {+ canonical: 'claude.ai',+ aliases: ['claude', 'claude.ai', 'anthropic', 'anthropic.com'],+ },+ {+ canonical: 'perplexity.ai',+ aliases: ['perplexity', 'perplexity.ai'],+ },+ {+ canonical: 'gemini.google.com',+ aliases: [+ 'gemini',+ 'google gemini',+ 'gemini.google.com',+ 'bard.google.com',+ ],+ },+ {+ canonical: 'copilot.com',+ aliases: [+ 'copilot',+ 'copilot.com',+ 'copilot.microsoft.com',+ 'microsoft copilot',+ ],+ },+ { canonical: 'grok.com', aliases: ['grok', 'grok.com'] },+ {+ canonical: 'mistral.ai',+ aliases: ['mistral', 'mistral.ai', 'chat.mistral.ai', 'le chat'],+ },+ {+ canonical: 'kagi.com',+ aliases: ['kagi', 'kagi.com', 'assistant.kagi.com'],+ },+] as const satisfies ReadonlyArray<{+ canonical: string;+ aliases: readonly string[];+}>;++const quoteList = (values: readonly string[]) =>+ values.map((value) => `'${value}'`).join(', ');++/** `norm` is the alias bound by AI_REFERRER_CTE below. */+const AI_REFERRER_CTE = `WITH ${NORMALIZED_REFERRER_NAME} AS norm`;++const AI_REFERRER_FILTER = `norm IN (${quoteList(+ AI_REFERRERS.flatMap((engine) => engine.aliases)+)})`;++/** Collapses every alias onto one row per engine. */+const AI_REFERRER_CANONICAL_NAME = `multiIf(${AI_REFERRERS.map(+ (engine) => `norm IN (${quoteList(engine.aliases)}), '${engine.canonical}'`+).join(', ')}, norm)`;++/**+ * Half-open windows so the last day of the range is included and the previous+ * window ends exactly where the current one starts (no gap, no overlap).+ */+function getComparisonWindows(startDate: string, endDate: string) {+ const start = new Date(`${startDate}T00:00:00Z`);+ const endExclusive = new Date(`${endDate}T00:00:00Z`);+ endExclusive.setUTCDate(endExclusive.getUTCDate() + 1);+ const previousStart = new Date(+ start.getTime() - (endExclusive.getTime() - start.getTime())+ );+ const fmt = (date: Date) => date.toISOString().slice(0, 19).replace('T', ' ');+ return {+ current: { start: fmt(start), end: fmt(endExclusive) },+ previous: { start: fmt(previousStart), end: fmt(start) },+ };+}++export async function getGscSearchEngines(+ projectId: string,+ startDate: string,+ endDate: string+) {+ const windows = getComparisonWindows(startDate, endDate);+ const { chQuery, TABLE_NAMES } = await loadChClient();++ const where = (window: { start: string; end: string }) =>+ `project_id = '${projectId}'+ AND referrer_type = 'search'+ AND created_at >= '${window.start}'+ AND created_at < '${window.end}'`;++ const [engines, [currentResult], [prevResult]] = await Promise.all([+ chQuery<{ name: string; sessions: number }>(+ `SELECT+ referrer_name as name,+ count(*) as sessions+ FROM ${TABLE_NAMES.sessions}+ WHERE ${where(windows.current)}+ GROUP BY name+ ORDER BY sessions DESC+ LIMIT 10`+ ),+ chQuery<{ sessions: number }>(+ `SELECT count(*) as sessions+ FROM ${TABLE_NAMES.sessions}+ WHERE ${where(windows.current)}`+ ),+ chQuery<{ sessions: number }>(+ `SELECT count(*) as sessions+ FROM ${TABLE_NAMES.sessions}+ WHERE ${where(windows.previous)}`+ ),+ ]);++ return {+ engines,+ // Counted separately from `engines`, which is capped at the top 10, so+ // the total compares like-for-like with the previous period.+ total: currentResult?.sessions ?? 0,+ previousTotal: prevResult?.sessions ?? 0,+ };+}++export async function getGscAiEngines(+ projectId: string,+ startDate: string,+ endDate: string+) {+ const windows = getComparisonWindows(startDate, endDate);+ const { chQuery, TABLE_NAMES } = await loadChClient();++ // Matched by name — will switch to referrer_type = 'ai' once available.+ const where = (window: { start: string; end: string }) =>+ `project_id = '${projectId}'+ AND ${AI_REFERRER_FILTER}+ AND created_at >= '${window.start}'+ AND created_at < '${window.end}'`;++ const [engines, [prevResult]] = await Promise.all([+ chQuery<{ name: string; sessions: number }>(+ `${AI_REFERRER_CTE}+ SELECT ${AI_REFERRER_CANONICAL_NAME} as name, count(*) as sessions+ FROM ${TABLE_NAMES.sessions}+ WHERE ${where(windows.current)}+ GROUP BY name+ ORDER BY sessions DESC`+ ),+ chQuery<{ sessions: number }>(+ `${AI_REFERRER_CTE}+ SELECT count(*) as sessions+ FROM ${TABLE_NAMES.sessions}+ WHERE ${where(windows.previous)}`+ ),+ ]);++ return {+ engines,+ // The filter is a fixed alias set, so the grouped rows are the whole+ // population — no LIMIT, and the total matches the previous period.+ total: engines.reduce((sum, engine) => sum + engine.sessions, 0),+ previousTotal: prevResult?.sessions ?? 0,+ };+}++export async function getGscPreviousOverview(+ projectId: string,+ startDate: string,+ endDate: string,+ interval: 'day' | 'week' | 'month' = 'day'+) {+ const startMs = new Date(startDate).getTime();+ const duration = new Date(endDate).getTime() - startMs;+ const prevEnd = new Date(startMs - 1);+ const prevStart = new Date(prevEnd.getTime() - duration);+ const fmt = (d: Date) => d.toISOString().slice(0, 10);++ return getGscOverview(projectId, fmt(prevStart), fmt(prevEnd), interval);+}++// ---------------------------------------------------------------------------+// AI-tool wrappers — moved from packages/db/src/services/gsc.service.ts.+// Consumed by apps/api's assistant SEO tools and the /insights REST surface+// (both still reach @openpanel/db, which re-exports these unchanged).+// ---------------------------------------------------------------------------++export interface GscQueryOpportunity {+ query: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ opportunity_score: number;+ reason: string;+}++const CTR_BENCHMARKS: Record<string, number> = {+ '1': 0.28,+ '2': 0.15,+ '3': 0.11,+ '4-6': 0.065,+ '7-10': 0.035,+ '11-20': 0.012,+};++function getCtrBenchmark(position: number): number {+ if (position <= 1) {+ return CTR_BENCHMARKS['1'] ?? 0.28;+ }+ if (position <= 2) {+ return CTR_BENCHMARKS['2'] ?? 0.15;+ }+ if (position <= 3) {+ return CTR_BENCHMARKS['3'] ?? 0.11;+ }+ if (position <= 6) {+ return CTR_BENCHMARKS['4-6'] ?? 0.065;+ }+ if (position <= 10) {+ return CTR_BENCHMARKS['7-10'] ?? 0.035;+ }+ return CTR_BENCHMARKS['11-20'] ?? 0.012;+}++function computeOpportunities(+ queries: Array<{+ query: string;+ clicks: number;+ impressions: number;+ ctr: number;+ position: number;+ }>+): GscQueryOpportunity[] {+ return queries+ .filter((q) => q.position >= 4 && q.position <= 20 && q.impressions >= 50)+ .map((q) => {+ const benchmark = getCtrBenchmark(q.position);+ const ctrGap = Math.max(0, benchmark - q.ctr);+ const opportunity_score =+ Math.round(q.impressions * (1 / q.position) * (1 + ctrGap) * 100) / 100;++ let reason: string;+ if (q.position <= 6) {+ reason = `Position ${q.position.toFixed(1)} — one rank improvement could significantly boost clicks`;+ } else if (q.ctr < benchmark * 0.5) {+ reason = `CTR (${(q.ctr * 100).toFixed(1)}%) is well below expected ${(benchmark * 100).toFixed(1)}% — title/meta optimization may help`;+ } else {+ reason = `Position ${q.position.toFixed(1)} with ${q.impressions} impressions — push to page 1 for major gains`;+ }++ return {+ query: q.query,+ clicks: q.clicks,+ impressions: q.impressions,+ ctr: Math.round(q.ctr * 10_000) / 100,+ position: Math.round(q.position * 10) / 10,+ opportunity_score,+ reason,+ };+ })+ .sort((a, b) => b.opportunity_score - a.opportunity_score)+ .slice(0, 50);+}++export async function gscGetOverviewCore(input: {+ projectId: string;+ startDate: string;+ endDate: string;+ interval?: 'day' | 'week' | 'month';+}) {+ const data = await getGscOverview(+ input.projectId,+ input.startDate,+ input.endDate,+ input.interval ?? 'day'+ );+ return {+ data,+ summary: {+ total_clicks: data.reduce((s, r) => s + r.clicks, 0),+ total_impressions: data.reduce((s, r) => s + r.impressions, 0),+ avg_ctr:+ data.length > 0+ ? Math.round(+ (data.reduce((s, r) => s + r.ctr, 0) / data.length) * 10_000+ ) / 100+ : 0,+ avg_position:+ data.length > 0+ ? Math.round(+ (data.reduce((s, r) => s + r.position, 0) / data.length) * 10+ ) / 10+ : 0,+ },+ };+}++export async function gscGetTopPagesCore(input: {+ projectId: string;+ startDate: string;+ endDate: string;+ limit?: number;+}) {+ return getGscPages(+ input.projectId,+ input.startDate,+ input.endDate,+ input.limit ?? 100+ );+}++export async function gscGetPageDetailsCore(input: {+ projectId: string;+ startDate: string;+ endDate: string;+ page: string;+}) {+ return getGscPageDetails(+ input.projectId,+ input.page,+ input.startDate,+ input.endDate+ );+}++export async function gscGetTopQueriesCore(input: {+ projectId: string;+ startDate: string;+ endDate: string;+ limit?: number;+}) {+ return getGscQueries(+ input.projectId,+ input.startDate,+ input.endDate,+ input.limit ?? 100+ );+}++export async function gscGetQueryOpportunitiesCore(input: {+ projectId: string;+ startDate: string;+ endDate: string;+ minImpressions?: number;+}) {+ const queries = await getGscQueries(+ input.projectId,+ input.startDate,+ input.endDate,+ 5000+ );+ const filtered = queries.filter(+ (q) => q.impressions >= (input.minImpressions ?? 50)+ );+ const opportunities = computeOpportunities(filtered);+ return {+ opportunities,+ total_analyzed: filtered.length,+ min_impressions: input.minImpressions ?? 50,+ };+}++export async function gscGetQueryDetailsCore(input: {+ projectId: string;+ startDate: string;+ endDate: string;+ query: string;+}) {+ return getGscQueryDetails(+ input.projectId,+ input.query,+ input.startDate,+ input.endDate+ );+}++export async function gscGetCannibalizationCore(input: {+ projectId: string;+ startDate: string;+ endDate: string;+}) {+ return getGscCannibalization(input.projectId, input.startDate, input.endDate);+}++// ---------------------------------------------------------------------------+// Job bodies — moved from apps/worker/src/jobs/gsc.ts (M5-002). Each fully+// owns its status-update bookkeeping so the worker file is a bare dispatch.+// ---------------------------------------------------------------------------++/** The `gscSync` cron fan-out: every project with a connected GSC site. */+export async function listGscConnectionsForSync(): Promise<+ { projectId: string }[]+> {+ const db = await loadDb();+ return db.gscConnection.findMany({+ where: { siteUrl: { not: '' } },+ select: { projectId: true },+ });+}++/** The `gscProjectSync` job body: rolling 3-day window (GSC data arrives late). */+export async function runGscProjectSync(+ projectId: string,+ logger: Logger = getLogger()+): Promise<void> {+ const db = await loadDb();+ const conn = await db.gscConnection.findUnique({ where: { projectId } });+ if (!conn?.siteUrl) {+ logger.warn({ projectId }, 'GSC sync skipped: no connection or siteUrl');+ return;+ }++ try {+ const endDate = new Date();+ endDate.setDate(endDate.getDate() - 1); // yesterday+ const startDate = new Date(endDate);+ startDate.setDate(startDate.getDate() - 2); // 3 days total++ await syncGscData(projectId, startDate, endDate);++ await db.gscConnection.update({+ where: { projectId },+ data: {+ lastSyncedAt: new Date(),+ lastSyncStatus: 'success',+ lastSyncError: null,+ },+ });+ logger.info({ projectId }, 'GSC sync completed');+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ await db.gscConnection.update({+ where: { projectId },+ data: {+ lastSyncedAt: new Date(),+ lastSyncStatus: 'error',+ lastSyncError: message,+ },+ });+ logger.error({ err: error, projectId }, 'GSC sync failed');+ throw error;+ }+}++/** The `gscProjectBackfill` job body: chunked to avoid timeouts and API limits. */+export async function runGscProjectBackfill(+ projectId: string,+ logger: Logger = getLogger()+): Promise<void> {+ const db = await loadDb();+ const conn = await db.gscConnection.findUnique({ where: { projectId } });+ if (!conn?.siteUrl) {+ logger.warn(+ { projectId },+ 'GSC backfill skipped: no connection or siteUrl'+ );+ return;+ }++ await db.gscConnection.update({+ where: { projectId },+ data: { backfillStatus: 'running' },+ });++ try {+ const endDate = new Date();+ endDate.setDate(endDate.getDate() - 1); // yesterday++ const startDate = new Date(endDate);+ startDate.setMonth(startDate.getMonth() - BACKFILL_MONTHS);++ // Process in chunks to avoid timeouts and respect API limits+ let chunkEnd = new Date(endDate);+ while (chunkEnd > startDate) {+ const chunkStart = new Date(chunkEnd);+ chunkStart.setDate(chunkStart.getDate() - CHUNK_DAYS + 1);+ if (chunkStart < startDate) {+ chunkStart.setTime(startDate.getTime());+ }++ logger.info(+ {+ projectId,+ from: chunkStart.toISOString().slice(0, 10),+ to: chunkEnd.toISOString().slice(0, 10),+ },+ 'GSC backfill chunk'+ );++ await syncGscData(projectId, chunkStart, chunkEnd);++ chunkEnd = new Date(chunkStart);+ chunkEnd.setDate(chunkEnd.getDate() - 1);+ }++ await db.gscConnection.update({+ where: { projectId },+ data: {+ backfillStatus: 'completed',+ lastSyncedAt: new Date(),+ lastSyncStatus: 'success',+ lastSyncError: null,+ },+ });+ logger.info({ projectId }, 'GSC backfill completed');+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ await db.gscConnection.update({+ where: { projectId },+ data: {+ backfillStatus: 'failed',+ lastSyncStatus: 'error',+ lastSyncError: message,+ },+ });+ logger.error({ err: error, projectId }, 'GSC backfill failed');+ throw error;+ }+}++// ---------------------------------------------------------------------------+// OAuth callback — moved from apps/api/src/controllers/gsc-oauth-callback.+// controller.ts's business logic (M5-002). Cookie reading/signing and the+// redirect stay with each transport (Fastify today, Elysia's gsc.routes.ts+// once it is live) — this is the part that is transport-agnostic.+// ---------------------------------------------------------------------------++export interface GscOAuthCallbackInput {+ code: string;+ state: string;+ storedState: string;+ codeVerifier: string;+ projectId: string;+}++export interface GscOAuthCallbackResult {+ organizationId: string;+}++export async function completeGscOAuthCallback(+ input: GscOAuthCallbackInput+): Promise<GscOAuthCallbackResult> {+ if (input.state !== input.storedState) {+ throw new Error('GSC OAuth state mismatch');+ }++ const tokens = await googleGsc.validateAuthorizationCode(+ input.code,+ input.codeVerifier+ );++ const accessToken = tokens.accessToken();+ const refreshToken = tokens.hasRefreshToken() ? tokens.refreshToken() : null;+ const accessTokenExpiresAt = tokens.accessTokenExpiresAt();++ if (!refreshToken) {+ throw new Error('No refresh token returned from Google GSC OAuth');+ }++ const db = await loadDb();+ const project = await db.project.findUnique({+ where: { id: input.projectId },+ select: { id: true, organizationId: true },+ });++ if (!project) {+ throw new Error('Project not found for GSC connection');+ }++ await db.gscConnection.upsert({+ where: { projectId: input.projectId },+ create: {+ projectId: input.projectId,+ accessToken: encrypt(accessToken),+ refreshToken: encrypt(refreshToken),+ accessTokenExpiresAt,+ siteUrl: '',+ },+ update: {+ accessToken: encrypt(accessToken),+ refreshToken: encrypt(refreshToken),+ accessTokenExpiresAt,+ lastSyncStatus: null,+ lastSyncError: null,+ },+ });++ return { organizationId: project.organizationId };+}++// ---------------------------------------------------------------------------+// `ctx.services.gsc` binding.+// ---------------------------------------------------------------------------++export interface GscService {+ getConnection(projectId: string): Promise<GscConnectionSummary | null>;+ listSites(projectId: string): Promise<GscSite[]>;+ /** Also enqueues the `gscProjectBackfill` job — same as V1's router. */+ selectSite(projectId: string, siteUrl: string): Promise<void>;+ disconnect(projectId: string): Promise<void>;+ resolveDateRange(+ projectId: string,+ input: GscDateRangeInput+ ): ReturnType<typeof resolveGscDateRange>;+ getOverview: typeof getGscOverview;+ getPreviousOverview: typeof getGscPreviousOverview;+ getPages: typeof getGscPages;+ getPageDetails: typeof getGscPageDetails;+ getQueryDetails: typeof getGscQueryDetails;+ getQueries: typeof getGscQueries;+ getSearchEngines: typeof getGscSearchEngines;+ getAiEngines: typeof getGscAiEngines;+ getCannibalization: typeof getGscCannibalization;+ listConnectionsForSync(): Promise<{ projectId: string }[]>;+ runProjectSync(projectId: string): Promise<void>;+ runProjectBackfill(projectId: string): Promise<void>;+}++/** `ctx.services.gsc` — a thin binding of the functions above to a Ctx's logger/queues. */+export function createGscService(deps: ServiceDeps): GscService {+ const logger = deps.logger.child({ module: 'gsc' });++ return {+ getConnection: getGscConnection,+ listSites: listGscSites,+ selectSite: async (projectId, siteUrl) => {+ await selectGscSite(projectId, siteUrl);+ await deps.queues.gsc.gscProjectBackfill.add({ projectId });+ },+ disconnect: disconnectGscConnection,+ resolveDateRange: resolveGscDateRange,+ getOverview: getGscOverview,+ getPreviousOverview: getGscPreviousOverview,+ getPages: getGscPages,+ getPageDetails: getGscPageDetails,+ getQueryDetails: getGscQueryDetails,+ getQueries: getGscQueries,+ getSearchEngines: getGscSearchEngines,+ getAiEngines: getGscAiEngines,+ getCannibalization: getGscCannibalization,+ listConnectionsForSync: listGscConnectionsForSync,+ runProjectSync: (projectId) => runGscProjectSync(projectId, logger),+ runProjectBackfill: (projectId) => runGscProjectBackfill(projectId, logger),+ };+}diff --git a/packages/core/src/modules/gsc/src/access.ts b/packages/core/src/modules/gsc/src/access.tsnew file mode 100644index 000000000..de4a4168e--- /dev/null+++ b/packages/core/src/modules/gsc/src/access.ts@@ -0,0 +1,21 @@+// Binds core's shared/access.ts ladder to @openpanel/db's real lookups, for+// this module's own router — the same binding packages/trpc/src/access.ts+// does for V1, self-contained here because the general core -> db binding+// itself moves with auth (P6), not this wave.++import {+ canWriteProject,+ getOrganizationAccess,+ getProjectAccess,+} from '@openpanel/db/src/services/access.service';+import { getProjectById } from '@openpanel/db/src/services/project.service';+import { createAccessChecks } from '../../../shared/access';++const checks = createAccessChecks({+ getProjectAccess,+ canWriteProject,+ getOrganizationAccess,+ getProjectById,+});++export const requireProjectAccess = checks.requireProjectAccess;diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.tsindex a8c9034a1..e02e69597 100644--- a/packages/core/src/modules/insight/insight.jobs.test.ts+++ b/packages/core/src/modules/insight/insight.jobs.test.ts@@ -39,6 +39,7 @@ function stubJobCtx(const services: Services = {auth: {} as Services['auth'],insight: insight as InsightService,+ gsc: {} as Services['gsc'],};return {db: {},@@ -62,8 +63,11 @@ test('the insight jobs registry declares insightsProject on the insights queue',});test('the insight cron fragment is spread into the cron queue', () => {- expect(Object.keys(queues.cron.jobs).sort()).toEqual(- ['insightCleanup', 'insightsDaily', 'weeklyDigest'].sort()+ // Subset, not exact equality: other modules (e.g. gsc, M5-002) spread+ // their own fragments into the same cron queue. jobs.registry.test.ts owns+ // the exhaustive membership check.+ expect(Object.keys(queues.cron.jobs)).toEqual(+ expect.arrayContaining(['insightCleanup', 'insightsDaily', 'weeklyDigest']));});@@ -113,6 +117,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId de{ projectId: 'p2', date },],} as InsightService,+ gsc: {} as Services['gsc'],},requestId: 'req_1',job: { id: 'job_1', attempt: 0, queue: 'cron', name: 'insightsDaily' },diff --git a/packages/core/src/modules/insight/insight.service.test.ts b/packages/core/src/modules/insight/insight.service.test.tsindex 663aa7f37..2fb12340a 100644--- a/packages/core/src/modules/insight/insight.service.test.ts+++ b/packages/core/src/modules/insight/insight.service.test.ts@@ -22,7 +22,17 @@ mock.module('@openpanel/db/src/prisma-client', () => ({db: { $executeRaw },}));-mock.module('@openpanel/db/src/clickhouse/client', () => ({ ch: {} }));+// `originalCh`/`chQuery`/`TABLE_NAMES` are unused here but included because+// `mock.module` replaces this specifier process-wide (bun runs every test+// file in one shared module registry without `--isolate` — see AGENTS.md) —+// gsc.service.test.ts mocks the same path, so both factories must be a+// superset of every consumer's needs, whichever one ends up registered last.+mock.module('@openpanel/db/src/clickhouse/client', () => ({+ ch: {},+ originalCh: { query: mock(async () => ({ json: async () => [] })), insert: mock(async () => undefined) },+ chQuery: mock(async () => []),+ TABLE_NAMES: { sessions: 'sessions' },+}));const spikesQuery = mock(async () => [{ anchorDate: '2026-09-01', spikes: [] },@@ -58,7 +68,12 @@ const getRedisCache = mock(() => ({return Promise.resolve();}),}));-mock.module('@openpanel/redis', () => ({ getRedisCache }));+// `cacheable` is unused here but included for the same cross-file+// mock.module reason as the clickhouse/client mock above.+mock.module('@openpanel/redis', () => ({+ getRedisCache,+ cacheable: <T>(fn: T) => fn,+}));let subject: typeof import('./insight.service');beforeAll(async () => {diff --git a/packages/core/src/rest.routes.test.ts b/packages/core/src/rest.routes.test.tsindex 851596492..a86620605 100644--- a/packages/core/src/rest.routes.test.ts+++ b/packages/core/src/rest.routes.test.ts@@ -21,7 +21,7 @@ test('opsRoutes answers a liveness probe through requestContext', async () => {expect(await response.json()).toEqual({ live: true });});-test('publicApiRoutes and dashboardRoutes compose with no modules yet', async () => {+test('publicApiRoutes and dashboardRoutes compose, dashboardRoutes carrying gsc', async () => {const { deps } = stubAppDeps();const app = new Elysia()diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.tsindex 89df30d08..6a1cd109d 100644--- a/packages/core/src/rest.routes.ts+++ b/packages/core/src/rest.routes.ts@@ -8,6 +8,7 @@ import { openapi } from '@elysiajs/openapi';import { Elysia } from 'elysia';import type { AppDeps } from './context';import { registry } from './metrics';+import { gscRoutes } from './modules/gsc/gsc.routes';import { healthRoutes } from './modules/health/health.routes';// ADR-002/ADR-003 pin: @elysiajs/openapi at the 1.4.15 fallback (2.0 is@@ -19,16 +20,19 @@ const OPENAPI_SPEC_PATH = '/openapi.json';// endpoint (ADR-003: "the transform that hides /metrics from the spec").const OPENAPI_EXCLUDED_PATHS = ['/metrics'];-// No module lands on either surface yet — the 35 modules' HTTP halves land-// with their waves (P5-P8). Both still take `deps` now so a module addition+// Both surfaces still take `deps` even where unused so a module addition// never changes the factory's signature, only its body. Return types are// left inferred, as `defineRoutes`'s `RouteApp` is: `.use()`'d plugins widen// Elysia's type parameters in a way the bare `Elysia` type cannot express.export const publicApiRoutes = (_deps: AppDeps) =>new Elysia({ name: 'core/public-api-routes' });-export const dashboardRoutes = (_deps: AppDeps) =>- new Elysia({ name: 'core/dashboard-routes' });+// gsc's callback is the first module to land here (M5-002). It is not yet+// reachable in production: main.ts only mounts `dashboardRoutes` once a real+// `AppDeps` exists (P3/P4/P8) — see gsc.routes.ts's header for the named gap+// that keeps it inert until then.+export const dashboardRoutes = (deps: AppDeps) =>+ new Elysia({ name: 'core/dashboard-routes' }).use(gscRoutes(deps));// healthz/metrics/misc share V1's ops surface (http/context.ts's// UNLOGGED_PATH_PREFIXES) — unauthenticated, uncorsed, unlogged. /metrics anddiff --git a/packages/core/src/rpc.router.ts b/packages/core/src/rpc.router.tsindex 317665686..4c2a3ddb5 100644--- a/packages/core/src/rpc.router.ts+++ b/packages/core/src/rpc.router.ts@@ -4,6 +4,7 @@// (ADR-007 decision 19). Only `health` exists so far; the 28 real routers// land with their module waves (P5-P8).+import { gscRouter } from './modules/gsc/gsc.rpc';import { healthRouter } from './modules/health/health.rpc';import { insightRouter } from './modules/insight/insight.rpc';import { createTRPCRouter } from './rpc/base';@@ -11,6 +12,7 @@ import { createTRPCRouter } from './rpc/base';export const appRouter = createTRPCRouter({health: healthRouter,insight: insightRouter,+ gsc: gscRouter,});export type AppRouter = typeof appRouter;diff --git a/packages/core/src/services.ts b/packages/core/src/services.tsindex 2c049de16..5284c1dd1 100644--- a/packages/core/src/services.ts+++ b/packages/core/src/services.ts@@ -7,6 +7,7 @@ import {type AuthService,createAuthService,} from './modules/auth/auth.service';+import { createGscService, type GscService } from './modules/gsc/gsc.service';import {createInsightService,type InsightService,@@ -30,6 +31,7 @@ export type ServiceDeps = Pick<export interface Services {auth: AuthService;insight: InsightService;+ gsc: GscService;}export function createServices(deps: ServiceDeps): Services {@@ -39,5 +41,6 @@ export function createServices(deps: ServiceDeps): Services {return {auth: createAuthService(deps),insight: createInsightService(deps),+ gsc: createGscService(deps),};}diff --git a/packages/db/src/gsc.ts b/packages/db/src/gsc.tsindex c23a98a86..3adbd1268 100644--- a/packages/db/src/gsc.ts+++ b/packages/db/src/gsc.ts@@ -1,653 +1,15 @@-import { cacheable } from '@openpanel/redis';-import { originalCh } from './clickhouse/client';-import { decrypt, encrypt } from './encryption';-import { createLogger } from './logger';-import { db } from './prisma-client';--const logger = createLogger({ name: 'db:gsc' });--export interface GscSite {- siteUrl: string;- permissionLevel: string;-}--async function refreshGscToken(- refreshToken: string-): Promise<{ accessToken: string; expiresAt: Date }> {- if (!(process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET)) {- throw new Error(- 'GOOGLE_CLIENT_ID or GOOGLE_CLIENT_SECRET is not set in this environment'- );- }-- const params = new URLSearchParams({- client_id: process.env.GOOGLE_CLIENT_ID,- client_secret: process.env.GOOGLE_CLIENT_SECRET,- refresh_token: refreshToken,- grant_type: 'refresh_token',- });-- const res = await fetch('https://oauth2.googleapis.com/token', {- method: 'POST',- headers: { 'Content-Type': 'application/x-www-form-urlencoded' },- body: params.toString(),- });-- if (!res.ok) {- const text = await res.text();- throw new Error(`Failed to refresh GSC token: ${text}`);- }-- const data = (await res.json()) as {- access_token: string;- expires_in: number;- };- const expiresAt = new Date(Date.now() + data.expires_in * 1000);- return { accessToken: data.access_token, expiresAt };-}--export async function getGscAccessToken(projectId: string): Promise<string> {- const conn = await db.gscConnection.findUniqueOrThrow({- where: { projectId },- });-- if (- conn.accessTokenExpiresAt &&- conn.accessTokenExpiresAt.getTime() > Date.now() + 60_000- ) {- logger.info(- { projectId, expiresAt: conn.accessTokenExpiresAt },- 'GSC using cached access token'- );- return decrypt(conn.accessToken);- }-- logger.info(- {- projectId,- expiresAt: conn.accessTokenExpiresAt,- hasRefreshToken: !!conn.refreshToken,- },- 'GSC access token expired, attempting refresh'- );-- try {- const { accessToken, expiresAt } = await refreshGscToken(- decrypt(conn.refreshToken)- );- await db.gscConnection.update({- where: { projectId },- data: {- accessToken: encrypt(accessToken),- accessTokenExpiresAt: expiresAt,- },- });- logger.info({ projectId, expiresAt }, 'GSC token refreshed successfully');- return accessToken;- } catch (error) {- const errorMessage =- error instanceof Error ? error.message : 'Failed to refresh token';- logger.error(- { err: error, projectId, errorMessage },- 'GSC token refresh failed'- );- await db.gscConnection.update({- where: { projectId },- data: {- lastSyncStatus: 'token_expired',- lastSyncError: errorMessage,- },- });- throw new Error(- `GSC token refresh failed for project ${projectId}: ${errorMessage}`- );- }-}--export async function listGscSites(projectId: string): Promise<GscSite[]> {- const accessToken = await getGscAccessToken(projectId);- const res = await fetch('https://www.googleapis.com/webmasters/v3/sites', {- headers: { Authorization: `Bearer ${accessToken}` },- });-- if (!res.ok) {- const text = await res.text();- throw new Error(`Failed to list GSC sites: ${text}`);- }-- const data = (await res.json()) as {- siteEntry?: Array<{ siteUrl: string; permissionLevel: string }>;- };- return data.siteEntry ?? [];-}--interface GscApiRow {- keys: string[];- clicks: number;- impressions: number;- ctr: number;- position: number;-}--interface GscDimensionFilter {- dimension: string;- operator: string;- expression: string;-}--interface GscFilterGroup {- filters: GscDimensionFilter[];-}--async function queryGscSearchAnalytics(- accessToken: string,- siteUrl: string,- startDate: string,- endDate: string,- dimensions: string[],- dimensionFilterGroups?: GscFilterGroup[]-): Promise<GscApiRow[]> {- const encodedSiteUrl = encodeURIComponent(siteUrl);- const url = `https://www.googleapis.com/webmasters/v3/sites/${encodedSiteUrl}/searchAnalytics/query`;-- const allRows: GscApiRow[] = [];- let startRow = 0;- const rowLimit = 25_000;-- while (true) {- const res = await fetch(url, {- method: 'POST',- headers: {- Authorization: `Bearer ${accessToken}`,- 'Content-Type': 'application/json',- },- body: JSON.stringify({- startDate,- endDate,- dimensions,- rowLimit,- startRow,- dataState: 'all',- ...(dimensionFilterGroups && { dimensionFilterGroups }),- }),- });-- if (!res.ok) {- const text = await res.text();- throw new Error(- `GSC query failed for dimensions [${dimensions.join(',')}]: ${text}`- );- }-- const data = (await res.json()) as { rows?: GscApiRow[] };- const rows = data.rows ?? [];- allRows.push(...rows);-- if (rows.length < rowLimit) {- break;- }- startRow += rowLimit;- }-- return allRows;-}--function formatDate(date: Date): string {- return date.toISOString().slice(0, 10);-}--function nowString(): string {- return new Date().toISOString().replace('T', ' ').replace('Z', '');-}--export async function syncGscData(- projectId: string,- startDate: Date,- endDate: Date-): Promise<void> {- const conn = await db.gscConnection.findUniqueOrThrow({- where: { projectId },- });-- if (!conn.siteUrl) {- throw new Error('No GSC site URL configured for this project');- }-- const accessToken = await getGscAccessToken(projectId);- const start = formatDate(startDate);- const end = formatDate(endDate);- const syncedAt = nowString();-- // 1. Daily totals — authoritative numbers for overview chart- const dailyRows = await queryGscSearchAnalytics(- accessToken,- conn.siteUrl,- start,- end,- ['date']- );-- if (dailyRows.length > 0) {- await originalCh.insert({- table: 'gsc_daily',- values: dailyRows.map((row) => ({- project_id: projectId,- date: row.keys[0] ?? '',- clicks: row.clicks,- impressions: row.impressions,- ctr: row.ctr,- position: row.position,- synced_at: syncedAt,- })),- format: 'JSONEachRow',- });- }-- // 2. Per-page breakdown- const pageRows = await queryGscSearchAnalytics(- accessToken,- conn.siteUrl,- start,- end,- ['date', 'page']- );-- if (pageRows.length > 0) {- await originalCh.insert({- table: 'gsc_pages_daily',- values: pageRows.map((row) => ({- project_id: projectId,- date: row.keys[0] ?? '',- page: row.keys[1] ?? '',- clicks: row.clicks,- impressions: row.impressions,- ctr: row.ctr,- position: row.position,- synced_at: syncedAt,- })),- format: 'JSONEachRow',- });- }-- // 3. Per-query breakdown- const queryRows = await queryGscSearchAnalytics(- accessToken,- conn.siteUrl,- start,- end,- ['date', 'query']- );-- if (queryRows.length > 0) {- await originalCh.insert({- table: 'gsc_queries_daily',- values: queryRows.map((row) => ({- project_id: projectId,- date: row.keys[0] ?? '',- query: row.keys[1] ?? '',- clicks: row.clicks,- impressions: row.impressions,- ctr: row.ctr,- position: row.position,- synced_at: syncedAt,- })),- format: 'JSONEachRow',- });- }-}--export async function getGscOverview(- projectId: string,- startDate: string,- endDate: string,- interval: 'day' | 'week' | 'month' = 'day'-): Promise<- Array<{- date: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- }>-> {- const dateExpr =- interval === 'month'- ? 'toStartOfMonth(date)'- : interval === 'week'- ? 'toStartOfWeek(date)'- : 'date';-- const result = await originalCh.query({- query: `- SELECT- ${dateExpr} as date,- sum(clicks) as clicks,- sum(impressions) as impressions,- avg(ctr) as ctr,- avg(position) as position- FROM gsc_daily- FINAL- WHERE project_id = {projectId: String}- AND date >= {startDate: String}- AND date <= {endDate: String}- GROUP BY date- ORDER BY date ASC- `,- query_params: { projectId, startDate, endDate },- format: 'JSONEachRow',- });- return result.json();-}--export async function getGscPages(- projectId: string,- startDate: string,- endDate: string,- limit = 100-): Promise<- Array<{- page: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- }>-> {- const result = await originalCh.query({- query: `- SELECT- page,- sum(clicks) as clicks,- sum(impressions) as impressions,- avg(ctr) as ctr,- avg(position) as position- FROM gsc_pages_daily- FINAL- WHERE project_id = {projectId: String}- AND date >= {startDate: String}- AND date <= {endDate: String}- GROUP BY page- ORDER BY clicks DESC- LIMIT {limit: UInt32}- `,- query_params: { projectId, startDate, endDate, limit },- format: 'JSONEachRow',- });- return result.json();-}--export interface GscCannibalizedQuery {- query: string;- totalImpressions: number;- totalClicks: number;- pages: Array<{- page: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- }>;-}--export const getGscCannibalization = cacheable(- async (- projectId: string,- startDate: string,- endDate: string- ): Promise<GscCannibalizedQuery[]> => {- const conn = await db.gscConnection.findUniqueOrThrow({- where: { projectId },- });- const accessToken = await getGscAccessToken(projectId);-- const rows = await queryGscSearchAnalytics(- accessToken,- conn.siteUrl,- startDate,- endDate,- ['query', 'page']- );-- const map = new Map<- string,- {- totalImpressions: number;- totalClicks: number;- pages: GscCannibalizedQuery['pages'];- }- >();-- for (const row of rows) {- const query = row.keys[0] ?? '';- // Strip hash fragments — GSC records heading anchors (e.g. /page#section)- // as separate URLs but Google treats them as the same page- let page = row.keys[1] ?? '';- try {- const u = new URL(page);- u.hash = '';- page = u.toString();- } catch {- page = page.split('#')[0] ?? page;- }-- const entry = map.get(query) ?? {- totalImpressions: 0,- totalClicks: 0,- pages: [],- };- entry.totalImpressions += row.impressions;- entry.totalClicks += row.clicks;- // Merge into existing page entry if already seen (from a different hash variant)- const existing = entry.pages.find((p) => p.page === page);- if (existing) {- const totalImpressions = existing.impressions + row.impressions;- if (totalImpressions > 0) {- existing.position =- (existing.position * existing.impressions +- row.position * row.impressions) /- totalImpressions;- }- existing.clicks += row.clicks;- existing.impressions += row.impressions;- existing.ctr =- existing.impressions > 0 ? existing.clicks / existing.impressions : 0;- } else {- entry.pages.push({- page,- clicks: row.clicks,- impressions: row.impressions,- ctr: row.ctr,- position: row.position,- });- }- map.set(query, entry);- }-- return [...map.entries()]- .filter(([, v]) => v.pages.length >= 2 && v.totalImpressions >= 100)- .sort(([, a], [, b]) => b.totalImpressions - a.totalImpressions)- .slice(0, 50)- .map(([query, v]) => ({- query,- totalImpressions: v.totalImpressions,- totalClicks: v.totalClicks,- pages: v.pages.sort((a, b) =>- a.position !== b.position- ? a.position - b.position- : b.impressions - a.impressions- ),- }));- },- 60 * 60 * 4-);--export async function getGscPageDetails(- projectId: string,- page: string,- startDate: string,- endDate: string-): Promise<{- timeseries: Array<{- date: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- }>;- queries: Array<{- query: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- }>;-}> {- const conn = await db.gscConnection.findUniqueOrThrow({- where: { projectId },- });- const accessToken = await getGscAccessToken(projectId);- const filterGroups: GscFilterGroup[] = [- { filters: [{ dimension: 'page', operator: 'equals', expression: page }] },- ];-- const [timeseriesRows, queryRows] = await Promise.all([- queryGscSearchAnalytics(- accessToken,- conn.siteUrl,- startDate,- endDate,- ['date'],- filterGroups- ),- queryGscSearchAnalytics(- accessToken,- conn.siteUrl,- startDate,- endDate,- ['query'],- filterGroups- ),- ]);-- return {- timeseries: timeseriesRows.map((row) => ({- date: row.keys[0] ?? '',- clicks: row.clicks,- impressions: row.impressions,- ctr: row.ctr,- position: row.position,- })),- queries: queryRows.map((row) => ({- query: row.keys[0] ?? '',- clicks: row.clicks,- impressions: row.impressions,- ctr: row.ctr,- position: row.position,- })),- };-}--export async function getGscQueryDetails(- projectId: string,- query: string,- startDate: string,- endDate: string-): Promise<{- timeseries: Array<{- date: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- }>;- pages: Array<{- page: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- }>;-}> {- const conn = await db.gscConnection.findUniqueOrThrow({- where: { projectId },- });- const accessToken = await getGscAccessToken(projectId);- const filterGroups: GscFilterGroup[] = [- {- filters: [{ dimension: 'query', operator: 'equals', expression: query }],- },- ];-- const [timeseriesRows, pageRows] = await Promise.all([- queryGscSearchAnalytics(- accessToken,- conn.siteUrl,- startDate,- endDate,- ['date'],- filterGroups- ),- queryGscSearchAnalytics(- accessToken,- conn.siteUrl,- startDate,- endDate,- ['page'],- filterGroups- ),- ]);-- return {- timeseries: timeseriesRows.map((row) => ({- date: row.keys[0] ?? '',- clicks: row.clicks,- impressions: row.impressions,- ctr: row.ctr,- position: row.position,- })),- pages: pageRows.map((row) => ({- page: row.keys[0] ?? '',- clicks: row.clicks,- impressions: row.impressions,- ctr: row.ctr,- position: row.position,- })),- };-}--export async function getGscQueries(- projectId: string,- startDate: string,- endDate: string,- limit = 100-): Promise<- Array<{- query: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- }>-> {- const result = await originalCh.query({- query: `- SELECT- query,- sum(clicks) as clicks,- sum(impressions) as impressions,- avg(ctr) as ctr,- avg(position) as position- FROM gsc_queries_daily- FINAL- WHERE project_id = {projectId: String}- AND date >= {startDate: String}- AND date <= {endDate: String}- GROUP BY query- ORDER BY clicks DESC- LIMIT {limit: UInt32}- `,- query_params: { projectId, startDate, endDate, limit },- format: 'JSONEachRow',- });- return result.json();-}+// The GSC OAuth token lifecycle, the Search Console API client and its+// ClickHouse read/write live in @openpanel/core now (M5-002). Re-exported+// here for existing `@openpanel/db` importers (packages/mcp's gsc tools) —+// same shape as packages/db/src/encryption.ts since M4-006.+export type { GscCannibalizedQuery, GscSite } from '@openpanel/core';+export {+ getGscCannibalization,+ getGscOverview,+ getGscPageDetails,+ getGscPages,+ getGscQueries,+ getGscQueryDetails,+ listGscSites,+ syncGscData,+} from '@openpanel/core';diff --git a/packages/db/src/services/gsc.service.ts b/packages/db/src/services/gsc.service.tsindex 8314ef361..75b8e28f4 100644--- a/packages/db/src/services/gsc.service.ts+++ b/packages/db/src/services/gsc.service.ts@@ -1,194 +1,14 @@-import { getGscCannibalization, getGscOverview, getGscPageDetails, getGscPages, getGscQueryDetails, getGscQueries } from '../gsc';--export interface GscQueryOpportunity {- query: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- opportunity_score: number;- reason: string;-}--function computeOpportunities(- queries: Array<{- query: string;- clicks: number;- impressions: number;- ctr: number;- position: number;- }>,-): GscQueryOpportunity[] {- const ctrBenchmarks: Record<string, number> = {- '1': 0.28,- '2': 0.15,- '3': 0.11,- '4-6': 0.065,- '7-10': 0.035,- '11-20': 0.012,- };-- function getBenchmark(position: number): number {- if (position <= 1) return ctrBenchmarks['1'] ?? 0.28;- if (position <= 2) return ctrBenchmarks['2'] ?? 0.15;- if (position <= 3) return ctrBenchmarks['3'] ?? 0.11;- if (position <= 6) return ctrBenchmarks['4-6'] ?? 0.065;- if (position <= 10) return ctrBenchmarks['7-10'] ?? 0.035;- return ctrBenchmarks['11-20'] ?? 0.012;- }-- return queries- .filter((q) => q.position >= 4 && q.position <= 20 && q.impressions >= 50)- .map((q) => {- const benchmark = getBenchmark(q.position);- const ctrGap = Math.max(0, benchmark - q.ctr);- const opportunity_score =- Math.round(q.impressions * (1 / q.position) * (1 + ctrGap) * 100) /- 100;-- let reason: string;- if (q.position <= 6) {- reason = `Position ${q.position.toFixed(1)} — one rank improvement could significantly boost clicks`;- } else if (q.ctr < benchmark * 0.5) {- reason = `CTR (${(q.ctr * 100).toFixed(1)}%) is well below expected ${(benchmark * 100).toFixed(1)}% — title/meta optimization may help`;- } else {- reason = `Position ${q.position.toFixed(1)} with ${q.impressions} impressions — push to page 1 for major gains`;- }-- return {- query: q.query,- clicks: q.clicks,- impressions: q.impressions,- ctr: Math.round(q.ctr * 10000) / 100,- position: Math.round(q.position * 10) / 10,- opportunity_score,- reason,- };- })- .sort((a, b) => b.opportunity_score - a.opportunity_score)- .slice(0, 50);-}--export async function gscGetOverviewCore(input: {- projectId: string;- startDate: string;- endDate: string;- interval?: 'day' | 'week' | 'month';-}) {- const data = await getGscOverview(- input.projectId,- input.startDate,- input.endDate,- input.interval ?? 'day',- );- return {- data,- summary: {- total_clicks: data.reduce((s, r) => s + r.clicks, 0),- total_impressions: data.reduce((s, r) => s + r.impressions, 0),- avg_ctr:- data.length > 0- ? Math.round(- (data.reduce((s, r) => s + r.ctr, 0) / data.length) * 10000,- ) / 100- : 0,- avg_position:- data.length > 0- ? Math.round(- (data.reduce((s, r) => s + r.position, 0) / data.length) * 10,- ) / 10- : 0,- },- };-}--export async function gscGetTopPagesCore(input: {- projectId: string;- startDate: string;- endDate: string;- limit?: number;-}) {- return getGscPages(- input.projectId,- input.startDate,- input.endDate,- input.limit ?? 100,- );-}--export async function gscGetPageDetailsCore(input: {- projectId: string;- startDate: string;- endDate: string;- page: string;-}) {- return getGscPageDetails(- input.projectId,- input.page,- input.startDate,- input.endDate,- );-}--export async function gscGetTopQueriesCore(input: {- projectId: string;- startDate: string;- endDate: string;- limit?: number;-}) {- return getGscQueries(- input.projectId,- input.startDate,- input.endDate,- input.limit ?? 100,- );-}--export async function gscGetQueryOpportunitiesCore(input: {- projectId: string;- startDate: string;- endDate: string;- minImpressions?: number;-}) {- const queries = await getGscQueries(- input.projectId,- input.startDate,- input.endDate,- 5000,- );- const filtered = queries.filter(- (q) => q.impressions >= (input.minImpressions ?? 50),- );- const opportunities = computeOpportunities(filtered);- return {- opportunities,- total_analyzed: filtered.length,- min_impressions: input.minImpressions ?? 50,- };-}--export async function gscGetQueryDetailsCore(input: {- projectId: string;- startDate: string;- endDate: string;- query: string;-}) {- return getGscQueryDetails(- input.projectId,- input.query,- input.startDate,- input.endDate,- );-}--export async function gscGetCannibalizationCore(input: {- projectId: string;- startDate: string;- endDate: string;-}) {- return getGscCannibalization(- input.projectId,- input.startDate,- input.endDate,- );-}+// The AI-tool wrapper functions live in @openpanel/core now (M5-002).+// Re-exported here for existing `@openpanel/db` importers (the assistant's+// SEO tools, the /insights REST controller) — same shape as+// packages/db/src/encryption.ts since M4-006.+export type { GscQueryOpportunity } from '@openpanel/core';+export {+ gscGetCannibalizationCore,+ gscGetOverviewCore,+ gscGetPageDetailsCore,+ gscGetQueryDetailsCore,+ gscGetQueryOpportunitiesCore,+ gscGetTopPagesCore,+ gscGetTopQueriesCore,+} from '@openpanel/core';diff --git a/packages/trpc/src/routers/gsc.ts b/packages/trpc/src/routers/gsc.tsindex d1a732aff..d86bd6df9 100644--- a/packages/trpc/src/routers/gsc.ts+++ b/packages/trpc/src/routers/gsc.ts@@ -1,25 +1,38 @@-import { Arctic, googleGsc } from '@openpanel/core';+// Dissolved into @openpanel/core's gsc module (M5-002): the OAuth token+// lifecycle, the Search Console client, the ClickHouse read path and the+// AI/search-engine breakdown logic moved to+// packages/core/src/modules/gsc/gsc.service.ts. This router stays+// (DELEGATE PATTERN): it keeps V1's protectedProcedure stack (session/access/+// logger/rate-limit middleware) and its own gscConnection CRUD untouched, and+// delegates every read/write of GSC data to the core service.import {- chQuery,- db,- getChartStartEndDate,+ Arctic,+ disconnectGscConnection,+ getGscAiEngines,getGscCannibalization,+ getGscConnection,getGscOverview,getGscPageDetails,getGscPages,+ getGscPreviousOverview,getGscQueries,getGscQueryDetails,- getSettingsForProject,+ getGscSearchEngines,+ googleGsc,listGscSites,- TABLE_NAMES,-} from '@openpanel/db';+ resolveGscDateRange,+ selectGscSite,+} from '@openpanel/core';import { gscQueue } from '@openpanel/queue';import { zRange, zTimeInterval } from '@openpanel/validation';import { z } from 'zod';import { getProjectAccess, requireProjectAccess } from '../access';-import { TRPCForbiddenError, TRPCNotFoundError } from '../errors';+import { TRPCForbiddenError } from '../errors';import { createTRPCRouter, protectedProcedure } from '../trpc';+const OAUTH_COOKIE_MAX_AGE_SECONDS = 60 * 10;+const GSC_OVERVIEW_INTERVALS = ['day', 'week', 'month'] as const;+const zGscDateInput = z.object({projectId: z.string(),range: zRange,@@ -28,142 +41,27 @@ const zGscDateInput = z.object({endDate: z.string().nullish(),});-async function resolveDates(- projectId: string,- input: { range: string; startDate?: string | null; endDate?: string | null }-) {- const { timezone } = await getSettingsForProject(projectId);- const { startDate, endDate } = getChartStartEndDate(- {- range: input.range as any,- startDate: input.startDate,- endDate: input.endDate,- },- timezone- );- return {- startDate: startDate.slice(0, 10),- endDate: endDate.slice(0, 10),- };+function toOverviewInterval(+ interval: string+): (typeof GSC_OVERVIEW_INTERVALS)[number] {+ return (GSC_OVERVIEW_INTERVALS as readonly string[]).includes(interval)+ ? (interval as (typeof GSC_OVERVIEW_INTERVALS)[number])+ : 'day';}-/**- * ClickHouse stores the same AI referrer under several spellings: the parsed- * display name ('ChatGPT', 'Google Gemini'), the bare host ('chatgpt.com') and- * the full origin ('https://kagi.com'). Normalize before matching so every- * spelling lands on the same engine.- */-const NORMALIZED_REFERRER_NAME =- "lower(regexp_replace(referrer_name, '^https?://(www[.])?', ''))";--const AI_REFERRERS = [- {- canonical: 'chatgpt.com',- aliases: [- 'chatgpt',- 'chatgpt.com',- 'chat.openai.com',- 'openai',- 'openai.com',- ],- },- {- canonical: 'claude.ai',- aliases: ['claude', 'claude.ai', 'anthropic', 'anthropic.com'],- },- {- canonical: 'perplexity.ai',- aliases: ['perplexity', 'perplexity.ai'],- },- {- canonical: 'gemini.google.com',- aliases: [- 'gemini',- 'google gemini',- 'gemini.google.com',- 'bard.google.com',- ],- },- {- canonical: 'copilot.com',- aliases: [- 'copilot',- 'copilot.com',- 'copilot.microsoft.com',- 'microsoft copilot',- ],- },- { canonical: 'grok.com', aliases: ['grok', 'grok.com'] },- {- canonical: 'mistral.ai',- aliases: ['mistral', 'mistral.ai', 'chat.mistral.ai', 'le chat'],- },- {- canonical: 'kagi.com',- aliases: ['kagi', 'kagi.com', 'assistant.kagi.com'],- },-] as const satisfies ReadonlyArray<{- canonical: string;- aliases: readonly string[];-}>;--const quoteList = (values: readonly string[]) =>- values.map((value) => `'${value}'`).join(', ');--/** `norm` is the alias bound by AI_REFERRER_CTE below. */-const AI_REFERRER_CTE = `WITH ${NORMALIZED_REFERRER_NAME} AS norm`;--const AI_REFERRER_FILTER = `norm IN (${quoteList(- AI_REFERRERS.flatMap((engine) => engine.aliases)-)})`;--/** Collapses every alias onto one row per engine. */-const AI_REFERRER_CANONICAL_NAME = `multiIf(${AI_REFERRERS.map(- (engine) => `norm IN (${quoteList(engine.aliases)}), '${engine.canonical}'`-).join(', ')}, norm)`;--/**- * Half-open windows so the last day of the range is included and the previous- * window ends exactly where the current one starts (no gap, no overlap).- */-function getComparisonWindows(startDate: string, endDate: string) {- const start = new Date(`${startDate}T00:00:00Z`);- const endExclusive = new Date(`${endDate}T00:00:00Z`);- endExclusive.setUTCDate(endExclusive.getUTCDate() + 1);- const previousStart = new Date(- start.getTime() - (endExclusive.getTime() - start.getTime())- );- const fmt = (date: Date) => date.toISOString().slice(0, 19).replace('T', ' ');- return {- current: { start: fmt(start), end: fmt(endExclusive) },- previous: { start: fmt(previousStart), end: fmt(start) },- };+async function requireRead(userId: string, projectId: string) {+ const access = await getProjectAccess({ userId, projectId });+ if (!access) {+ throw new TRPCForbiddenError('You do not have access to this project');+ }}export const gscRouter = createTRPCRouter({getConnection: protectedProcedure.input(z.object({ projectId: z.string() })).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- return db.gscConnection.findUnique({- where: { projectId: input.projectId },- select: {- id: true,- siteUrl: true,- lastSyncedAt: true,- lastSyncStatus: true,- lastSyncError: true,- backfillStatus: true,- createdAt: true,- updatedAt: true,- },- });+ await requireRead(ctx.session.userId, input.projectId);+ return getGscConnection(input.projectId);}),initiateOAuth: protectedProcedure@@ -183,7 +81,7 @@ export const gscRouter = createTRPCRouter({url.searchParams.set('access_type', 'offline');url.searchParams.set('prompt', 'consent');- const cookieOpts = { maxAge: 60 * 10, signed: true };+ const cookieOpts = { maxAge: OAUTH_COOKIE_MAX_AGE_SECONDS, signed: true };ctx.setCookie('gsc_oauth_state', state, cookieOpts);ctx.setCookie('gsc_code_verifier', codeVerifier, cookieOpts);ctx.setCookie('gsc_project_id', input.projectId, cookieOpts);@@ -194,13 +92,7 @@ export const gscRouter = createTRPCRouter({getSites: protectedProcedure.input(z.object({ projectId: z.string() })).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }+ await requireRead(ctx.session.userId, input.projectId);return listGscSites(input.projectId);}),@@ -213,20 +105,7 @@ export const gscRouter = createTRPCRouter({level: 'write',});- const conn = await db.gscConnection.findUnique({- where: { projectId: input.projectId },- });- if (!conn) {- throw new TRPCNotFoundError('GSC connection not found');- }-- await db.gscConnection.update({- where: { projectId: input.projectId },- data: {- siteUrl: input.siteUrl,- backfillStatus: 'pending',- },- });+ await selectGscSite(input.projectId, input.siteUrl);await gscQueue.add('gscProjectBackfill', {type: 'gscProjectBackfill',@@ -245,28 +124,24 @@ export const gscRouter = createTRPCRouter({level: 'write',});- await db.gscConnection.deleteMany({- where: { projectId: input.projectId },- });-+ await disconnectGscConnection(input.projectId);return { ok: true };}),getOverview: protectedProcedure.input(zGscDateInput).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- const { startDate, endDate } = await resolveDates(input.projectId, input);- const interval = ['day', 'week', 'month'].includes(input.interval)- ? (input.interval as 'day' | 'week' | 'month')- : 'day';- return getGscOverview(input.projectId, startDate, endDate, interval);+ await requireRead(ctx.session.userId, input.projectId);+ const { startDate, endDate } = await resolveGscDateRange(+ input.projectId,+ input+ );+ return getGscOverview(+ input.projectId,+ startDate,+ endDate,+ toOverviewInterval(input.interval)+ );}),getPages: protectedProcedure@@ -276,42 +151,33 @@ export const gscRouter = createTRPCRouter({})).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- const { startDate, endDate } = await resolveDates(input.projectId, input);+ await requireRead(ctx.session.userId, input.projectId);+ const { startDate, endDate } = await resolveGscDateRange(+ input.projectId,+ input+ );return getGscPages(input.projectId, startDate, endDate, input.limit);}),getPageDetails: protectedProcedure.input(zGscDateInput.extend({ page: z.string() })).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- const { startDate, endDate } = await resolveDates(input.projectId, input);+ await requireRead(ctx.session.userId, input.projectId);+ const { startDate, endDate } = await resolveGscDateRange(+ input.projectId,+ input+ );return getGscPageDetails(input.projectId, input.page, startDate, endDate);}),getQueryDetails: protectedProcedure.input(zGscDateInput.extend({ query: z.string() })).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- const { startDate, endDate } = await resolveDates(input.projectId, input);+ await requireRead(ctx.session.userId, input.projectId);+ const { startDate, endDate } = await resolveGscDateRange(+ input.projectId,+ input+ );return getGscQueryDetails(input.projectId,input.query,@@ -327,157 +193,60 @@ export const gscRouter = createTRPCRouter({})).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- const { startDate, endDate } = await resolveDates(input.projectId, input);+ await requireRead(ctx.session.userId, input.projectId);+ const { startDate, endDate } = await resolveGscDateRange(+ input.projectId,+ input+ );return getGscQueries(input.projectId, startDate, endDate, input.limit);}),getSearchEngines: protectedProcedure.input(zGscDateInput).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- const { startDate, endDate } = await resolveDates(input.projectId, input);- const windows = getComparisonWindows(startDate, endDate);-- const where = (window: { start: string; end: string }) =>- `project_id = '${input.projectId}'- AND referrer_type = 'search'- AND created_at >= '${window.start}'- AND created_at < '${window.end}'`;-- const [engines, [currentResult], [prevResult]] = await Promise.all([- chQuery<{ name: string; sessions: number }>(- `SELECT- referrer_name as name,- count(*) as sessions- FROM ${TABLE_NAMES.sessions}- WHERE ${where(windows.current)}- GROUP BY name- ORDER BY sessions DESC- LIMIT 10`- ),- chQuery<{ sessions: number }>(- `SELECT count(*) as sessions- FROM ${TABLE_NAMES.sessions}- WHERE ${where(windows.current)}`- ),- chQuery<{ sessions: number }>(- `SELECT count(*) as sessions- FROM ${TABLE_NAMES.sessions}- WHERE ${where(windows.previous)}`- ),- ]);-- return {- engines,- // Counted separately from `engines`, which is capped at the top 10, so- // the total compares like-for-like with the previous period.- total: currentResult?.sessions ?? 0,- previousTotal: prevResult?.sessions ?? 0,- };+ await requireRead(ctx.session.userId, input.projectId);+ const { startDate, endDate } = await resolveGscDateRange(+ input.projectId,+ input+ );+ return getGscSearchEngines(input.projectId, startDate, endDate);}),getAiEngines: protectedProcedure.input(zGscDateInput).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- const { startDate, endDate } = await resolveDates(input.projectId, input);- const windows = getComparisonWindows(startDate, endDate);-- // Matched by name — will switch to referrer_type = 'ai' once available.- const where = (window: { start: string; end: string }) =>- `project_id = '${input.projectId}'- AND ${AI_REFERRER_FILTER}- AND created_at >= '${window.start}'- AND created_at < '${window.end}'`;-- const [engines, [prevResult]] = await Promise.all([- chQuery<{ name: string; sessions: number }>(- `${AI_REFERRER_CTE}- SELECT ${AI_REFERRER_CANONICAL_NAME} as name, count(*) as sessions- FROM ${TABLE_NAMES.sessions}- WHERE ${where(windows.current)}- GROUP BY name- ORDER BY sessions DESC`- ),- chQuery<{ sessions: number }>(- `${AI_REFERRER_CTE}- SELECT count(*) as sessions- FROM ${TABLE_NAMES.sessions}- WHERE ${where(windows.previous)}`- ),- ]);-- return {- engines,- // The filter is a fixed alias set, so the grouped rows are the whole- // population — no LIMIT, and the total matches the previous period.- total: engines.reduce((sum, engine) => sum + engine.sessions, 0),- previousTotal: prevResult?.sessions ?? 0,- };+ await requireRead(ctx.session.userId, input.projectId);+ const { startDate, endDate } = await resolveGscDateRange(+ input.projectId,+ input+ );+ return getGscAiEngines(input.projectId, startDate, endDate);}),getPreviousOverview: protectedProcedure.input(zGscDateInput).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- const { startDate, endDate } = await resolveDates(input.projectId, input);-- const startMs = new Date(startDate).getTime();- const duration = new Date(endDate).getTime() - startMs;- const prevEnd = new Date(startMs - 1);- const prevStart = new Date(prevEnd.getTime() - duration);- const fmt = (d: Date) => d.toISOString().slice(0, 10);-- const interval = (['day', 'week', 'month'] as const).includes(- input.interval as 'day' | 'week' | 'month'- )- ? (input.interval as 'day' | 'week' | 'month')- : 'day';-- return getGscOverview(+ await requireRead(ctx.session.userId, input.projectId);+ const { startDate, endDate } = await resolveGscDateRange(input.projectId,- fmt(prevStart),- fmt(prevEnd),- interval+ input+ );+ return getGscPreviousOverview(+ input.projectId,+ startDate,+ endDate,+ toOverviewInterval(input.interval));}),getCannibalization: protectedProcedure.input(zGscDateInput).query(async ({ input, ctx }) => {- const access = await getProjectAccess({- projectId: input.projectId,- userId: ctx.session.userId,- });- if (!access) {- throw new TRPCForbiddenError('You do not have access to this project');- }- const { startDate, endDate } = await resolveDates(input.projectId, input);+ await requireRead(ctx.session.userId, input.projectId);+ const { startDate, endDate } = await resolveGscDateRange(+ input.projectId,+ input+ );return getGscCannibalization(input.projectId, startDate, endDate);}),});