Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): move from Supabase SDK to Drizzle (#3698)
Gergő Móricz0566d52a6a45
diff --git a/apps/api/knip.config.ts b/apps/api/knip.config.tsindex ee1316465..9e9b4eee4 100644--- a/apps/api/knip.config.ts+++ b/apps/api/knip.config.ts@@ -15,10 +15,6 @@ const config: KnipConfig = {ignore: ["native/**","src/scraper/scrapeURL/engines/fire-engine/branding-script/**",- // Legacy auto-recharge files — kept but disabled (Autumn handles auto-recharge now)- "src/services/billing/auto_charge.ts",- "src/services/billing/issue_credits.ts",- "src/services/billing/stripe.ts",// Superseded by the integration proxy added in #3520; kept for now."src/controllers/v0/admin/rotate-api-key.ts",],diff --git a/apps/api/package.json b/apps/api/package.jsonindex 41f4628a9..6ef5b5c73 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -93,7 +93,6 @@"@openrouter/ai-sdk-provider": "^0.4.5","@sentry/cli": "^2.58.2","@sentry/node": "^10.27.0",- "@supabase/supabase-js": "^2.52.0","@types/ws": "^8.5.12","@x402/core": "^2.4.0","@x402/evm": "^2.4.0",@@ -112,6 +111,7 @@"cors": "^2.8.5","culori": "^4.0.2","dotenv": "^16.3.1",+ "drizzle-orm": "1.0.0-rc.3","esbuild": "^0.27.2","escape-html": "^1.0.3","express": "4.22.0",@@ -179,7 +179,6 @@"msgpackr-extract","oxc-resolver","protobufjs",- "supabase","utf-8-validate","wordpos"],diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 50df44203..d8ded7e20 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -89,9 +89,6 @@ importers:'@sentry/node':specifier: ^10.27.0version: 10.27.0- '@supabase/supabase-js':- specifier: ^2.52.0- version: 2.52.0([email protected])([email protected])'@types/ws':specifier: ^8.5.12version: 8.5.12@@ -146,6 +143,9 @@ importers:dotenv:specifier: ^16.3.1version: 16.4.5+ drizzle-orm:+ specifier: 1.0.0-rc.3+ version: 1.0.0-rc.3(@opentelemetry/[email protected])(@sinclair/[email protected])(@types/[email protected])([email protected]([email protected]))([email protected])esbuild:specifier: ^0.27.2version: 0.27.2@@ -2616,28 +2616,6 @@ packages:'@standard-schema/[email protected]':resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==}- '@supabase/[email protected]':- resolution: {integrity: sha512-mMIQHBRc+SKpZFRB2qtupuzulaUhFYupNyxqDj5Jp/LyPvcWvjaJzZzObv6URtL/O6lPxkanASnotGtNpS3H2Q==}-- '@supabase/[email protected]':- resolution: {integrity: sha512-v5GSqb9zbosquTo6gBwIiq7W9eQ7rE5QazsK/ezNiQXdCbY+bH8D9qEaBIkhVvX4ZRW5rP03gEfw5yw9tiq4EQ==}-- '@supabase/[email protected]':- resolution: {integrity: sha512-1ibVeYUacxWYi9i0cf5efil6adJ9WRyZBLivgjs+AUpewx1F3xPi7gLgaASI2SmIQxPoCEjAsLAzKPgMJVgOUQ==}- engines: {node: 4.x || >=6.0.0}-- '@supabase/[email protected]':- resolution: {integrity: sha512-O4soKqKtZIW3olqmbXXbKugUtByD2jPa8kL2m2c1oozAO11uCcGrRhkZL0kVxjBLrXHE0mdSkFsMj7jDSfyNpw==}-- '@supabase/[email protected]':- resolution: {integrity: sha512-HQKRnwAqdVqJW/P9TjKVK+/ETpW4yQ8tyDPPtRMKOH4Uh3vQD74vmj353CYs8+YwVBKubeUOOEpI9CT8mT4obw==}-- '@supabase/[email protected]':- resolution: {integrity: sha512-asYHcyDR1fKqrMpytAS1zjyEfvxuOIp1CIXX7ji4lHHcJKqyk+sLl/Vxgm4sN6u8zvuUtae9e4kDxQP2qrwWBA==}-- '@supabase/[email protected]':- resolution: {integrity: sha512-jbs3CV1f2+ge7sgBeEduboT9v/uGjF22v0yWi/5/XFn5tbM8MfWRccsMtsDwAwu24XK8H6wt2LJDiNnZLtx/bg==}-'@swc/[email protected]':resolution: {integrity: sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==}@@ -2772,9 +2750,6 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==}- '@types/[email protected]':- resolution: {integrity: sha512-PIzZZlEppgrpoT2QgbnDU+MMzuR6BbCjllj0bM70lWoejMeNJAxCchxnv7J3XFkI8MpygtRpzXrIlmWUBclP5A==}-'@types/[email protected]':resolution: {integrity: sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==}@@ -3582,6 +3557,122 @@ packages:resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==}engines: {node: '>=12'}+ resolution: {integrity: sha512-akZOa5UxapFbdBG8IDkfBRpSZJpMHaOJtGgp7oi1oHaiU8S3KN92waHo2l5aRuv1D9tMGYpv3BQFOsiGcNjLTQ==}+ peerDependencies:+ '@aws-sdk/client-rds-data': '>=3'+ '@cloudflare/workers-types': '>=4'+ '@effect/sql-pg': '>=4.0.0-beta.58 || >=4.0.0'+ '@electric-sql/pglite': '>=0.2.0'+ '@libsql/client': '>=0.10.0'+ '@libsql/client-wasm': '>=0.10.0'+ '@neondatabase/serverless': '>=0.10.0'+ '@op-engineering/op-sqlite': '>=2'+ '@opentelemetry/api': ^1.4.1+ '@planetscale/database': '>=1.13'+ '@sinclair/typebox': '>=0.34.8'+ '@sqlitecloud/drivers': '>=1.0.653'+ '@tidbcloud/serverless': '*'+ '@tursodatabase/database': '>=0.2.1'+ '@tursodatabase/database-common': '>=0.2.1'+ '@tursodatabase/database-wasm': '>=0.2.1'+ '@types/better-sqlite3': '*'+ '@types/mssql': ^9.1.4+ '@types/pg': '*'+ '@types/sql.js': '*'+ '@upstash/redis': '>=1.34.7'+ '@vercel/postgres': '>=0.8.0'+ '@xata.io/client': '*'+ arktype: '>=2.0.0'+ better-sqlite3: '>=9.3.0'+ bun-types: '*'+ effect: '>=4.0.0-beta.58 || >=4.0.0'+ expo-sqlite: '>=14.0.0'+ mssql: ^11.0.1+ mysql2: '>=2'+ pg: '>=8'+ postgres: '>=3'+ sql.js: '>=1'+ sqlite3: '>=5'+ typebox: '>=1.0.0'+ valibot: '>=1.0.0-beta.7'+ zod: ^3.25.0 || ^4.0.0+ peerDependenciesMeta:+ '@aws-sdk/client-rds-data':+ optional: true+ '@cloudflare/workers-types':+ optional: true+ '@effect/sql-pg':+ optional: true+ '@electric-sql/pglite':+ optional: true+ '@libsql/client':+ optional: true+ '@libsql/client-wasm':+ optional: true+ '@neondatabase/serverless':+ optional: true+ '@op-engineering/op-sqlite':+ optional: true+ '@opentelemetry/api':+ optional: true+ '@planetscale/database':+ optional: true+ '@sinclair/typebox':+ optional: true+ '@sqlitecloud/drivers':+ optional: true+ '@tidbcloud/serverless':+ optional: true+ '@tursodatabase/database':+ optional: true+ '@tursodatabase/database-common':+ optional: true+ '@tursodatabase/database-wasm':+ optional: true+ '@types/better-sqlite3':+ optional: true+ '@types/mssql':+ optional: true+ '@types/pg':+ optional: true+ '@types/sql.js':+ optional: true+ '@upstash/redis':+ optional: true+ '@vercel/postgres':+ optional: true+ '@xata.io/client':+ optional: true+ arktype:+ optional: true+ better-sqlite3:+ optional: true+ bun-types:+ optional: true+ effect:+ optional: true+ expo-sqlite:+ optional: true+ mssql:+ optional: true+ mysql2:+ optional: true+ pg:+ optional: true+ postgres:+ optional: true+ sql.js:+ optional: true+ sqlite3:+ optional: true+ typebox:+ optional: true+ valibot:+ optional: true+ zod:+ optional: true+resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}engines: {node: '>= 0.4'}@@ -8264,49 +8355,6 @@ snapshots:'@standard-schema/[email protected]': {}- '@supabase/[email protected]':- dependencies:- '@supabase/node-fetch': 2.6.15-- '@supabase/[email protected]':- dependencies:- '@supabase/node-fetch': 2.6.15-- '@supabase/[email protected]':- dependencies:- whatwg-url: 5.0.0-- '@supabase/[email protected]':- dependencies:- '@supabase/node-fetch': 2.6.15-- dependencies:- '@supabase/node-fetch': 2.6.15- '@types/phoenix': 1.6.6- '@types/ws': 8.18.1- ws: 8.20.1([email protected])([email protected])- transitivePeerDependencies:- - bufferutil- - utf-8-validate-- '@supabase/[email protected]':- dependencies:- '@supabase/node-fetch': 2.6.15-- dependencies:- '@supabase/auth-js': 2.71.1- '@supabase/functions-js': 2.4.5- '@supabase/node-fetch': 2.6.15- '@supabase/postgrest-js': 1.19.4- '@supabase/realtime-js': 2.11.15([email protected])([email protected])- '@supabase/storage-js': 2.7.1- transitivePeerDependencies:- - bufferutil- - utf-8-validate-'@swc/[email protected]':dependencies:tslib: 2.8.1@@ -8470,8 +8518,6 @@ snapshots:pg-protocol: 1.10.3pg-types: 2.2.0- '@types/[email protected]': {}-'@types/[email protected]': {}'@types/[email protected]': {}@@ -9238,6 +9284,14 @@ snapshots:+ [email protected](@opentelemetry/[email protected])(@sinclair/[email protected])(@types/[email protected])([email protected]([email protected]))([email protected]):+ optionalDependencies:+ '@opentelemetry/api': 1.9.0+ '@sinclair/typebox': 0.34.41+ '@types/pg': 8.15.5+ pg: 8.16.3([email protected])+ zod: 4.1.12+dependencies:call-bind-apply-helpers: 1.0.2diff --git a/apps/api/src/__tests__/e2e_noAuth/index.test.ts b/apps/api/src/__tests__/e2e_noAuth/index.test.tsindex 65bf979f0..2e3af0196 100644--- a/apps/api/src/__tests__/e2e_noAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_noAuth/index.test.ts@@ -13,9 +13,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {beforeAll(() => {originalConfig = {USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,- SUPABASE_ANON_TOKEN: config.SUPABASE_ANON_TOKEN,- SUPABASE_URL: config.SUPABASE_URL,- SUPABASE_SERVICE_TOKEN: config.SUPABASE_SERVICE_TOKEN,+ DATABASE_URL: config.DATABASE_URL,OPENAI_API_KEY: config.OPENAI_API_KEY,BULL_AUTH_KEY: config.BULL_AUTH_KEY,PLAYWRIGHT_MICROSERVICE_URL: config.PLAYWRIGHT_MICROSERVICE_URL,@@ -23,9 +21,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {TEST_API_KEY: config.TEST_API_KEY,};config.USE_DB_AUTHENTICATION = false;- config.SUPABASE_ANON_TOKEN = "";- config.SUPABASE_URL = "";- config.SUPABASE_SERVICE_TOKEN = "";+ config.DATABASE_URL = "";config.OPENAI_API_KEY = "";config.BULL_AUTH_KEY = "";config.PLAYWRIGHT_MICROSERVICE_URL = "";diff --git a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts b/apps/api/src/__tests__/routes/check-credits-middleware.test.tsindex fd6f09f18..0394bfbe0 100644--- a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts+++ b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts@@ -6,10 +6,6 @@ jest.mock("../../services/autumn/autumn.service", () => ({},}));-jest.mock("../../services/supabase", () => ({- supabase_service: { rpc: jest.fn() },-}));-jest.mock("../../lib/http-metrics", () => ({httpRequestDurationSeconds: { observe: jest.fn() },getRoutePattern: jest.fn(() => "/v1/crawl"),diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.tsindex ce17ebb07..c50895b25 100644--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts@@ -15,11 +15,12 @@ describe("Audio format engine routing (buildFallbackList)", () => {let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines").buildFallbackList;const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;- const originalIndexUrl = process.env.INDEX_SUPABASE_URL;+ const originalIndexUrl = process.env.INDEX_DATABASE_URL;beforeAll(() => {process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";- process.env.INDEX_SUPABASE_URL = "http://test-index-supabase";+ process.env.INDEX_DATABASE_URL =+ "postgresql://postgres:postgres@localhost:5432/postgres";jest.isolateModules(() => {buildFallbackList =@@ -34,9 +35,9 @@ describe("Audio format engine routing (buildFallbackList)", () => {process.env.FIRE_ENGINE_BETA_URL = originalFireEngineUrl;}if (originalIndexUrl === undefined) {- delete process.env.INDEX_SUPABASE_URL;+ delete process.env.INDEX_DATABASE_URL;} else {- process.env.INDEX_SUPABASE_URL = originalIndexUrl;+ process.env.INDEX_DATABASE_URL = originalIndexUrl;}});diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.tsindex 5aa990956..3d1c3613e 100644--- a/apps/api/src/__tests__/snips/v2/parse.test.ts+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts@@ -6,7 +6,9 @@ import {TEST_SUITE_WEBSITE,} from "../lib";import { idmux, Identity, parse, parseWithFailure, scrapeTimeout } from "./lib";-import { supabase_service } from "../../../services/supabase";+import { and, desc, eq } from "drizzle-orm";+import { db } from "../../../db/connection";+import * as schema from "../../../db/schema";import { config } from "../../../config";const DOCX_FIXTURE_BASE64 =@@ -212,16 +214,19 @@ describe("/v2/parse", () => {);const requestLog = await waitForSingleRow<{ id: string }>(async () => {- const { data, error } = await supabase_service- .from("requests")- .select("id")- .eq("team_id", identity.teamId)- .eq("kind", "parse")- .eq("target_hint", filename)- .order("created_at", { ascending: false })+ const data = await db+ .select({ id: schema.requests.id })+ .from(schema.requests)+ .where(+ and(+ eq(schema.requests.team_id, identity.teamId),+ eq(schema.requests.kind, "parse"),+ eq(schema.requests.target_hint, filename),+ ),+ )+ .orderBy(desc(schema.requests.created_at)).limit(1);- if (error) throw error;- return data?.[0] ?? null;+ return data[0] ?? null;});expect(requestLog).not.toBeNull();@@ -230,14 +235,16 @@ describe("/v2/parse", () => {request_id: string;url: string;}>(async () => {- const { data, error } = await supabase_service- .from("parses")- .select("request_id, url")- .eq("request_id", requestLog!.id)- .order("created_at", { ascending: false })+ const data = await db+ .select({+ request_id: schema.parses.request_id,+ url: schema.parses.url,+ })+ .from(schema.parses)+ .where(eq(schema.parses.request_id, requestLog!.id))+ .orderBy(desc(schema.parses.created_at)).limit(1);- if (error) throw error;- return data?.[0] ?? null;+ return data[0] ?? null;});expect(parseLog).not.toBeNull();@@ -246,14 +253,12 @@ describe("/v2/parse", () => {`https://parse.firecrawl.dev/uploads/${encodeURIComponent(filename)}`,);- const { data: scrapeRows, error: scrapeRowsError } =- await supabase_service- .from("scrapes")- .select("id")- .eq("request_id", requestLog!.id)- .limit(1);- expect(scrapeRowsError).toBeFalsy();- expect(scrapeRows ?? []).toHaveLength(0);+ const scrapeRows = await db+ .select({ id: schema.scrapes.id })+ .from(schema.scrapes)+ .where(eq(schema.scrapes.request_id, requestLog!.id))+ .limit(1);+ expect(scrapeRows).toHaveLength(0);},scrapeTimeout,);diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex df41959bf..6899a3ad1 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -1203,7 +1203,7 @@ describe("Scrape tests", () => {identity,);- expect(response.markdown).toContain("| Country | United States |");+ expect(response.markdown).toContain("| Country | United States "); // either United States or United States of America},scrapeTimeout,);@@ -1346,7 +1346,7 @@ describe("Scrape tests", () => {);expect(response.markdown).toContain("PDF Test File");- expect(response.metadata.title).toBe("PDF Test Page");+ expect(response.metadata.title).toContain("PDF Test Page");expect(response.metadata.numPages).toBe(1);},scrapeTimeout,@@ -1373,7 +1373,9 @@ describe("Scrape tests", () => {identity,);- expect(response.error).toContain("Insufficient time to process PDF");+ expect(response.error).toContain(+ "pages, which requires more processing time than your current timeout allows.",+ );},12000,);diff --git a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts b/apps/api/src/__tests__/snips/v2/search-feedback.test.tsindex a36288a94..df36ad307 100644--- a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts+++ b/apps/api/src/__tests__/snips/v2/search-feedback.test.ts@@ -7,7 +7,9 @@ import {idmux,Identity,} from "./lib";-import { supabase_service } from "../../../services/supabase";+import { and, eq } from "drizzle-orm";+import { db } from "../../../db/connection";+import * as schema from "../../../db/schema";let identity: Identity;let secondaryIdentity: Identity;@@ -28,425 +30,418 @@ beforeAll(async () => {// Skipped in self-hosted mode: depends on Supabase for the `searches` row// lookup, Autumn for credit refunds, and the per-team daily refund cap —// none of which exist in self-hosted setups.-describeIf(TEST_PRODUCTION)(- "Search feedback tests",- () => {- it.concurrent(- "records feedback and refunds 1 credit on first submission",- async () => {- const raw = await searchRawFull(- { query: "firecrawl", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);- expect(typeof raw.body.id).toBe("string");- expect((raw.body.data?.web ?? []).length).toBeGreaterThan(0);-- const result = await searchFeedback(- raw.body.id,- {- rating: "good",- valuableSources: [- {- url: raw.body.data.web[0].url,- reason: "Most directly answered the question.",- },- ],- missingContent: [- {- topic: "Enterprise pricing",- description:- "Pricing tier table for the Enterprise plan was not in any result.",- },- {- topic: "SLA terms",- description: "Uptime SLA and support SLAs not surfaced.",- },- ],- querySuggestions:- "Include site:firecrawl.dev when the user mentions firecrawl by name.",- },- identity,- );-- expect(result.success).toBe(true);- expect(result.creditsRefunded).toBe(1);- expect(result.alreadySubmitted).toBeFalsy();- expect(typeof result.feedbackId).toBe("string");- },- 90000,- );-- it.concurrent(- "is idempotent — second submission returns 0 refund",- async () => {- const raw = await searchRawFull(- { query: "firecrawl docs", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);- expect(typeof raw.body.id).toBe("string");-- const first = await searchFeedback(- raw.body.id,- {- rating: "partial",- missingContent: [- { topic: "Recent results", description: "Nothing past 2024." },- ],- },- identity,- );- expect(first.creditsRefunded).toBe(1);-- const second = await searchFeedback(- raw.body.id,- {- rating: "bad",- missingContent: [- { topic: "Recent results", description: "Still nothing past 2024." },- ],- },- identity,- );- expect(second.creditsRefunded).toBe(0);- expect(second.alreadySubmitted).toBe(true);- },- 90000,- );-- it.concurrent(- "rejects feedback for a search owned by another team",- async () => {- const raw = await searchRawFull(- { query: "firecrawl api", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);- const searchId = raw.body.id;-- const failed = await searchFeedbackWithFailure(- searchId,- {- rating: "good",- valuableSources: [{ url: "https://firecrawl.dev/" }],- },- secondaryIdentity,- );- expect(failed.error.toLowerCase()).toContain("not found");- expect((failed as any).feedbackErrorCode).toBe("SEARCH_NOT_FOUND");- },- 90000,- );-- it.concurrent(- "rejects feedback for a non-existent search id",- async () => {- const failed = await searchFeedbackWithFailure(- "00000000-0000-7000-8000-000000000000",- {- rating: "bad",- missingContent: [{ topic: "Anything at all" }],- },- identity,- );- expect(failed.error.toLowerCase()).toContain("not found");- expect((failed as any).feedbackErrorCode).toBe("SEARCH_NOT_FOUND");- },- 30000,- );-- it.concurrent(- "rejects an invalid jobId format with 400",- async () => {- const raw = await searchFeedbackRaw(- "not-a-uuid",- {- rating: "good",- valuableSources: [{ url: "https://firecrawl.dev/" }],- },- identity,- );- expect(raw.statusCode).toBe(400);- expect(raw.body.success).toBe(false);- },- 30000,- );-- it.concurrent(- "rejects an invalid rating value",- async () => {- const raw = await searchRawFull(- { query: "firecrawl", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);-- const failed = await searchFeedbackRaw(- raw.body.id,- { rating: "amazing" as any },- identity,- );- expect(failed.statusCode).toBe(400);- expect(failed.body.success).toBe(false);- },- 90000,- );-- it.concurrent(- "rejects feedback with a non-http URL in valuableSources",- async () => {- const raw = await searchRawFull(- { query: "firecrawl", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);-- const failed = await searchFeedbackRaw(- raw.body.id,- {- rating: "good",- valuableSources: [- { url: "ftp://example.com/file", reason: "valuable" },- ],- },- identity,- );- expect(failed.statusCode).toBe(400);- expect(failed.body.success).toBe(false);- },- 90000,- );-- it.concurrent(- "rejects 'good' rating without any valuableSources",- async () => {- const raw = await searchRawFull(- { query: "firecrawl", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);-- const failed = await searchFeedbackRaw(- raw.body.id,- {- rating: "good",- missingContent: [- { topic: "Irrelevant for good rating" },- ],- },- identity,- );- expect(failed.statusCode).toBe(400);- expect(failed.body.success).toBe(false);- expect(String(failed.body.error).toLowerCase()).toContain(- "substantive",- );- },- 90000,- );-- it.concurrent(- "rejects 'partial' rating with no sources and no missing content",- async () => {- const raw = await searchRawFull(- { query: "firecrawl", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);-- const failed = await searchFeedbackRaw(- raw.body.id,- { rating: "partial" },- identity,- );- expect(failed.statusCode).toBe(400);- expect(failed.body.success).toBe(false);- },- 90000,- );-- it.concurrent(- "rejects 'bad' rating with no missing content or query suggestions",- async () => {- const raw = await searchRawFull(- { query: "firecrawl", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);-- const failed = await searchFeedbackRaw(- raw.body.id,- {- rating: "bad",- valuableSources: [{ url: "https://firecrawl.dev/" }],- },- identity,- );- expect(failed.statusCode).toBe(400);- expect(failed.body.success).toBe(false);- },- 90000,- );-- it.concurrent(- "rejects missingContent entries without a topic",- async () => {- const raw = await searchRawFull(- { query: "firecrawl", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);-- const failed = await searchFeedbackRaw(- raw.body.id,- {- rating: "bad",- // @ts-expect-error testing invalid shape- missingContent: [{ description: "no topic supplied" }],- },- identity,- );- expect(failed.statusCode).toBe(400);- expect(failed.body.success).toBe(false);- },- 90000,- );-- it.concurrent(- "accepts a structured 'partial' rating with multiple missing topics",- async () => {- const raw = await searchRawFull(- { query: "firecrawl pricing", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);-- const result = await searchFeedback(- raw.body.id,- {- rating: "partial",- missingContent: [- {- topic: "Enterprise pricing",- description:- "Pricing tier table for Enterprise was not surfaced.",- },- { topic: "Self-hosted pricing" },- {- topic: "Annual discount",- description: "Annual vs monthly discount comparison.",- },- ],- },- identity,+describeIf(TEST_PRODUCTION)("Search feedback tests", () => {+ it.concurrent(+ "records feedback and refunds 1 credit on first submission",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ expect(typeof raw.body.id).toBe("string");+ expect((raw.body.data?.web ?? []).length).toBeGreaterThan(0);++ const result = await searchFeedback(+ raw.body.id,+ {+ rating: "good",+ valuableSources: [+ {+ url: raw.body.data.web[0].url,+ reason: "Most directly answered the question.",+ },+ ],+ missingContent: [+ {+ topic: "Enterprise pricing",+ description:+ "Pricing tier table for the Enterprise plan was not in any result.",+ },+ {+ topic: "SLA terms",+ description: "Uptime SLA and support SLAs not surfaced.",+ },+ ],+ querySuggestions:+ "Include site:firecrawl.dev when the user mentions firecrawl by name.",+ },+ identity,+ );++ expect(result.success).toBe(true);+ expect(result.creditsRefunded).toBe(1);+ expect(result.alreadySubmitted).toBeFalsy();+ expect(typeof result.feedbackId).toBe("string");+ },+ 90000,+ );++ it.concurrent(+ "is idempotent — second submission returns 0 refund",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl docs", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ expect(typeof raw.body.id).toBe("string");++ const first = await searchFeedback(+ raw.body.id,+ {+ rating: "partial",+ missingContent: [+ { topic: "Recent results", description: "Nothing past 2024." },+ ],+ },+ identity,+ );+ expect(first.creditsRefunded).toBe(1);++ const second = await searchFeedback(+ raw.body.id,+ {+ rating: "bad",+ missingContent: [+ {+ topic: "Recent results",+ description: "Still nothing past 2024.",+ },+ ],+ },+ identity,+ );+ expect(second.creditsRefunded).toBe(0);+ expect(second.alreadySubmitted).toBe(true);+ },+ 90000,+ );++ it.concurrent(+ "rejects feedback for a search owned by another team",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl api", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ const searchId = raw.body.id;++ const failed = await searchFeedbackWithFailure(+ searchId,+ {+ rating: "good",+ valuableSources: [{ url: "https://firecrawl.dev/" }],+ },+ secondaryIdentity,+ );+ expect(failed.error.toLowerCase()).toContain("not found");+ expect((failed as any).feedbackErrorCode).toBe("SEARCH_NOT_FOUND");+ },+ 90000,+ );++ it.concurrent(+ "rejects feedback for a non-existent search id",+ async () => {+ const failed = await searchFeedbackWithFailure(+ "00000000-0000-7000-8000-000000000000",+ {+ rating: "bad",+ missingContent: [{ topic: "Anything at all" }],+ },+ identity,+ );+ expect(failed.error.toLowerCase()).toContain("not found");+ expect((failed as any).feedbackErrorCode).toBe("SEARCH_NOT_FOUND");+ },+ 30000,+ );++ it.concurrent(+ "rejects an invalid jobId format with 400",+ async () => {+ const raw = await searchFeedbackRaw(+ "not-a-uuid",+ {+ rating: "good",+ valuableSources: [{ url: "https://firecrawl.dev/" }],+ },+ identity,+ );+ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ },+ 30000,+ );++ it.concurrent(+ "rejects an invalid rating value",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);++ const failed = await searchFeedbackRaw(+ raw.body.id,+ { rating: "amazing" as any },+ identity,+ );+ expect(failed.statusCode).toBe(400);+ expect(failed.body.success).toBe(false);+ },+ 90000,+ );++ it.concurrent(+ "rejects feedback with a non-http URL in valuableSources",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);++ const failed = await searchFeedbackRaw(+ raw.body.id,+ {+ rating: "good",+ valuableSources: [+ { url: "ftp://example.com/file", reason: "valuable" },+ ],+ },+ identity,+ );+ expect(failed.statusCode).toBe(400);+ expect(failed.body.success).toBe(false);+ },+ 90000,+ );++ it.concurrent(+ "rejects 'good' rating without any valuableSources",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);++ const failed = await searchFeedbackRaw(+ raw.body.id,+ {+ rating: "good",+ missingContent: [{ topic: "Irrelevant for good rating" }],+ },+ identity,+ );+ expect(failed.statusCode).toBe(400);+ expect(failed.body.success).toBe(false);+ expect(String(failed.body.error).toLowerCase()).toContain("substantive");+ },+ 90000,+ );++ it.concurrent(+ "rejects 'partial' rating with no sources and no missing content",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);++ const failed = await searchFeedbackRaw(+ raw.body.id,+ { rating: "partial" },+ identity,+ );+ expect(failed.statusCode).toBe(400);+ expect(failed.body.success).toBe(false);+ },+ 90000,+ );++ it.concurrent(+ "rejects 'bad' rating with no missing content or query suggestions",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);++ const failed = await searchFeedbackRaw(+ raw.body.id,+ {+ rating: "bad",+ valuableSources: [{ url: "https://firecrawl.dev/" }],+ },+ identity,+ );+ expect(failed.statusCode).toBe(400);+ expect(failed.body.success).toBe(false);+ },+ 90000,+ );++ it.concurrent(+ "rejects missingContent entries without a topic",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);++ const failed = await searchFeedbackRaw(+ raw.body.id,+ {+ rating: "bad",+ // @ts-expect-error testing invalid shape+ missingContent: [{ description: "no topic supplied" }],+ },+ identity,+ );+ expect(failed.statusCode).toBe(400);+ expect(failed.body.success).toBe(false);+ },+ 90000,+ );++ it.concurrent(+ "accepts a structured 'partial' rating with multiple missing topics",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl pricing", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);++ const result = await searchFeedback(+ raw.body.id,+ {+ rating: "partial",+ missingContent: [+ {+ topic: "Enterprise pricing",+ description:+ "Pricing tier table for Enterprise was not surfaced.",+ },+ { topic: "Self-hosted pricing" },+ {+ topic: "Annual discount",+ description: "Annual vs monthly discount comparison.",+ },+ ],+ },+ identity,+ );++ expect(result.success).toBe(true);+ expect(result.creditsRefunded).toBe(1);+ },+ 90000,+ );++ // TEAM_OPTED_OUT (server-side opt-out via searchFeedbackOptOut flag)+ // is not covered by E2E because idmux identities can't set ACUC flags.++ it.concurrent(+ "stops refunding once the team's daily refund cap is reached",+ async () => {+ const cappedIdentity = await idmux({+ name: "search-feedback-cap",+ concurrency: 100,+ credits: 1000000,+ });++ const dailyCap = 100;++ const seedSearchId =+ "00000000-0000-7000-8000-" ++ Math.floor(Math.random() * 1e12)+ .toString(16)+ .padStart(12, "0");+ await db.insert(schema.search_feedback).values({+ search_id: seedSearchId,+ team_id: cappedIdentity.teamId,+ overall_rating: "good",+ valuable_sources: [{ url: "https://firecrawl.dev/" }],+ missing_content: [],+ integration: null,+ origin: "test-seed",+ credits_refunded: dailyCap,+ });++ // Now do a real search and submit feedback.+ const raw = await searchRawFull(+ { query: "firecrawl daily cap", limit: 3 },+ cappedIdentity,+ );+ expect(raw.statusCode).toBe(200);++ const result = await searchFeedback(+ raw.body.id,+ {+ rating: "good",+ valuableSources: [{ url: raw.body.data.web[0].url }],+ },+ cappedIdentity,+ );++ expect(result.success).toBe(true);+ expect(result.creditsRefunded).toBe(0);+ expect(result.dailyCapReached).toBe(true);+ expect(result.creditsRefundedToday).toBeGreaterThanOrEqual(dailyCap);+ expect(result.dailyRefundCap).toBe(dailyCap);+ expect(String(result.warning ?? "").toLowerCase()).toContain(+ "daily refund cap",+ );++ await db+ .delete(schema.search_feedback)+ .where(eq(schema.search_feedback.search_id, seedSearchId));+ },+ 120000,+ );++ // Back-date the searches row so we don't have to wait the full window.+ it.concurrent(+ "rejects feedback submitted outside the configured time window",+ async () => {+ const raw = await searchRawFull(+ { query: "firecrawl windowed", limit: 3 },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ const searchId = raw.body.id;++ await new Promise(r => setTimeout(r, 750));+ const aged = new Date(Date.now() - 60 * 60 * 1000).toISOString();+ await db+ .update(schema.searches)+ .set({ created_at: aged })+ .where(+ and(+ eq(schema.searches.id, searchId),+ eq(schema.searches.team_id, identity.teamId),+ ),);- expect(result.success).toBe(true);- expect(result.creditsRefunded).toBe(1);- },- 90000,- );-- // TEAM_OPTED_OUT (server-side opt-out via searchFeedbackOptOut flag)- // is not covered by E2E because idmux identities can't set ACUC flags.-- it.concurrent(- "stops refunding once the team's daily refund cap is reached",- async () => {- const cappedIdentity = await idmux({- name: "search-feedback-cap",- concurrency: 100,- credits: 1000000,- });-- const dailyCap = 100;-- const seedSearchId =- "00000000-0000-7000-8000-" +- Math.floor(Math.random() * 1e12)- .toString(16)- .padStart(12, "0");- const { error: seedErr } = await supabase_service- .from("search_feedback")- .insert({- search_id: seedSearchId,- team_id: cappedIdentity.teamId,- overall_rating: "good",- valuable_sources: [{ url: "https://firecrawl.dev/" }],- missing_content: [],- integration: null,- origin: "test-seed",- credits_refunded: dailyCap,- });- expect(seedErr).toBeFalsy();-- // Now do a real search and submit feedback.- const raw = await searchRawFull(- { query: "firecrawl daily cap", limit: 3 },- cappedIdentity,- );- expect(raw.statusCode).toBe(200);-- const result = await searchFeedback(- raw.body.id,- {- rating: "good",- valuableSources: [{ url: raw.body.data.web[0].url }],- },- cappedIdentity,- );-- expect(result.success).toBe(true);- expect(result.creditsRefunded).toBe(0);- expect(result.dailyCapReached).toBe(true);- expect(result.creditsRefundedToday).toBeGreaterThanOrEqual(dailyCap);- expect(result.dailyRefundCap).toBe(dailyCap);- expect(String(result.warning ?? "").toLowerCase()).toContain(- "daily refund cap",- );-- await supabase_service- .from("search_feedback")- .delete()- .eq("search_id", seedSearchId);- },- 120000,- );-- // Back-date the searches row so we don't have to wait the full window.- it.concurrent(- "rejects feedback submitted outside the configured time window",- async () => {- const raw = await searchRawFull(- { query: "firecrawl windowed", limit: 3 },- identity,- );- expect(raw.statusCode).toBe(200);- const searchId = raw.body.id;-- await new Promise(r => setTimeout(r, 750));- const aged = new Date(Date.now() - 60 * 60 * 1000).toISOString();- const { error: updateErr } = await supabase_service- .from("searches")- .update({ created_at: aged })- .eq("id", searchId)- .eq("team_id", identity.teamId);- expect(updateErr).toBeFalsy();-- const failed = await searchFeedbackWithFailure(- searchId,- {- rating: "good",- valuableSources: [{ url: "https://firecrawl.dev/" }],- },- identity,- );- expect((failed as any).feedbackErrorCode).toBe(- "FEEDBACK_WINDOW_EXPIRED",- );- },- 90000,- );- },-);+ const failed = await searchFeedbackWithFailure(+ searchId,+ {+ rating: "good",+ valuableSources: [{ url: "https://firecrawl.dev/" }],+ },+ identity,+ );+ expect((failed as any).feedbackErrorCode).toBe("FEEDBACK_WINDOW_EXPIRED");+ },+ 90000,+ );+});diff --git a/apps/api/src/__tests__/snips/zdr-helpers.ts b/apps/api/src/__tests__/snips/zdr-helpers.tsindex 654c1d924..35da88409 100644--- a/apps/api/src/__tests__/snips/zdr-helpers.ts+++ b/apps/api/src/__tests__/snips/zdr-helpers.ts@@ -1,5 +1,7 @@import { readFile, stat } from "node:fs/promises";-import { supabase_service } from "../../services/supabase";+import { eq } from "drizzle-orm";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";import { getJobFromGCS } from "../../lib/gcs-jobs";type Identity = { apiKey: string; teamId: string };@@ -86,16 +88,15 @@ export async function getLogs() {}export async function expectScrapeIsCleanedUp(scrapeId: string) {- const { data: scrapeData, error: scrapeError } = await supabase_service- .from("scrapes")- .select("*")- .eq("id", scrapeId)+ const scrapeData = await db+ .select()+ .from(schema.scrapes)+ .where(eq(schema.scrapes.id, scrapeId)).limit(1);- expect(scrapeError).toBeFalsy();expect(scrapeData).toHaveLength(1);- if (scrapeData && scrapeData.length === 1) {+ if (scrapeData.length === 1) {const record = scrapeData[0];expect(record.url).not.toContain("://"); // no url storedexpect(record.options).toBeNull();@@ -103,32 +104,29 @@ export async function expectScrapeIsCleanedUp(scrapeId: string) {}export async function expectCrawlIsCleanedUp(crawlId: string) {- const { data: requestData, error: requestError } = await supabase_service- .from("requests")- .select("*")- .eq("id", crawlId)+ const requestData = await db+ .select()+ .from(schema.requests)+ .where(eq(schema.requests.id, crawlId)).limit(1);- expect(requestError).toBeFalsy();expect(requestData).toHaveLength(1);-- if (requestData && requestData.length === 1) {++ if (requestData.length === 1) {const record = requestData[0];- expect(record.url).not.toContain("://"); // no url storedexpect(record.kind).toBe("crawl");expect(record.dr_clean_by).not.toBeNull();}- const { data: crawlData, error: crawlError } = await supabase_service- .from("crawls")- .select("*")- .eq("id", crawlId)+ const crawlData = await db+ .select()+ .from(schema.crawls)+ .where(eq(schema.crawls.id, crawlId)).limit(1);- expect(crawlError).toBeFalsy();expect(crawlData).toHaveLength(1);- if (crawlData && crawlData.length === 1) {+ if (crawlData.length === 1) {const record = crawlData[0];expect(record.url).not.toContain("://"); // no url storedexpect(record.options).toBeNull();@@ -136,65 +134,55 @@ export async function expectCrawlIsCleanedUp(crawlId: string) {}export async function expectBatchScrapeIsCleanedUp(batchScrapeId: string) {- const { data: requestData, error: requestError } = await supabase_service- .from("requests")- .select("*")- .eq("id", batchScrapeId)+ const requestData = await db+ .select()+ .from(schema.requests)+ .where(eq(schema.requests.id, batchScrapeId)).limit(1);- expect(requestError).toBeFalsy();expect(requestData).toHaveLength(1);-- if (requestData && requestData.length === 1) {++ if (requestData.length === 1) {const record = requestData[0];- expect(record.url).not.toContain("://"); // no url storedexpect(record.kind).toBe("batch_scrape");expect(record.dr_clean_by).not.toBeNull();}- const { data: batchScrapeData, error: batchScrapeError } = await supabase_service- .from("batch_scrapes")- .select("*")- .eq("id", batchScrapeId)+ const batchScrapeData = await db+ .select()+ .from(schema.batch_scrapes)+ .where(eq(schema.batch_scrapes.id, batchScrapeId)).limit(1);- expect(batchScrapeError).toBeFalsy();expect(batchScrapeData).toHaveLength(1);-- if (batchScrapeData && batchScrapeData.length === 1) {- const record = batchScrapeData[0];- expect(record.url).not.toContain("://"); // no url stored- }}export async function expectScrapesOfRequestAreCleanedUp(requestId: string,expectedScrapeCount?: number,) {- const { data: scrapes, error: scrapesError } = await supabase_service- .from("scrapes")- .select("*")- .eq("request_id", requestId);-- expect(scrapesError).toBeFalsy();+ const scrapes = await db+ .select()+ .from(schema.scrapes)+ .where(eq(schema.scrapes.request_id, requestId));if (expectedScrapeCount !== undefined) {- expect((scrapes ?? []).length).toBe(expectedScrapeCount);+ expect(scrapes.length).toBe(expectedScrapeCount);} else {- expect((scrapes ?? []).length).toBeGreaterThanOrEqual(1);+ expect(scrapes.length).toBeGreaterThanOrEqual(1);}- for (const scrape of scrapes ?? []) {+ for (const scrape of scrapes) {expect(scrape.url).not.toContain("://"); // no url storedexpect(scrape.options).toBeNull();- if (scrape.success) {+ if (scrape.is_successful) {const gcsJob = await getJobFromGCS(scrape.id);expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry}}- return scrapes ?? [];+ return scrapes;}export async function expectScrapesAreFullyCleanedAfterZDRCleaner(diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex e6209ea62..850973be0 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -81,6 +81,9 @@ const configSchema = z.object({POSTGRES_DB: z.string().default("postgres"),POSTGRES_USER: z.string().default("postgres"),POSTGRES_PASSWORD: z.string().default("postgres"),+ DATABASE_URL: z.string().optional(),+ DATABASE_REPLICA_URL: z.string().optional(),+ INDEX_DATABASE_URL: z.string().optional(),REDIS_URL: z.string().optional(),REDIS_EVICT_URL: z.string().optional(),REDIS_RATE_LIMIT_URL: z.string().optional(),@@ -88,15 +91,6 @@ const configSchema = z.object({NUQ_DATABASE_URL_LISTEN: z.string().optional(),NUQ_RABBITMQ_URL: z.string().optional(),- // Supabase- SUPABASE_URL: z.string().optional(),- SUPABASE_ANON_TOKEN: z.string().optional(),- SUPABASE_SERVICE_TOKEN: z.string().optional(),- SUPABASE_REPLICA_URL: z.string().optional(),- INDEX_SUPABASE_URL: z.string().optional(),- INDEX_SUPABASE_SERVICE_TOKEN: z.string().optional(),- SEARCH_INDEX_SUPABASE_URL: z.string().optional(),-// Google Cloud StorageGCS_BUCKET_NAME: z.string().optional(),GCS_CREDENTIALS: z.string().optional(),@@ -268,6 +262,7 @@ const configSchema = z.object({GITHUB_REF_NAME: z.string().optional(),RESTRICTED_COUNTRIES: delimitedList(",").optional(),DISABLE_ENGPICKER: z.stringbool().optional(),+ DISABLE_MONITORING: z.stringbool().default(false),EXTRACT_V3_BETA_URL: z.string().optional(),AGENT_INTEROP_SECRET: z.string().optional(),diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 4444268e5..86d116c29 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -10,7 +10,14 @@ import { getRedisConnection } from "../services/queue-service";import { getRateLimiter } from "../services/rate-limiter";import { deleteKey, getValue, setValue } from "../services/redis";import { redlock } from "../services/redlock";-import { supabase_rr_service, supabase_service } from "../services/supabase";+import { eq } from "drizzle-orm";+import { db, dbRr } from "../db/connection";+import * as schema from "../db/schema";+import {+ authCreditUsageChunk,+ authCreditUsageChunkFromTeam,+ AuthCreditUsageChunkRow,+} from "../db/rpc";import { AuthResponse, RateLimiterMode } from "../types";import { AuthCreditUsageChunk, AuthCreditUsageChunkFromTeam } from "./v1/types";@@ -260,45 +267,38 @@ export async function getACUC(}if (!cacheOnly) {- let data;- let error;+ let data: AuthCreditUsageChunkRow[] = [];let retries = 0;const maxRetries = 5;while (retries < maxRetries) {- const client =- Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;- ({ data, error } = await client.rpc(- "auth_credit_usage_chunk_47",- {- input_key: api_key,- i_is_extract: isExtract,- tally_untallied_credits: true,- },- { get: true },- ));-- if (!error) {+ const database = Math.random() > 2 / 3 ? dbRr : db;+ try {+ data = await authCreditUsageChunk(database, api_key, isExtract);break;- }-- logger.warn(- `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,- { error },- );- retries++;- if (retries === maxRetries) {- throw new Error(- "Failed to retrieve authentication and credit usage data after 3 attempts: " +- JSON.stringify(error),+ } catch (error) {+ logger.warn(+ `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,+ { error },);- }+ retries++;+ if (retries === maxRetries) {+ throw new Error(+ "Failed to retrieve authentication and credit usage data after 3 attempts: " ++ JSON.stringify(error),+ );+ }- // Wait for a short time before retrying- await new Promise(resolve => setTimeout(resolve, 200));+ // Wait for a short time before retrying+ await new Promise(resolve => setTimeout(resolve, 200));+ }}const chunk: AuthCreditUsageChunk | null =- data.length === 0 ? null : data[0].team_id === null ? null : data[0];+ data.length === 0+ ? null+ : data[0].team_id === null+ ? null+ : (data[0] as any);if (chunk) {chunk.is_extract = isExtract;@@ -386,46 +386,39 @@ export async function getACUCTeam(}if (!cacheOnly) {- let data;- let error;+ let data: AuthCreditUsageChunkRow[] = [];let retries = 0;const maxRetries = 5;while (retries < maxRetries) {- const client =- Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;- ({ data, error } = await client.rpc(- "auth_credit_usage_chunk_47_from_team",- {- input_team: team_id,- i_is_extract: isExtract,- tally_untallied_credits: true,- },- { get: true },- ));-- if (!error) {+ const database = Math.random() > 2 / 3 ? dbRr : db;+ try {+ data = await authCreditUsageChunkFromTeam(database, team_id, isExtract);break;- }-- logger.warn(- `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,- { error },- );- retries++;- if (retries === maxRetries) {- throw new Error(- "Failed to retrieve authentication and credit usage data after 3 attempts: " +- JSON.stringify(error),+ } catch (error) {+ logger.warn(+ `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,+ { error },);- }+ retries++;+ if (retries === maxRetries) {+ throw new Error(+ "Failed to retrieve authentication and credit usage data after 3 attempts: " ++ JSON.stringify(error),+ );+ }- // Wait for a short time before retrying- await new Promise(resolve => setTimeout(resolve, 200));+ // Wait for a short time before retrying+ await new Promise(resolve => setTimeout(resolve, 200));+ }}const chunk: AuthCreditUsageChunk | null =- data.length === 0 ? null : data[0].team_id === null ? null : data[0];+ data.length === 0+ ? null+ : data[0].team_id === null+ ? null+ : (data[0] as any);// NOTE: Should we cache null chunks? - mogeryif (chunk !== null && useCache) {@@ -493,13 +486,14 @@ async function ensureChunkOrgId(return chunk;}- const { data, error } = await supabase_rr_service- .from("teams")- .select("org_id")- .eq("id", chunk.team_id)- .single();-- if (error || !data?.org_id) {+ let data: { org_id: string | null } | undefined;+ try {+ [data] = await dbRr+ .select({ org_id: schema.teams.org_id })+ .from(schema.teams)+ .where(eq(schema.teams.id, chunk.team_id))+ .limit(1);+ } catch (error) {logger.warn("Failed to backfill org_id for auth chunk", {teamId: chunk.team_id,error,@@ -507,6 +501,13 @@ async function ensureChunkOrgId(return chunk;}+ if (!data?.org_id) {+ logger.warn("Failed to backfill org_id for auth chunk", {+ teamId: chunk.team_id,+ });+ return chunk;+ }+chunk.org_id = data.org_id;await setCachedACUC(apiKey, !!chunk.is_extract, chunk);return chunk;diff --git a/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts b/apps/api/src/controllers/v0/admin/acuc-cache-clear.tsindex bbe0d43e4..5e69f1507 100644--- a/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts+++ b/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts@@ -1,5 +1,7 @@import { Request, Response } from "express";-import { supabase_service } from "../../../services/supabase";+import { eq } from "drizzle-orm";+import { db } from "../../../db/connection";+import * as schema from "../../../db/schema";import { clearACUC, clearACUCTeam } from "../../auth";import { logger } from "../../../lib/logger";@@ -11,12 +13,12 @@ export async function acucCacheClearController(req: Request, res: Response) {return res.status(400).json({ error: "team_id is required" });}- const keys = await supabase_service- .from("api_keys")- .select("*")- .eq("team_id", team_id);+ const keys = await db+ .select()+ .from(schema.api_keys)+ .where(eq(schema.api_keys.team_id, team_id));- await Promise.all((keys.data ?? []).map(x => clearACUC(x.key)));+ await Promise.all(keys.map(x => clearACUC(x.key!)));await clearACUCTeam(team_id);logger.info(`ACUC cache cleared for team ${team_id}`);diff --git a/apps/api/src/controllers/v0/admin/cclog.ts b/apps/api/src/controllers/v0/admin/cclog.tsindex 48a2dea4c..d77159a34 100644--- a/apps/api/src/controllers/v0/admin/cclog.ts+++ b/apps/api/src/controllers/v0/admin/cclog.ts@@ -1,5 +1,6 @@import { getRedisConnection } from "../../../services/queue-service";-import { supabase_service } from "../../../services/supabase";+import { db } from "../../../db/connection";+import * as schema from "../../../db/schema";import { logger as _logger } from "../../../lib/logger";import { Request, Response } from "express";@@ -46,15 +47,13 @@ async function cclog() {}try {- const { error } = await supabase_service- .from("concurrency_log")- .insert(entries);- if (error) {- logger.error("Error inserting", {- error,- entryCount: entries.length,- });- }+ await db.insert(schema.concurrency_log).values(+ entries.map(e => ({+ team_id: e.team_id,+ concurrency: e.concurrency,+ created_at: e.created_at.toISOString(),+ })),+ );} catch (e) {logger.error("Error inserting", { error: e });}diff --git a/apps/api/src/controllers/v1/activity.ts b/apps/api/src/controllers/v1/activity.tsindex fc0154e2f..dfd24e5be 100644--- a/apps/api/src/controllers/v1/activity.ts+++ b/apps/api/src/controllers/v1/activity.ts@@ -1,6 +1,8 @@import { Response } from "express";+import { and, desc, eq, gte, lt, or } from "drizzle-orm";import { RequestWithAuth, ErrorResponse } from "./types";-import { supabase_rr_service } from "../../services/supabase";+import { dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { logger as _logger } from "../../lib/logger";const ACTIVITY_WINDOW_HOURS = 24;@@ -98,30 +100,50 @@ export async function activityController(Date.now() - ACTIVITY_WINDOW_HOURS * 60 * 60 * 1000,).toISOString();- let query = supabase_rr_service- .from("requests")- .select("id, kind, api_version, created_at, target_hint")- .eq("team_id", req.auth.team_id)- .gte("created_at", windowStart)- .order("created_at", { ascending: false })- .order("id", { ascending: false })- .limit(limit + 1); // fetch one extra to determine has_more+ const conditions = [+ eq(schema.requests.team_id, req.auth.team_id),+ gte(schema.requests.created_at, windowStart),+ ];if (endpoint) {- query = query.eq("kind", endpoint);+ conditions.push(eq(schema.requests.kind, endpoint));}if (cursor) {// For keyset pagination: fetch rows where (created_at, id) < (cursor.createdAt, cursor.id)// This translates to: created_at < cursor OR (created_at = cursor AND id < cursor.id)- query = query.or(- `created_at.lt.${cursor.createdAt},and(created_at.eq.${cursor.createdAt},id.lt.${cursor.id})`,+ conditions.push(+ or(+ lt(schema.requests.created_at, cursor.createdAt),+ and(+ eq(schema.requests.created_at, cursor.createdAt),+ lt(schema.requests.id, cursor.id),+ ),+ )!,);}- const { data, error } = await query;-- if (error) {+ let data: {+ id: string;+ kind: string | null;+ api_version: string | null;+ created_at: string | null;+ target_hint: string | null;+ }[];+ try {+ data = await dbRr+ .select({+ id: schema.requests.id,+ kind: schema.requests.kind,+ api_version: schema.requests.api_version,+ created_at: schema.requests.created_at,+ target_hint: schema.requests.target_hint,+ })+ .from(schema.requests)+ .where(and(...conditions))+ .orderBy(desc(schema.requests.created_at), desc(schema.requests.id))+ .limit(limit + 1); // fetch one extra to determine has_more+ } catch (error) {logger.error("Failed to fetch activity", { error });return res.status(500).json({success: false,diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex 2067d89f0..d89fe4ce0 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -8,7 +8,9 @@ import {import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis";import { redisEvictConnection } from "../../../src/services/redis";import { configDotenv } from "dotenv";-import { supabase_rr_service } from "../../services/supabase";+import { and, eq } from "drizzle-orm";+import { dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { logger as _logger } from "../../lib/logger";import { deserializeTransportableError } from "../../lib/error-serde";import { TransportableError } from "../../lib/error";@@ -75,14 +77,14 @@ export async function crawlErrorsController(});} else if (config.USE_DB_AUTHENTICATION) {// Check the requests table for the crawl/batch scrape request- const { data: request, error: requestError } = await supabase_rr_service- .from("requests")- .select("*")- .eq("id", req.params.jobId)- .limit(1)- .throwOnError();-- if (requestError) {+ let request: (typeof schema.requests.$inferSelect)[];+ try {+ request = await dbRr+ .select()+ .from(schema.requests)+ .where(eq(schema.requests.id, req.params.jobId))+ .limit(1);+ } catch (requestError) {_logger.error("Error getting request", { error: requestError });throw requestError;}@@ -98,7 +100,7 @@ export async function crawlErrorsController(if (requestData &&- new Date().valueOf() - new Date(requestData.created_at).valueOf() >+ new Date().valueOf() - new Date(requestData.created_at!).valueOf() >crawlTtlMs) {return res.status(404).json({ success: false, error: "Job expired" });@@ -109,16 +111,19 @@ export async function crawlErrorsController(}// Get failed scrapes from the scrapes table- const { data: failedScrapes, error: failedScrapesError } =- await supabase_rr_service- .from("scrapes")- .select("*")- .eq("request_id", req.params.jobId)- .eq("team_id", req.auth.team_id)- .eq("is_successful", false)- .throwOnError();-- if (failedScrapesError) {+ let failedScrapes: (typeof schema.scrapes.$inferSelect)[];+ try {+ failedScrapes = await dbRr+ .select()+ .from(schema.scrapes)+ .where(+ and(+ eq(schema.scrapes.request_id, req.params.jobId),+ eq(schema.scrapes.team_id, req.auth.team_id),+ eq(schema.scrapes.is_successful, false),+ ),+ );+ } catch (failedScrapesError) {_logger.error("Error getting failed scrapes", {error: failedScrapesError,});@@ -134,10 +139,9 @@ export async function crawlErrorsController(: null;return {id: scrape.id,- timestamp:- scrape.created_at !== undefined- ? new Date(scrape.created_at).toISOString()- : undefined,+ timestamp: scrape.created_at+ ? new Date(scrape.created_at).toISOString()+ : undefined,url: scrape.url,...(error? {diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 619f67e26..9bb4cbe58 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -21,7 +21,7 @@ import {} from "../../lib/supabase-jobs";import { configDotenv } from "dotenv";import { logger } from "../../lib/logger";-import { supabase_rr_service, supabase_service } from "../../services/supabase";+import { creditsBilledByCrawlId } from "../../db/rpc";import { getJobFromGCS } from "../../lib/gcs-jobs";import {scrapeQueue,@@ -196,14 +196,8 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);- const creditsRpc = config.USE_DB_AUTHENTICATION- ? await supabase_service.rpc(- "credits_billed_by_crawl_id_2",- {- i_crawl_id: req.params.jobId,- },- { get: true },- )+ const creditsBilled = config.USE_DB_AUTHENTICATION+ ? await creditsBilledByCrawlId(req.params.jobId).catch(() => null): null;// check if the crawl failed during kickoff (e.g. queue full)@@ -226,7 +220,7 @@ export async function crawlStatusController((numericStats.active ?? 0) +(numericStats.queued ?? 0) +(numericStats.backlog ?? 0),- creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,+ creditsUsed: creditsBilled?.[0]?.credits_billed ?? -1,};// if the crawl has a stored error and no jobs were ever created, mark as faileddiff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex bc5b685c8..4b07896d9 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -68,7 +68,7 @@ export async function scrapeController(account: req.account,});- logRequest({+ const logRequestPromise = logRequest({id: jobId,kind: "scrape",api_version: "v1",@@ -171,6 +171,7 @@ export async function scrapeController(zeroDataRetention: zeroDataRetention ?? false,apiKeyId: req.acuc?.api_key_id ?? null,concurrencyLimited: limited,+ logRequestPromise: logRequestPromise,},};diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex fe044a6e1..606f2254f 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -18,7 +18,9 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";import { CostTracking } from "../../lib/cost-tracking";-import { supabase_service } from "../../services/supabase";+import { eq } from "drizzle-orm";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";import { fromV1ScrapeOptions } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";@@ -132,17 +134,14 @@ async function scrapeX402SearchResult(let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;if (config.USE_DB_AUTHENTICATION) {- const { data: costTrackingResponse, error: costTrackingError } =- await supabase_service- .from("scrapes")- .select("cost_tracking")- .eq("id", jobId);-- if (costTrackingError) {- throw costTrackingError;- }-- costTracking = costTrackingResponse?.[0]?.cost_tracking;+ const costTrackingResponse = await db+ .select({ cost_tracking: schema.scrapes.cost_tracking })+ .from(schema.scrapes)+ .where(eq(schema.scrapes.id, jobId));++ costTracking = costTrackingResponse?.[0]?.cost_tracking as ReturnType<+ typeof CostTracking.prototype.toJSON+ >;} else {costTracking = new CostTracking().toJSON();}diff --git a/apps/api/src/controllers/v2/__tests__/browser-billing.test.ts b/apps/api/src/controllers/v2/__tests__/browser-billing.test.tsindex db2b07dfb..fc01717da 100644--- a/apps/api/src/controllers/v2/__tests__/browser-billing.test.ts+++ b/apps/api/src/controllers/v2/__tests__/browser-billing.test.ts@@ -16,11 +16,6 @@ jest.mock("../../../services/redis", () => ({deleteKey: (key: string) => mockDeleteKey(key),}));-jest.mock("../../../services/supabase", () => ({- supabase_service: {},- isPostgrestNoRowsError: jest.fn(() => true),-}));-jest.mock("../../../lib/logger", () => ({logger: {info: jest.fn(),diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex 34136a277..e478f4128 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -9,7 +9,7 @@ import {import { logger as _logger } from "../../lib/logger";import { logRequest } from "../../services/logging/log_job";import { config } from "../../config";-import { supabase_service } from "../../services/supabase";+import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";import { getScrapeZDR } from "../../lib/zdr-helpers";export async function agentController(@@ -57,18 +57,7 @@ export async function agentController(let freeRequest: any;if (config.USE_DB_AUTHENTICATION && !highCreditRequest) {- const { data, error: freeRequestError } = await supabase_service.rpc(- "agent_consume_free_request_if_left",- {- i_team_id: req.auth.team_id,- },- );-- if (freeRequestError) {- throw freeRequestError;- }-- freeRequest = data;+ freeRequest = await agentConsumeFreeRequestIfLeft(req.auth.team_id);}const isFreeRequest = highCreditRequestdiff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex 2067d89f0..d89fe4ce0 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -8,7 +8,9 @@ import {import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis";import { redisEvictConnection } from "../../../src/services/redis";import { configDotenv } from "dotenv";-import { supabase_rr_service } from "../../services/supabase";+import { and, eq } from "drizzle-orm";+import { dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { logger as _logger } from "../../lib/logger";import { deserializeTransportableError } from "../../lib/error-serde";import { TransportableError } from "../../lib/error";@@ -75,14 +77,14 @@ export async function crawlErrorsController(});} else if (config.USE_DB_AUTHENTICATION) {// Check the requests table for the crawl/batch scrape request- const { data: request, error: requestError } = await supabase_rr_service- .from("requests")- .select("*")- .eq("id", req.params.jobId)- .limit(1)- .throwOnError();-- if (requestError) {+ let request: (typeof schema.requests.$inferSelect)[];+ try {+ request = await dbRr+ .select()+ .from(schema.requests)+ .where(eq(schema.requests.id, req.params.jobId))+ .limit(1);+ } catch (requestError) {_logger.error("Error getting request", { error: requestError });throw requestError;}@@ -98,7 +100,7 @@ export async function crawlErrorsController(if (requestData &&- new Date().valueOf() - new Date(requestData.created_at).valueOf() >+ new Date().valueOf() - new Date(requestData.created_at!).valueOf() >crawlTtlMs) {return res.status(404).json({ success: false, error: "Job expired" });@@ -109,16 +111,19 @@ export async function crawlErrorsController(}// Get failed scrapes from the scrapes table- const { data: failedScrapes, error: failedScrapesError } =- await supabase_rr_service- .from("scrapes")- .select("*")- .eq("request_id", req.params.jobId)- .eq("team_id", req.auth.team_id)- .eq("is_successful", false)- .throwOnError();-- if (failedScrapesError) {+ let failedScrapes: (typeof schema.scrapes.$inferSelect)[];+ try {+ failedScrapes = await dbRr+ .select()+ .from(schema.scrapes)+ .where(+ and(+ eq(schema.scrapes.request_id, req.params.jobId),+ eq(schema.scrapes.team_id, req.auth.team_id),+ eq(schema.scrapes.is_successful, false),+ ),+ );+ } catch (failedScrapesError) {_logger.error("Error getting failed scrapes", {error: failedScrapesError,});@@ -134,10 +139,9 @@ export async function crawlErrorsController(: null;return {id: scrape.id,- timestamp:- scrape.created_at !== undefined- ? new Date(scrape.created_at).toISOString()- : undefined,+ timestamp: scrape.created_at+ ? new Date(scrape.created_at).toISOString()+ : undefined,url: scrape.url,...(error? {diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 98ec8c32e..784804e78 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -21,7 +21,7 @@ import {} from "../../lib/supabase-jobs";import { configDotenv } from "dotenv";import { logger } from "../../lib/logger";-import { supabase_rr_service, supabase_service } from "../../services/supabase";+import { creditsBilledByCrawlId } from "../../db/rpc";import { getJobFromGCS } from "../../lib/gcs-jobs";import {scrapeQueue,@@ -216,14 +216,8 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);- const creditsRpc = config.USE_DB_AUTHENTICATION- ? await supabase_service.rpc(- "credits_billed_by_crawl_id_2",- {- i_crawl_id: req.params.jobId,- },- { get: true },- )+ const creditsBilled = config.USE_DB_AUTHENTICATION+ ? await creditsBilledByCrawlId(req.params.jobId).catch(() => null): null;// check if the crawl failed during kickoff (e.g. queue full)@@ -246,7 +240,7 @@ export async function crawlStatusController((numericStats.active ?? 0) +(numericStats.queued ?? 0) +(numericStats.backlog ?? 0),- creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,+ creditsUsed: creditsBilled?.[0]?.credits_billed ?? -1,};// if the crawl has a stored error and no jobs were ever created, mark as faileddiff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex a96140144..2b6df4743 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -132,8 +132,10 @@ export async function scrapeController(account: req.account,});+ let logRequestPromise: Promise<any> | undefined = undefined;+if (!agentRequestId) {- logRequest({+ logRequestPromise = logRequest({id: jobId,kind: "scrape",api_version: "v2",@@ -260,6 +262,7 @@ export async function scrapeController(apiKeyId: req.acuc?.api_key_id ?? null,concurrencyLimited: limited,requestId: agentRequestId ?? undefined,+ logRequestPromise: logRequestPromise,},};diff --git a/apps/api/src/controllers/v2/search-feedback.ts b/apps/api/src/controllers/v2/search-feedback.tsindex bde625827..29325fe6d 100644--- a/apps/api/src/controllers/v2/search-feedback.ts+++ b/apps/api/src/controllers/v2/search-feedback.ts@@ -4,11 +4,9 @@ import { z } from "zod";import { config } from "../../config";import { logger as _logger } from "../../lib/logger";import { autumnService } from "../../services/autumn/autumn.service";-import {- isPostgrestNoRowsError,- supabase_service,- supabase_rr_service,-} from "../../services/supabase";+import { and, eq, gte } from "drizzle-orm";+import { db, dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { captureExceptionWithZdrCheck } from "../../services/sentry";import {RequestWithAuth,@@ -66,18 +64,24 @@ async function lookupSearchRow(searchId: string,dbTeamId: string,): Promise<SearchRowForFeedback | null> {- const { data, error } = await supabase_rr_service- .from("searches")- .select("id, team_id, credits_cost, created_at, is_successful")- .eq("id", searchId)- .eq("team_id", dbTeamId)- .single();-- if (error) {- if (isPostgrestNoRowsError(error)) return null;- throw error;- }- return data as SearchRowForFeedback | null;+ const [data] = await dbRr+ .select({+ id: schema.searches.id,+ team_id: schema.searches.team_id,+ credits_cost: schema.searches.credits_cost,+ created_at: schema.searches.created_at,+ is_successful: schema.searches.is_successful,+ })+ .from(schema.searches)+ .where(+ and(+ eq(schema.searches.id, searchId),+ eq(schema.searches.team_id, dbTeamId),+ ),+ )+ .limit(1);++ return (data ?? null) as SearchRowForFeedback | null;}function startOfUtcDay(now: Date = new Date()): Date {@@ -94,13 +98,18 @@ async function sumTeamCreditsRefundedToday(): Promise<number> {const since = startOfUtcDay().toISOString();- const { data, error } = await supabase_rr_service- .from("search_feedback")- .select("credits_refunded")- .eq("team_id", dbTeamId)- .gte("created_at", since);-- if (error) {+ let data: { credits_refunded: number | null }[];+ try {+ data = await dbRr+ .select({ credits_refunded: schema.search_feedback.credits_refunded })+ .from(schema.search_feedback)+ .where(+ and(+ eq(schema.search_feedback.team_id, dbTeamId),+ gte(schema.search_feedback.created_at, since),+ ),+ );+ } catch (error) {logger.warn("Failed to compute today's refund total; allowing refund this call",{ error },@@ -108,11 +117,7 @@ async function sumTeamCreditsRefundedToday(return 0;}- return (data ?? []).reduce(- (sum, row: { credits_refunded: number | null }) =>- sum + (row.credits_refunded ?? 0),- 0,- );+ return data.reduce((sum, row) => sum + (row.credits_refunded ?? 0), 0);}export async function searchFeedbackController(@@ -237,9 +242,8 @@ export async function searchFeedbackController(}const feedbackId = uuidv7();- const { error: insertErr } = await supabase_service- .from("search_feedback")- .insert({+ try {+ await db.insert(schema.search_feedback).values({id: feedbackId,search_id: searchId,team_id: dbTeamId,@@ -251,14 +255,16 @@ export async function searchFeedbackController(origin: parsedBody.origin ?? null,credits_refunded: 0,});-- if (insertErr) {+ } catch (insertErr) {if ((insertErr as { code?: string }).code === POSTGRES_UNIQUE_VIOLATION) {- const { data: existing } = await supabase_rr_service- .from("search_feedback")- .select("id, credits_refunded")- .eq("search_id", searchId)- .single();+ const [existing] = await dbRr+ .select({+ id: schema.search_feedback.id,+ credits_refunded: schema.search_feedback.credits_refunded,+ })+ .from(schema.search_feedback)+ .where(eq(schema.search_feedback.search_id, searchId))+ .limit(1);const dailyCapForResponse = config.SEARCH_FEEDBACK_DAILY_CAP_CREDITS;const refundedTodayForResponse = await sumTeamCreditsRefundedToday(@@ -335,11 +341,12 @@ export async function searchFeedbackController(}if (creditsRefunded > 0) {- const { error: updateErr } = await supabase_service- .from("search_feedback")- .update({ credits_refunded: creditsRefunded })- .eq("id", feedbackId);- if (updateErr) {+ try {+ await db+ .update(schema.search_feedback)+ .set({ credits_refunded: creditsRefunded })+ .where(eq(schema.search_feedback.id, feedbackId));+ } catch (updateErr) {logger.warn("Failed to persist credits_refunded on search_feedback row",{ error: updateErr, feedbackId, creditsRefunded },diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex 46e94532f..1f6114373 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -18,7 +18,9 @@ import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";import { getJobPriority } from "../../lib/job-priority";import { CostTracking } from "../../lib/cost-tracking";-import { supabase_service } from "../../services/supabase";+import { eq } from "drizzle-orm";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";import { SearchV2Response } from "../../lib/entities";import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";@@ -155,20 +157,14 @@ async function scrapeX420SearchResult(let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;if (config.USE_DB_AUTHENTICATION) {- const { data: costTrackingResponse, error: costTrackingError } =- await supabase_service- .from("scrapes")- .select("cost_tracking")- .eq("id", jobId);-- if (costTrackingError) {- logger.error("Error getting cost tracking [x402]", {- error: costTrackingError,- });- throw costTrackingError;- }-- costTracking = costTrackingResponse?.[0]?.cost_tracking;+ const costTrackingResponse = await db+ .select({ cost_tracking: schema.scrapes.cost_tracking })+ .from(schema.scrapes)+ .where(eq(schema.scrapes.id, jobId));++ costTracking = costTrackingResponse?.[0]?.cost_tracking as ReturnType<+ typeof CostTracking.prototype.toJSON+ >;} else {costTracking = new CostTracking().toJSON();}diff --git a/apps/api/src/db/connection.ts b/apps/api/src/db/connection.tsnew file mode 100644index 000000000..9d38bd3c8--- /dev/null+++ b/apps/api/src/db/connection.ts@@ -0,0 +1,94 @@+import { Pool } from "pg";+import { drizzle, NodePgDatabase } from "drizzle-orm/node-postgres";+import { config } from "../config";+import { logger } from "../lib/logger";++type DB = NodePgDatabase;++function makeDb(+ connectionString: string | undefined,+ applicationName: string,+): DB | null {+ if (!connectionString) {+ return null;+ }++ const pool = new Pool({+ connectionString,+ application_name: applicationName,+ min: 2,+ keepAlive: true,+ });+ pool.on("error", err =>+ logger.error("Error in idle Postgres client", {+ err,+ module: "db",+ applicationName,+ }),+ );++ let lastWarn = 0;+ pool.on("acquire", () => {+ const max = pool.options.max ?? 10;+ if (+ pool.waitingCount > 0 &&+ pool.totalCount >= max &&+ Date.now() - lastWarn > 1000+ ) {+ lastWarn = Date.now();+ logger.warn("Postgres pool exhausted, queries are queuing", {+ module: "db",+ applicationName,+ total: pool.totalCount,+ idle: pool.idleCount,+ waiting: pool.waitingCount,+ max,+ });+ }+ });++ return drizzle({ client: pool });+}++const useDbAuthentication = config.USE_DB_AUTHENTICATION;++const mainDb = useDbAuthentication+ ? makeDb(config.DATABASE_URL, "firecrawl-api")+ : null;+const replicaDb = useDbAuthentication+ ? makeDb(+ config.DATABASE_REPLICA_URL ?? config.DATABASE_URL,+ "firecrawl-api-rr",+ )+ : null;+const indexDb = makeDb(config.INDEX_DATABASE_URL, "firecrawl-index");++if (useDbAuthentication && !mainDb) {+ logger.error(+ "DATABASE_URL is not configured. Drizzle client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable",+ );+}++function proxyDb(get: () => DB | null, name: string): DB {+ return new Proxy(+ {},+ {+ get(_target, prop, receiver) {+ const client = get();+ if (client === null) {+ throw new Error(`${name} is not configured.`);+ }+ return Reflect.get(client, prop, receiver);+ },+ },+ ) as DB;+}++/** Main Postgres database (writes + reads). */+export const db: DB = proxyDb(() => mainDb, "Database client");++/** Read replica. Falls back to the main connection string when no replica URL is set. */+export const dbRr: DB = proxyDb(() => replicaDb, "Database replica client");++/** Separate index project database. */+export const dbIndex: DB = proxyDb(() => indexDb, "Index database client");diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.tsnew file mode 100644index 000000000..2b8397446--- /dev/null+++ b/apps/api/src/db/rpc.ts@@ -0,0 +1,275 @@+import { sql, SQL } from "drizzle-orm";+import { NodePgDatabase } from "drizzle-orm/node-postgres";+import { db, dbIndex } from "./connection";++type DB = NodePgDatabase;++async function execRows<T = Record<string, any>>(+ database: DB,+ query: SQL,+): Promise<T[]> {+ const res = await database.execute(query);+ return (res.rows ?? []) as T[];+}++// ============================================================================+// Main database RPCs+// ============================================================================++export type AuthCreditUsageChunkRow = Record<string, any> & {+ team_id: string | null;+ api_key: string;+};++export function authCreditUsageChunk(+ database: DB,+ input_key: string,+ i_is_extract: boolean,+): Promise<AuthCreditUsageChunkRow[]> {+ return execRows(+ database,+ sql`select * from auth_credit_usage_chunk_47(input_key => ${input_key}, i_is_extract => ${i_is_extract}, tally_untallied_credits => ${true})`,+ );+}++export function authCreditUsageChunkFromTeam(+ database: DB,+ input_team: string,+ i_is_extract: boolean,+): Promise<AuthCreditUsageChunkRow[]> {+ return execRows(+ database,+ sql`select * from auth_credit_usage_chunk_47_from_team(input_team => ${input_team}, i_is_extract => ${i_is_extract}, tally_untallied_credits => ${true})`,+ );+}++export function getAgentFreeRequestsLeft(+ i_team_id: string,+): Promise<{ free_requests_left: number }[]> {+ return execRows(+ db,+ sql`select * from get_agent_free_requests_left(i_team_id => ${i_team_id})`,+ );+}++export function agentConsumeFreeRequestIfLeft(+ i_team_id: string,+): Promise<{ consumed: boolean }[]> {+ return execRows(+ db,+ sql`select * from agent_consume_free_request_if_left(i_team_id => ${i_team_id})`,+ );+}++export function billTeam6(params: {+ team_id: string;+ subscription_id: string | null;+ fetch_subscription: boolean;+ credits: number;+ api_key_id: number | null;+ is_extract: boolean;+}): Promise<{ api_key: string }[]> {+ return execRows(+ db,+ sql`select * from bill_team_6(_team_id => ${params.team_id}, sub_id => ${params.subscription_id}, fetch_subscription => ${params.fetch_subscription}, credits => ${params.credits}, i_api_key_id => ${params.api_key_id}, is_extract_param => ${params.is_extract})`,+ );+}++export async function changeTrackingInsertScrape(params: {+ team_id: string;+ url: string;+ job_id: string;+ change_tracking_tag: string | null;+ date_added: string;+}): Promise<void> {+ await db.execute(+ sql`select change_tracking_insert_scrape(p_team_id => ${params.team_id}, p_url => ${params.url}, p_job_id => ${params.job_id}, p_change_tracking_tag => ${params.change_tracking_tag}, p_date_added => ${params.date_added}::timestamptz)`,+ );+}++export function creditsBilledByCrawlId(+ i_crawl_id: string,+): Promise<{ credits_billed: number }[]> {+ return execRows(+ db,+ sql`select * from credits_billed_by_crawl_id_2(i_crawl_id => ${i_crawl_id})`,+ );+}++export function diffGetLastScrape(+ i_team_id: string,+ i_url: string,+ i_tag: string | null,+): Promise<{ o_job_id: string; o_date_added: string }[]> {+ return execRows(+ db,+ sql`select * from diff_get_last_scrape_v7(i_team_id => ${i_team_id}, i_url => ${i_url}, i_tag => ${i_tag})`,+ );+}++export function getZdrCleanupBatch(+ p_limit: number,+): Promise<{ request_id: string; ids: string[] }[]> {+ return execRows(+ db,+ sql`select * from get_zdr_cleanup_batch_2(p_limit => ${p_limit})`,+ );+}++export function monitoringClaimDueMonitors<T = Record<string, any>>(params: {+ workerId: string;+ limit: number;+ leaseSeconds: number;+}): Promise<T[]> {+ return execRows(+ db,+ sql`select * from monitoring_claim_due_monitors(p_worker_id => ${params.workerId}, p_limit => ${params.limit}, p_lease_seconds => ${params.leaseSeconds})`,+ );+}++export async function updateTallyTeam(i_team_id: string): Promise<void> {+ await db.execute(sql`select update_tally_10_team(i_team_id => ${i_team_id})`);+}++// ============================================================================+// Index database RPCs+// ============================================================================++export async function insertOmceJobIfNeeded(+ i_domain_level: number,+ i_domain_hash: Buffer,+): Promise<void> {+ await dbIndex.execute(+ sql`select insert_omce_job_if_needed(i_domain_level => ${i_domain_level}, i_domain_hash => ${i_domain_hash})`,+ );+}++export function queryIndexAtSplitLevel(+ i_level: number,+ i_url_hash: Buffer,+ i_newer_than: string,+): Promise<{ resolved_url: string }[]> {+ return execRows(+ dbIndex,+ sql`select * from query_index_at_split_level(i_level => ${i_level}, i_url_hash => ${i_url_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,+ );+}++export function queryIndexAtDomainSplitLevel(+ i_level: number,+ i_domain_hash: Buffer,+ i_newer_than: string,+): Promise<{ resolved_url: string }[]> {+ return execRows(+ dbIndex,+ sql`select * from query_index_at_domain_split_level(i_level => ${i_level}, i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,+ );+}++export function queryOmceSignatures(+ i_domain_hash: Buffer,+ i_newer_than: string,+): Promise<{ signatures: any[] }[]> {+ return execRows(+ dbIndex,+ sql`select * from query_omce_signatures(i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,+ );+}++export function queryEngpickerVerdict(+ i_domain_hash: Buffer,+): Promise<{ verdict: string }[]> {+ return execRows(+ dbIndex,+ sql`select * from query_engpicker_verdict(i_domain_hash => ${i_domain_hash})`,+ );+}++export function queryIndexAtSplitLevelWithMeta(+ i_level: number,+ i_url_hash: Buffer,+ i_newer_than: string,+): Promise<+ { resolved_url: string; title: string | null; description: string | null }[]+> {+ return execRows(+ dbIndex,+ sql`select * from query_index_at_split_level_with_meta(i_level => ${i_level}, i_url_hash => ${i_url_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,+ );+}++export function queryIndexAtDomainSplitLevelWithMeta(+ i_level: number,+ i_domain_hash: Buffer,+ i_newer_than: string,+): Promise<+ { resolved_url: string; title: string | null; description: string | null }[]+> {+ return execRows(+ dbIndex,+ sql`select * from query_index_at_domain_split_level_with_meta(i_level => ${i_level}, i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,+ );+}++export function queryDomainPriority(+ p_min_total: number,+ p_min_priority: number,+ p_lim: number,+ p_time: string,+): Promise<{ domain_hash: Buffer; priority: number }[]> {+ return execRows(+ dbIndex,+ sql`select * from query_domain_priority(p_min_total => ${p_min_total}, p_min_priority => ${p_min_priority}, p_lim => ${p_lim}, p_time => ${p_time}::timestamptz)`,+ );+}++export function queryIndexAtDomainSplitLevelOmce<T = Record<string, any>>(+ i_level: number,+ i_domain_hash: Buffer,+ i_newer_than: string,+ limit?: number,+): Promise<T[]> {+ return execRows(+ dbIndex,+ sql`select * from query_index_at_domain_split_level_omce(i_level => ${i_level}, i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)${limit !== undefined ? sql` limit ${limit}` : sql``}`,+ );+}++export function queryMaxAge(+ i_domain_hash: Buffer,+): Promise<{ max_age: number | null }[]> {+ return execRows(+ dbIndex,+ sql`select * from query_max_age(i_domain_hash => ${i_domain_hash})`,+ );+}++export function indexGetRecent4(params: {+ url_hash: Buffer;+ max_age_ms: number;+ is_mobile: boolean;+ block_ads: boolean;+ feature_screenshot: boolean;+ feature_screenshot_fullscreen: boolean;+ location_country: string | null;+ location_languages: string[] | null;+ wait_time_ms: number;+ is_stealth: boolean;+ min_age_ms: number | null;+}): Promise<{ id: string; created_at: string; status: number }[]> {+ return execRows(+ dbIndex,+ sql`select * from index_get_recent_4(p_url_hash => ${params.url_hash}, p_max_age_ms => ${params.max_age_ms}, p_is_mobile => ${params.is_mobile}, p_block_ads => ${params.block_ads}, p_feature_screenshot => ${params.feature_screenshot}, p_feature_screenshot_fullscreen => ${params.feature_screenshot_fullscreen}, p_location_country => ${params.location_country}, p_location_languages => ${sql.param(params.location_languages)}::text[], p_wait_time_ms => ${params.wait_time_ms}, p_is_stealth => ${params.is_stealth}, p_min_age_ms => ${params.min_age_ms})`,+ );+}++export function queryTopUrlsForDomain<T = Record<string, any>>(+ p_domain_hash: Buffer,+ p_time_window: string,+ p_top_n: number,+): Promise<T[]> {+ return execRows(+ dbIndex,+ sql`select * from query_top_urls_for_domain(p_domain_hash => ${p_domain_hash}, p_time_window => ${p_time_window}::interval, p_top_n => ${p_top_n})`,+ );+}diff --git a/apps/api/src/db/schema/index-db.ts b/apps/api/src/db/schema/index-db.tsnew file mode 100644index 000000000..ec1cda32b--- /dev/null+++ b/apps/api/src/db/schema/index-db.ts@@ -0,0 +1,72 @@+import {+ pgTable,+ bigint,+ text,+ integer,+ boolean,+ timestamp,+ bytea,+ uuid,+ varchar,+} from "drizzle-orm/pg-core";++// Tables in the separate index Postgres project (INDEX_DATABASE_URL).++const ts = (name: string) =>+ timestamp(name, { withTimezone: true, mode: "string" });++const urlSplitHashes = Object.fromEntries(+ Array.from({ length: 10 }, (_, i) => {+ const col = bytea(`url_split_${i}_hash`);+ return [`url_split_${i}_hash`, i === 0 ? col.notNull() : col];+ }),+);++const domainSplitHashes = Object.fromEntries(+ Array.from({ length: 5 }, (_, i) => [+ `domain_splits_${i}_hash`,+ bytea(`domain_splits_${i}_hash`),+ ]),+);++export const index = pgTable("index", {+ id: uuid("id").primaryKey().defaultRandom(),+ created_at: ts("created_at").notNull().defaultNow(),+ url: text("url").notNull(),+ url_hash: bytea("url_hash").notNull(),+ original_url: text("original_url").notNull(),+ resolved_url: text("resolved_url").notNull(),+ has_screenshot: boolean("has_screenshot").notNull(),+ has_screenshot_fullscreen: boolean("has_screenshot_fullscreen").notNull(),+ is_mobile: boolean("is_mobile").notNull(),+ block_ads: boolean("block_ads").notNull(),+ location_country: text("location_country"),+ location_languages: text("location_languages").array(),+ status: integer("status").notNull(),+ title: text("title"),+ description: text("description"),+ invalidated_at: ts("invalidated_at"),+ is_precrawl: boolean("is_precrawl"),+ wait_time_ms: bigint("wait_time_ms", { mode: "number" }),+ is_stealth: boolean("is_stealth").notNull().default(false),+ ...urlSplitHashes,+ ...domainSplitHashes,+});++export const engpicker_queue = pgTable("engpicker_queue", {+ id: bigint("id", { mode: "number" })+ .primaryKey()+ .generatedByDefaultAsIdentity(),+ domain_hash: bytea("domain_hash").notNull(),+ domain_level: integer("domain_level").notNull(),+ picked_up_at: ts("picked_up_at"),+ done: boolean("done").notNull().default(false),+ created_at: ts("created_at").notNull().defaultNow(),+});++export const engpicker_verdicts = pgTable("engpicker_verdicts", {+ id: uuid("id").primaryKey().defaultRandom(),+ domain_hash: bytea("domain_hash").notNull(),+ verdict: varchar("verdict").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+});diff --git a/apps/api/src/db/schema/index.ts b/apps/api/src/db/schema/index.tsnew file mode 100644index 000000000..402cd0bb1--- /dev/null+++ b/apps/api/src/db/schema/index.ts@@ -0,0 +1,3 @@+export * from "./public";+export * from "./ledger";+export * from "./index-db";diff --git a/apps/api/src/db/schema/ledger.ts b/apps/api/src/db/schema/ledger.tsnew file mode 100644index 000000000..cc4a4da9d--- /dev/null+++ b/apps/api/src/db/schema/ledger.ts@@ -0,0 +1,40 @@+import {+ pgSchema,+ bigint,+ boolean,+ text,+ timestamp,+ uuid,+ jsonb,+} from "drizzle-orm/pg-core";++const ledger = pgSchema("ledger");++const ts = (name: string) =>+ timestamp(name, { withTimezone: true, mode: "string" });+const bigintNum = (name: string) => bigint(name, { mode: "number" });++export const provider_definitions = ledger.table("provider_definitions", {+ id: bigintNum("id").notNull().generatedAlwaysAsIdentity(),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+ is_active: boolean("is_active").notNull().default(true),+ provider_id: bigintNum("provider_id").notNull(),+ slug: text("slug").notNull(),+ name: text("name").notNull(),+ description: text("description"),+ provider_definition_type_id: bigintNum(+ "provider_definition_type_id",+ ).notNull(),+});++export const tracks = ledger.table("tracks", {+ id: bigintNum("id").notNull().generatedAlwaysAsIdentity(),+ is_active: boolean("is_active").notNull().default(true),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+ received_at: ts("received_at").notNull().defaultNow(),+ provider_definition_id: bigintNum("provider_definition_id").notNull(),+ uuid: uuid("uuid").notNull().defaultRandom(),+ data: jsonb("data").notNull().default({}),+});diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsnew file mode 100644index 000000000..9b6c2c7e0--- /dev/null+++ b/apps/api/src/db/schema/public.ts@@ -0,0 +1,542 @@+import { sql } from "drizzle-orm";+import {+ pgTable,+ uuid,+ text,+ varchar,+ integer,+ smallint,+ bigint,+ boolean,+ jsonb,+ numeric,+ timestamp,+ date,+ bytea,+} from "drizzle-orm/pg-core";++const ts = (name: string) =>+ timestamp(name, { withTimezone: true, mode: "string" });+const bigintNum = (name: string) => bigint(name, { mode: "number" });+const num = (name: string) => numeric(name, { mode: "number" });++export const agent_sponsors = pgTable("agent_sponsors", {+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),+ email: text("email").notNull(),+ status: text("status").notNull().default("pending"),+ verification_deadline: ts("verification_deadline").notNull(),+ agent_name: text("agent_name").notNull(),+ sandboxed_team_id: uuid("sandboxed_team_id"),+ api_key_id: bigintNum("api_key_id"),+ requesting_ip: text("requesting_ip"),+ tos_version: text("tos_version"),+ tos_hash: text("tos_hash"),+ verification_token: text("verification_token").notNull(),+ created_at: ts("created_at").defaultNow(),+ verified_at: ts("verified_at"),+ updated_at: ts("updated_at").defaultNow(),+});++export const agents = pgTable("agents", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ team_id: uuid("team_id").notNull(),+ options: jsonb("options"),+ created_at: ts("created_at").notNull().defaultNow(),+ time_taken: num("time_taken").notNull(),+ credits_cost: integer("credits_cost").notNull(),+ cost_tracking: jsonb("cost_tracking"),+ is_successful: boolean("is_successful").notNull(),+ error: text("error"),+});++export const api_keys = pgTable("api_keys", {+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),+ created_at: ts("created_at").defaultNow(),+ key: uuid("key").defaultRandom(),+ name: text("name"),+ team_id: uuid("team_id"),+ owner_id: uuid("owner_id"),+ agent_provisioned: boolean("agent_provisioned").default(false),+});++export const batch_scrapes = pgTable("batch_scrapes", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ team_id: uuid("team_id").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ num_docs: integer("num_docs").notNull(),+ credits_cost: integer("credits_cost").notNull(),+ cancelled: boolean("cancelled").notNull(),+});++export const blocklist = pgTable("blocklist", {+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),+ data: jsonb("data").notNull(),+});++export const blocklist_hits = pgTable("blocklist_hits", {+ id: uuid("id").notNull(),+ domain: text("domain").notNull(),+ url: text("url"),+ team_id: uuid("team_id"),+ origin: text("origin"),+ created_at: ts("created_at").notNull().defaultNow(),+});++export const browser_session_activities = pgTable(+ "browser_session_activities",+ {+ id: bigintNum("id").notNull().generatedAlwaysAsIdentity(),+ team_id: text("team_id").notNull(),+ session_id: text("session_id").notNull(),+ language: text("language").notNull(),+ timeout: integer("timeout").notNull(),+ exit_code: integer("exit_code"),+ killed: boolean("killed").notNull().default(false),+ created_at: ts("created_at").notNull().defaultNow(),+ source: text("source").default("browser"),+ },+);++export const browser_sessions = pgTable("browser_sessions", {+ id: uuid("id").notNull(),+ team_id: text("team_id").notNull(),+ browser_id: text("browser_id").notNull(),+ workspace_id: text("workspace_id").notNull(),+ context_id: text("context_id").notNull(),+ cdp_url: text("cdp_url").notNull(),+ cdp_path: text("cdp_path").notNull(),+ stream_web_view: boolean("stream_web_view").notNull().default(false),+ status: text("status").notNull().default("active"),+ ttl_total: integer("ttl_total").notNull(),+ ttl_without_activity: integer("ttl_without_activity"),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+ deleted_at: ts("deleted_at"),+ credits_used: integer("credits_used"),+ cdp_interactive_path: text("cdp_interactive_path"),+ scrape_id: uuid("scrape_id"),+});++export const concurrency_log = pgTable("concurrency_log", {+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),+ created_at: ts("created_at").notNull().defaultNow(),+ team_id: uuid("team_id").notNull(),+ concurrency: integer("concurrency").notNull(),+});++export const crawls = pgTable("crawls", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ url: text("url").notNull(),+ team_id: uuid("team_id").notNull(),+ options: jsonb("options"),+ created_at: ts("created_at").notNull().defaultNow(),+ num_docs: integer("num_docs").notNull(),+ credits_cost: integer("credits_cost").notNull(),+ cancelled: boolean("cancelled").notNull(),+ monitor_id: uuid("monitor_id"),+ monitor_check_id: uuid("monitor_check_id"),+});++export const deep_researches = pgTable("deep_researches", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ query: text("query").notNull(),+ team_id: uuid("team_id").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ time_taken: num("time_taken").notNull(),+ credits_cost: integer("credits_cost").notNull(),+ cost_tracking: jsonb("cost_tracking"),+ options: jsonb("options"),+});++export const eb_sync = pgTable("eb-sync", {+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),+ created_at: ts("created_at").notNull().defaultNow(),+ team_id: text("team_id"),+});++export const extracts = pgTable("extracts", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ urls: text("urls").array().notNull(),+ options: jsonb("options"),+ model_kind: text("model_kind").notNull(),+ team_id: uuid("team_id").notNull(),+ is_successful: boolean("is_successful").notNull(),+ error: text("error"),+ created_at: ts("created_at").notNull().defaultNow(),+ credits_cost: integer("credits_cost").notNull(),+ cost_tracking: jsonb("cost_tracking"),+});++export const idempotency_keys = pgTable("idempotency_keys", {+ key: uuid("key").notNull().defaultRandom(),+ created_at: ts("created_at").notNull().defaultNow(),+});++export const llm_texts = pgTable("llm_texts", {+ id: uuid("id").notNull().defaultRandom(),+ origin_url: text("origin_url").notNull(),+ llmstxt: text("llmstxt").notNull(),+ llmstxt_full: text("llmstxt_full").notNull(),+ max_urls: integer("max_urls").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at"),+});++export const llmstxts = pgTable("llmstxts", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ url: text("url").notNull(),+ team_id: uuid("team_id").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ num_urls: integer("num_urls").notNull(),+ options: jsonb("options"),+ cost_tracking: jsonb("cost_tracking"),+ credits_cost: integer("credits_cost").notNull(),+});++export const maps = pgTable("maps", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ url: text("url").notNull(),+ options: jsonb("options"),+ team_id: uuid("team_id").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ num_results: integer("num_results").notNull(),+ credits_cost: integer("credits_cost").notNull(),+});++export const monitor_check_pages = pgTable("monitor_check_pages", {+ id: uuid("id").notNull(),+ check_id: uuid("check_id").notNull(),+ monitor_id: uuid("monitor_id").notNull(),+ team_id: uuid("team_id").notNull(),+ target_id: text("target_id").notNull(),+ url: text("url").notNull(),+ url_hash: bytea("url_hash").notNull(),+ status: text("status").notNull(),+ previous_scrape_id: uuid("previous_scrape_id"),+ current_scrape_id: uuid("current_scrape_id"),+ diff_gcs_key: text("diff_gcs_key"),+ diff_text_bytes: integer("diff_text_bytes"),+ diff_json_bytes: integer("diff_json_bytes"),+ status_code: integer("status_code"),+ error: text("error"),+ metadata: jsonb("metadata"),+ created_at: ts("created_at").notNull().defaultNow(),+ judgment: jsonb("judgment"),+});++export const monitor_checks = pgTable("monitor_checks", {+ id: uuid("id").notNull(),+ monitor_id: uuid("monitor_id").notNull(),+ team_id: uuid("team_id").notNull(),+ trigger: text("trigger").notNull(),+ status: text("status").notNull().default("queued"),+ scheduled_for: ts("scheduled_for"),+ started_at: ts("started_at"),+ finished_at: ts("finished_at"),+ estimated_credits: integer("estimated_credits"),+ reserved_credits: integer("reserved_credits"),+ actual_credits: integer("actual_credits"),+ autumn_lock_id: text("autumn_lock_id"),+ billing_status: text("billing_status").notNull().default("not_applicable"),+ total_pages: integer("total_pages").notNull().default(0),+ same_count: integer("same_count").notNull().default(0),+ changed_count: integer("changed_count").notNull().default(0),+ new_count: integer("new_count").notNull().default(0),+ removed_count: integer("removed_count").notNull().default(0),+ error_count: integer("error_count").notNull().default(0),+ target_results: jsonb("target_results"),+ webhook_payload: jsonb("webhook_payload"),+ email_payload: jsonb("email_payload"),+ notification_status: jsonb("notification_status"),+ error: text("error"),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+});++export const monitor_email_recipients = pgTable("monitor_email_recipients", {+ id: uuid("id").notNull().defaultRandom(),+ monitor_id: uuid("monitor_id").notNull(),+ team_id: uuid("team_id").notNull(),+ email: text("email").notNull(),+ status: text("status").notNull().default("pending"),+ token: text("token").notNull(),+ source: text("source").notNull().default("opt_in"),+ confirmation_sent_at: ts("confirmation_sent_at"),+ confirmed_at: ts("confirmed_at"),+ unsubscribed_at: ts("unsubscribed_at"),+ last_notified_at: ts("last_notified_at"),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+});++export const monitor_pages = pgTable("monitor_pages", {+ id: uuid("id").notNull().defaultRandom(),+ monitor_id: uuid("monitor_id").notNull(),+ team_id: uuid("team_id").notNull(),+ target_id: text("target_id").notNull(),+ url: text("url").notNull(),+ url_hash: bytea("url_hash").notNull(),+ source: text("source").notNull(),+ first_seen_check_id: uuid("first_seen_check_id"),+ last_seen_check_id: uuid("last_seen_check_id"),+ last_changed_check_id: uuid("last_changed_check_id"),+ last_scrape_id: uuid("last_scrape_id"),+ last_status: text("last_status").notNull(),+ is_removed: boolean("is_removed").notNull().default(false),+ removed_at: ts("removed_at"),+ metadata: jsonb("metadata"),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+});++export const monitors = pgTable("monitors", {+ id: uuid("id").notNull(),+ team_id: uuid("team_id").notNull(),+ name: text("name").notNull(),+ status: text("status").notNull().default("active"),+ schedule_cron: text("schedule_cron").notNull(),+ schedule_timezone: text("schedule_timezone").notNull().default("UTC"),+ next_run_at: ts("next_run_at"),+ last_run_at: ts("last_run_at"),+ last_check_id: uuid("last_check_id"),+ current_check_id: uuid("current_check_id"),+ locked_at: ts("locked_at"),+ locked_until: ts("locked_until"),+ retention_days: integer("retention_days").notNull().default(30),+ estimated_credits_per_month: integer("estimated_credits_per_month"),+ targets: jsonb("targets").notNull().default([]),+ webhook: jsonb("webhook"),+ notification: jsonb("notification"),+ last_check_summary: jsonb("last_check_summary"),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+ deleted_at: ts("deleted_at"),+ goal: text("goal"),+ judge_enabled: boolean("judge_enabled").notNull().default(false),+});++export const notification_preferences = pgTable("notification_preferences", {+ id: uuid("id").notNull().defaultRandom(),+ user_id: uuid("user_id").notNull(),+ last_referral_notification: ts("last_referral_notification"),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+ email_preferences: text("email_preferences")+ .array()+ .default(["rate_limit_warnings", "system_alerts"]),+ unsubscribed_all: boolean("unsubscribed_all").default(false),+});++export const parses = pgTable("parses", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ url: text("url").notNull(),+ is_successful: boolean("is_successful").notNull(),+ error: text("error"),+ time_taken: num("time_taken").notNull(),+ team_id: uuid("team_id").notNull(),+ options: jsonb("options"),+ cost_tracking: jsonb("cost_tracking"),+ pdf_num_pages: integer("pdf_num_pages"),+ credits_cost: integer("credits_cost").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+});++export const prices = pgTable("prices", {+ id: text("id").notNull(),+ product_id: text("product_id"),+ active: boolean("active").notNull().default(true),+ description: text("description"),+ unit_amount: bigintNum("unit_amount"),+ currency: text("currency").default("usd"),+ type: text("type").default("recurring"),+ interval: text("interval"),+ interval_count: integer("interval_count").default(1),+ trial_period_days: integer("trial_period_days").default(0),+ metadata: jsonb("metadata").default({}),+ credits: integer("credits").default(50),+ is_usage: boolean("is_usage").default(false),+ rate_limits: jsonb("rate_limits"),+ concurrency: integer("concurrency"),+ plan_priority: jsonb("plan_priority"),+ upfront: boolean("upfront").default(false),+ slug: text("slug")+ .notNull()+ .default(sql`gen_random_uuid()`),+ should_be_graceful: boolean("should_be_graceful").default(false),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+ unit_amount_rollover_cap: smallint("unit_amount_rollover_cap")+ .notNull()+ .default(0),+ associated_auto_recharge_price_id: text("associated_auto_recharge_price_id"),+ exp_pack_max_per_month: integer("exp_pack_max_per_month"),+});++export const products = pgTable("products", {+ id: text("id").notNull(),+ active: boolean("active").notNull().default(true),+ name: text("name"),+ description: text("description"),+ image: text("image"),+ metadata: jsonb("metadata").notNull().default({}),+ order: smallint("order"),+ test_mode: boolean("test_mode").default(true),+ is_enterprise: boolean("is_enterprise").default(false),+ is_extract: boolean("is_extract").default(false),+ is_displayed: boolean("is_displayed").default(false),+ is_stripe_test: boolean("is_stripe_test").default(false),+ slug: text("slug"),+ updated_at: ts("updated_at").notNull().defaultNow(),+ type: text("type"),+});++export const requests = pgTable("requests", {+ id: uuid("id").notNull(),+ kind: text("kind").notNull(),+ api_version: text("api_version").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ team_id: uuid("team_id").notNull(),+ origin: text("origin").notNull(),+ integration: text("integration"),+ target_hint: text("target_hint").notNull(),+ dr_clean_by: ts("dr_clean_by"),+ api_key_id: bigintNum("api_key_id"),+});++export const scrapes = pgTable("scrapes", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ url: text("url").notNull(),+ is_successful: boolean("is_successful").notNull(),+ error: text("error"),+ time_taken: num("time_taken").notNull(),+ team_id: uuid("team_id").notNull(),+ options: jsonb("options"),+ cost_tracking: jsonb("cost_tracking"),+ pdf_num_pages: integer("pdf_num_pages"),+ credits_cost: integer("credits_cost").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ monitor_id: uuid("monitor_id"),+ monitor_check_id: uuid("monitor_check_id"),+});++export const search_feedback = pgTable("search_feedback", {+ id: uuid("id").notNull().defaultRandom(),+ search_id: uuid("search_id").notNull(),+ team_id: uuid("team_id").notNull(),+ overall_rating: text("overall_rating").notNull(),+ valuable_sources: jsonb("valuable_sources").notNull().default([]),+ missing_content: jsonb("missing_content").notNull().default([]),+ query_suggestions: text("query_suggestions"),+ integration: text("integration"),+ origin: text("origin"),+ credits_refunded: integer("credits_refunded").notNull().default(0),+ created_at: ts("created_at").notNull().defaultNow(),+});++export const searches = pgTable("searches", {+ id: uuid("id").notNull(),+ request_id: uuid("request_id").notNull(),+ query: text("query").notNull(),+ team_id: uuid("team_id").notNull(),+ options: jsonb("options"),+ time_taken: num("time_taken").notNull(),+ created_at: ts("created_at").notNull().defaultNow(),+ credits_cost: integer("credits_cost").notNull(),+ is_successful: boolean("is_successful").notNull(),+ error: text("error"),+ num_results: integer("num_results").notNull(),+});++export const subscriptions = pgTable("subscriptions", {+ id: text("id").notNull(),+ user_id: uuid("user_id").notNull(),+ status: text("status"),+ metadata: jsonb("metadata"),+ price_id: text("price_id"),+ quantity: integer("quantity"),+ cancel_at_period_end: boolean("cancel_at_period_end"),+ created: ts("created").notNull().defaultNow(),+ current_period_start: ts("current_period_start").notNull().defaultNow(),+ current_period_end: ts("current_period_end").notNull().defaultNow(),+ ended_at: ts("ended_at").defaultNow(),+ cancel_at: ts("cancel_at").defaultNow(),+ canceled_at: ts("canceled_at").defaultNow(),+ trial_start: ts("trial_start").defaultNow(),+ trial_end: ts("trial_end").defaultNow(),+ team_id: uuid("team_id"),+ is_usage: boolean("is_usage").default(false),+ is_extract: boolean("is_extract").default(false),+ updated_at: ts("updated_at").notNull().defaultNow(),+ expansion_associated_coupon: bigintNum("expansion_associated_coupon"),+ pending_price_id: text("pending_price_id"),+ pending_price_effective_at: ts("pending_price_effective_at"),+ pending_schedule_id: text("pending_schedule_id"),+});++export const teams = pgTable("teams", {+ id: uuid("id").notNull().defaultRandom(),+ created_at: ts("created_at").defaultNow(),+ name: text("name"),+ banned: boolean("banned").default(false),+ idmux_expires_at: ts("idmux_expires_at"),+ updated_at: ts("updated_at").defaultNow(),+ hmac_secret: text("hmac_secret")+ .notNull()+ .default(sql`encode(extensions.gen_random_bytes(32), 'hex'::text)`),+ referrer_integration: varchar("referrer_integration"),+ allocated_concurrent_browsers: integer("allocated_concurrent_browsers"),+ org_id: uuid("org_id").notNull(),+});++export const user_notifications = pgTable("user_notifications", {+ id: uuid("id").notNull().defaultRandom(),+ team_id: uuid("team_id"),+ notification_type: text("notification_type").notNull(),+ sent_date: date("sent_date", { mode: "string" }).notNull(),+ email_id: text("email_id"),+ timestamp: ts("timestamp"),+ read: boolean("read").default(false),+ metadata: jsonb("metadata"),+});++export const user_teams = pgTable("user_teams", {+ user_id: uuid("user_id").notNull(),+ team_id: uuid("team_id").notNull(),+ role: text("role").default("admin"),+ created_at: ts("created_at").defaultNow(),+});++export const users = pgTable("users", {+ id: uuid("id").notNull(),+ full_name: text("full_name"),+ avatar_url: text("avatar_url"),+ billing_address: jsonb("billing_address"),+ payment_method: jsonb("payment_method"),+ email: text("email"),+ team_id: uuid("team_id"),+ created_on: ts("created_on").defaultNow(),+ referrer_integration: varchar("referrer_integration"),+});++export const webhook_logs = pgTable("webhook_logs", {+ id: uuid("id").notNull().defaultRandom(),+ success: boolean("success").notNull(),+ error: text("error"),+ team_id: uuid("team_id").notNull(),+ crawl_id: uuid("crawl_id").notNull(),+ scrape_id: uuid("scrape_id"),+ created_at: ts("created_at").notNull().defaultNow(),+ url: text("url").notNull(),+ status_code: smallint("status_code"),+ event: text("event").notNull(),+ latency_ms: integer("latency_ms").default(0),+});diff --git a/apps/api/src/lib/browser-session-activity.ts b/apps/api/src/lib/browser-session-activity.tsindex 74d7659b8..7f833b425 100644--- a/apps/api/src/lib/browser-session-activity.ts+++ b/apps/api/src/lib/browser-session-activity.ts@@ -1,5 +1,6 @@import { redisEvictConnection } from "../services/redis";-import { supabase_service } from "../services/supabase";+import { db } from "../db/connection";+import * as schema from "../db/schema";import { logger as _logger } from "./logger";const logger = _logger.child({ module: "browser-sessions" });@@ -36,16 +37,7 @@ export async function processBrowserSessionActivityJobs() {const rows: BrowserSessionActivityEvent[] = raw.map(x => JSON.parse(x));try {- const { error } = await supabase_service- .from("browser_session_activities")- .insert(rows);-- if (error) {- logger.error("Failed to insert browser session activities", {- error,- count: rows.length,- });- }+ await db.insert(schema.browser_session_activities).values(rows);} catch (err) {logger.error("Error inserting browser session activities", {err,diff --git a/apps/api/src/lib/browser-sessions.ts b/apps/api/src/lib/browser-sessions.tsindex d3b27a3cc..05aa3dcd2 100644--- a/apps/api/src/lib/browser-sessions.ts+++ b/apps/api/src/lib/browser-sessions.ts@@ -1,5 +1,7 @@+import { and, desc, eq } from "drizzle-orm";import { deleteKey, getValue, setValue } from "../services/redis";-import { isPostgrestNoRowsError, supabase_service } from "../services/supabase";+import { db } from "../db/connection";+import * as schema from "../db/schema";import { logger as _logger } from "./logger";const logger = _logger.child({ module: "browser-sessions" });@@ -29,8 +31,6 @@ interface BrowserSessionRow {updated_at: string; // ISO timestamp}-const TABLE = "browser_sessions";-// ---------------------------------------------------------------------------// CRUD helpers// ---------------------------------------------------------------------------@@ -49,29 +49,19 @@ export async function insertBrowserSession(let lastError: any = null;for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {try {- const { data, error } = await supabase_service- .from(TABLE)- .insert(full)- .select()- .single();-- if (error) {- lastError = error;- logger.error(- "Error inserting browser session due to Supabase error, trying again",- { error, id: row.id, attempt },- );- await new Promise(resolve => setTimeout(resolve, 75));- continue;- }+ const [data] = await db+ .insert(schema.browser_sessions)+ .values(full)+ .returning();return data as BrowserSessionRow;} catch (error) {lastError = error;- logger.error(- "Error inserting browser session due to unknown error, trying again",- { error, id: row.id, attempt },- );+ logger.error("Error inserting browser session, trying again", {+ error,+ id: row.id,+ attempt,+ });await new Promise(resolve => setTimeout(resolve, 75));}}@@ -89,72 +79,70 @@ export async function insertBrowserSession(export async function getBrowserSession(id: string,): Promise<BrowserSessionRow | null> {- const { data, error } = await supabase_service- .from(TABLE)- .select("*")- .eq("id", id)- .single();-- if (error) {- if (isPostgrestNoRowsError(error)) return null;+ try {+ const [data] = await db+ .select()+ .from(schema.browser_sessions)+ .where(eq(schema.browser_sessions.id, id))+ .limit(1);+ return (data ?? null) as BrowserSessionRow | null;+ } catch (error) {logger.error("Failed to get browser session", { error, id });- throw new Error(`Failed to get browser session: ${error.message}`);+ throw new Error(+ `Failed to get browser session: ${error instanceof Error ? error.message : JSON.stringify(error)}`,+ );}-- return data as BrowserSessionRow;}export async function getBrowserSessionFromScrape(id: string,): Promise<BrowserSessionRow | null> {- const { data, error } = await supabase_service- .from(TABLE)- .select("*")- .eq("scrape_id", id)- .single();-- if (error) {- if (isPostgrestNoRowsError(error)) return null;+ try {+ const [data] = await db+ .select()+ .from(schema.browser_sessions)+ .where(eq(schema.browser_sessions.scrape_id, id))+ .limit(1);+ return (data ?? null) as BrowserSessionRow | null;+ } catch (error) {logger.error("Failed to get browser session from scrape", { error, id });throw new Error(- `Failed to get browser session from scrape: ${error.message}`,+ `Failed to get browser session from scrape: ${error instanceof Error ? error.message : JSON.stringify(error)}`,);}-- return data as BrowserSessionRow;}export async function listBrowserSessions(teamId: string,opts?: { status?: BrowserSessionStatus },): Promise<BrowserSessionRow[]> {- let query = supabase_service- .from(TABLE)- .select("*")- .eq("team_id", teamId)- .order("created_at", { ascending: false });-+ const conditions = [eq(schema.browser_sessions.team_id, teamId)];if (opts?.status) {- query = query.eq("status", opts.status);+ conditions.push(eq(schema.browser_sessions.status, opts.status));}- const { data, error } = await query;-- if (error) {+ try {+ const data = await db+ .select()+ .from(schema.browser_sessions)+ .where(and(...conditions))+ .orderBy(desc(schema.browser_sessions.created_at));+ return data as BrowserSessionRow[];+ } catch (error) {logger.error("Failed to list browser sessions", { error, teamId });- throw new Error(`Failed to list browser sessions: ${error.message}`);+ throw new Error(+ `Failed to list browser sessions: ${error instanceof Error ? error.message : JSON.stringify(error)}`,+ );}-- return (data ?? []) as BrowserSessionRow[];}export async function updateBrowserSessionActivity(id: string): Promise<void> {- const { error } = await supabase_service- .from(TABLE)- .update({ updated_at: new Date().toISOString() })- .eq("id", id);-- if (error) {+ try {+ await db+ .update(schema.browser_sessions)+ .set({ updated_at: new Date().toISOString() })+ .where(eq(schema.browser_sessions.id, id));+ } catch (error) {logger.warn("Failed to update browser session activity", { error, id });}}@@ -162,40 +150,38 @@ export async function updateBrowserSessionActivity(id: string): Promise<void> {export async function getBrowserSessionByBrowserId(browserId: string,): Promise<BrowserSessionRow | null> {- const { data, error } = await supabase_service- .from(TABLE)- .select("*")- .eq("browser_id", browserId)- .single();-- if (error) {- if (isPostgrestNoRowsError(error)) return null;+ try {+ const [data] = await db+ .select()+ .from(schema.browser_sessions)+ .where(eq(schema.browser_sessions.browser_id, browserId))+ .limit(1);+ return (data ?? null) as BrowserSessionRow | null;+ } catch (error) {logger.error("Failed to get browser session by browser_id", {error,browserId,});throw new Error(- `Failed to get browser session by browser_id: ${error.message}`,+ `Failed to get browser session by browser_id: ${error instanceof Error ? error.message : JSON.stringify(error)}`,);}-- return data as BrowserSessionRow;}export async function updateBrowserSessionStatus(id: string,status: BrowserSessionStatus,): Promise<void> {- const { error } = await supabase_service- .from(TABLE)- .update({- status,- updated_at: new Date().toISOString(),- deleted_at: status === "destroyed" ? new Date().toISOString() : null,- })- .eq("id", id);-- if (error) {+ try {+ await db+ .update(schema.browser_sessions)+ .set({+ status,+ updated_at: new Date().toISOString(),+ deleted_at: status === "destroyed" ? new Date().toISOString() : null,+ })+ .where(eq(schema.browser_sessions.id, id));+ } catch (error) {logger.warn("Failed to update browser session status", { error, id });}}@@ -204,35 +190,38 @@ export async function claimBrowserSessionDestroyed(id: string,): Promise<boolean> {const now = new Date().toISOString();- const { data, error } = await supabase_service- .from(TABLE)- .update({- status: "destroyed" as BrowserSessionStatus,- updated_at: now,- deleted_at: now,- })- .eq("id", id)- .eq("status", "active")- .select("id");-- if (error) {+ try {+ const data = await db+ .update(schema.browser_sessions)+ .set({+ status: "destroyed" as BrowserSessionStatus,+ updated_at: now,+ deleted_at: now,+ })+ .where(+ and(+ eq(schema.browser_sessions.id, id),+ eq(schema.browser_sessions.status, "active"),+ ),+ )+ .returning({ id: schema.browser_sessions.id });+ return data.length > 0;+ } catch (error) {logger.warn("Failed to claim browser session destroyed", { error, id });return false;}-- return (data?.length ?? 0) > 0;}export async function updateBrowserSessionScrapeId(id: string,scrapeId: string,): Promise<void> {- const { error } = await supabase_service- .from(TABLE)- .update({ scrape_id: scrapeId, updated_at: new Date().toISOString() })- .eq("id", id);-- if (error) {+ try {+ await db+ .update(schema.browser_sessions)+ .set({ scrape_id: scrapeId, updated_at: new Date().toISOString() })+ .where(eq(schema.browser_sessions.id, id));+ } catch (error) {logger.warn("Failed to update browser session scrape_id", {error,id,@@ -245,12 +234,15 @@ export async function updateBrowserSessionCreditsUsed(id: string,creditsUsed: number,): Promise<void> {- const { error } = await supabase_service- .from(TABLE)- .update({ credits_used: creditsUsed, updated_at: new Date().toISOString() })- .eq("id", id);-- if (error) {+ try {+ await db+ .update(schema.browser_sessions)+ .set({+ credits_used: creditsUsed,+ updated_at: new Date().toISOString(),+ })+ .where(eq(schema.browser_sessions.id, id));+ } catch (error) {logger.warn("Failed to update browser session credits_used", {error,id,diff --git a/apps/api/src/lib/engpicker.ts b/apps/api/src/lib/engpicker.tsindex dc9104686..74fddfdb8 100644--- a/apps/api/src/lib/engpicker.ts+++ b/apps/api/src/lib/engpicker.ts@@ -3,7 +3,10 @@ import { z } from "zod";import { scrapeOptions } from "../controllers/v2/types";import { scrapeURL } from "../scraper/scrapeURL";import type { Engine } from "../scraper/scrapeURL/engines";-import { index_supabase_service } from "../services";+import { and, desc, eq, inArray, isNull } from "drizzle-orm";+import { dbIndex } from "../db/connection";+import * as schema from "../db/schema";+import { queryIndexAtDomainSplitLevelOmce } from "../db/rpc";import { CostTracking } from "./cost-tracking";import { getModel } from "./generic-ai";import { logger as _logger } from "./logger";@@ -15,7 +18,7 @@ import {type EngpickerJob = {id: number;- domain_hash: string;+ domain_hash: Buffer;domain_level: number;picked_up_at: string | null;done: boolean;@@ -115,17 +118,26 @@ export async function processEngpickerJob() {method: "processEngpickerJob",});- const { data: jobData, error: getJobError } = await index_supabase_service- .from("engpicker_queue")- .update({- picked_up_at: new Date().toISOString(),- })- .is("picked_up_at", null)- .order("created_at", { ascending: false })- .limit(1)- .select("*");-- if (getJobError) {+ let jobData: EngpickerJob[];+ try {+ const candidate = dbIndex+ .select({ id: schema.engpicker_queue.id })+ .from(schema.engpicker_queue)+ .where(isNull(schema.engpicker_queue.picked_up_at))+ .orderBy(desc(schema.engpicker_queue.created_at))+ .limit(1);++ jobData = await dbIndex+ .update(schema.engpicker_queue)+ .set({ picked_up_at: new Date().toISOString() })+ .where(+ and(+ inArray(schema.engpicker_queue.id, candidate),+ isNull(schema.engpicker_queue.picked_up_at),+ ),+ )+ .returning();+ } catch (getJobError) {logger.error("Error picking up engpicker job", { getJobError });await new Promise(resolve => setTimeout(resolve, 1000));return;@@ -146,18 +158,17 @@ export async function processEngpickerJob() {// main shit here- const { data: indexRows, error: indexRowsError } =- await index_supabase_service- .rpc("query_index_at_domain_split_level_omce", {- i_level: job.domain_level,- i_domain_hash: job.domain_hash,- i_newer_than: new Date(- new Date(job.created_at).valueOf() - 1000 * 60 * 60 * 24,- ).toISOString(),- })- .limit(100);-- if (indexRowsError || !indexRows) {+ let indexRows: { url: string }[];+ try {+ indexRows = await queryIndexAtDomainSplitLevelOmce<{ url: string }>(+ job.domain_level,+ job.domain_hash,+ new Date(+ new Date(job.created_at).valueOf() - 1000 * 60 * 60 * 24,+ ).toISOString(),+ 100,+ );+ } catch (indexRowsError) {logger.error("Error querying index rows", { indexRowsError });await new Promise(resolve => setTimeout(resolve, 1000));return;@@ -287,27 +298,23 @@ export async function processEngpickerJob() {// This is the verdict - "TlsClientOk", "ChromeCdpRequired", or "Uncertain"const verdict = verdictResult.verdict;- const { error: insertVerdictError } = await index_supabase_service- .from("engpicker_verdicts")- .insert({+ try {+ await dbIndex.insert(schema.engpicker_verdicts).values({domain_hash: job.domain_hash,verdict,});-- if (insertVerdictError) {+ } catch (insertVerdictError) {logger.error("Error inserting engpicker verdict", { insertVerdictError });await new Promise(resolve => setTimeout(resolve, 1000));return;}- const { error: updateJobError } = await index_supabase_service- .from("engpicker_queue")- .update({- done: true,- })- .eq("id", job.id);-- if (updateJobError) {+ try {+ await dbIndex+ .update(schema.engpicker_queue)+ .set({ done: true })+ .where(eq(schema.engpicker_queue.id, job.id));+ } catch (updateJobError) {logger.error("Error updating engpicker job", { updateJobError });await new Promise(resolve => setTimeout(resolve, 1000));return;diff --git a/apps/api/src/lib/extract/team-id-sync.ts b/apps/api/src/lib/extract/team-id-sync.tsindex e477583ab..2c0c079b0 100644--- a/apps/api/src/lib/extract/team-id-sync.ts+++ b/apps/api/src/lib/extract/team-id-sync.ts@@ -1,18 +1,17 @@-import { supabase_rr_service, supabase_service } from "../../services/supabase";+import { eq } from "drizzle-orm";+import { dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { logger } from "../logger";import { withAuth } from "../withAuth";async function getTeamIdSyncBOriginal(teamId: string) {try {- const { data, error } = await supabase_rr_service- .from("eb-sync")- .select("team_id")- .eq("team_id", teamId)+ const data = await dbRr+ .select({ team_id: schema.eb_sync.team_id })+ .from(schema.eb_sync)+ .where(eq(schema.eb_sync.team_id, teamId)).limit(1);- if (error) {- throw new Error("Error getting team id (sync b)");- }return data[0] ?? null;} catch (error) {logger.error("Error getting team id (sync b)", error);diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.tsindex a7372e3fe..5943fc929 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts@@ -1,4 +1,6 @@-import { supabase_service } from "../../services/supabase";+import { and, desc, eq, gte } from "drizzle-orm";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";import { config } from "../../config";import { logger } from "../logger";import { normalizeUrl, normalizeUrlOnlyHostname } from "../canonical-url";@@ -21,28 +23,27 @@ export async function getLlmsTextFromCache(const originUrl = normalizeUrlOnlyHostname(url);try {- const { data, error } = await supabase_service- .from("llm_texts")- .select("*")- .eq("origin_url", originUrl)- .gte("max_urls", maxUrls) // Changed to gte since we want cached results with more URLs than requested- .order("updated_at", { ascending: false })- .limit(1)- .single();-- if (error) {- return null;- }+ const [data] = await db+ .select()+ .from(schema.llm_texts)+ .where(+ and(+ eq(schema.llm_texts.origin_url, originUrl),+ gte(schema.llm_texts.max_urls, maxUrls), // gte since we want cached results with more URLs than requested+ ),+ )+ .orderBy(desc(schema.llm_texts.updated_at))+ .limit(1);// Check if data is older than 1 weekconst oneWeekAgo = new Date();oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);- if (!data || new Date(data.updated_at) < oneWeekAgo) {+ if (!data || !data.updated_at || new Date(data.updated_at) < oneWeekAgo) {return null;}- return data;+ return data as LlmsTextCache;} catch (error) {logger.error("Failed to fetch LLMs text from cache", { error, originUrl });return null;@@ -63,52 +64,50 @@ export async function saveLlmsTextToCache(try {// First check if there's an existing entry- const { data: existingData } = await supabase_service- .from("llm_texts")- .select("*")- .eq("origin_url", originUrl)- .single();+ const [existingData] = await db+ .select({ id: schema.llm_texts.id })+ .from(schema.llm_texts)+ .where(eq(schema.llm_texts.origin_url, originUrl))+ .limit(1);if (existingData) {// Update existing entry- const { error } = await supabase_service- .from("llm_texts")- .update({- llmstxt,- llmstxt_full,- max_urls: maxUrls,- updated_at: new Date().toISOString(),- })- .eq("origin_url", originUrl);-- if (error) {- logger.error("Error updating LLMs text in cache", { error, originUrl });- } else {+ try {+ await db+ .update(schema.llm_texts)+ .set({+ llmstxt,+ llmstxt_full,+ max_urls: maxUrls,+ updated_at: new Date().toISOString(),+ })+ .where(eq(schema.llm_texts.origin_url, originUrl));logger.debug("Successfully updated cached LLMs text", {originUrl,maxUrls,});+ } catch (error) {+ logger.error("Error updating LLMs text in cache", { error, originUrl });}} else {// Insert new entry- const { error } = await supabase_service.from("llm_texts").insert({- origin_url: originUrl,- llmstxt,- llmstxt_full,- max_urls: maxUrls,- updated_at: new Date().toISOString(),- });-- if (error) {- logger.error("Error inserting LLMs text to cache", {- error,- originUrl,+ try {+ await db.insert(schema.llm_texts).values({+ origin_url: originUrl,+ llmstxt,+ llmstxt_full,+ max_urls: maxUrls,+ updated_at: new Date().toISOString(),});- } else {logger.debug("Successfully inserted new cached LLMs text", {originUrl,maxUrls,});+ } catch (error) {+ logger.error("Error inserting LLMs text to cache", {+ error,+ originUrl,+ });}}} catch (error) {diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.tsindex b9741e6ee..0c9d8db59 100644--- a/apps/api/src/lib/supabase-jobs.ts+++ b/apps/api/src/lib/supabase-jobs.ts@@ -1,81 +1,66 @@import type { Logger } from "winston";-import { supabase_rr_service, supabase_service } from "../services/supabase";+import { eq, inArray, and } from "drizzle-orm";+import { db, dbRr } from "../db/connection";+import * as schema from "../db/schema";import { logger } from "./logger";import * as Sentry from "@sentry/node";-// ============================================================================-// NEW TABLES: scrapes, requests, crawls, etc.-// ============================================================================-/**- * Get a single scrape by ID from the new scrapes table+ * Get a single scrape by ID from the scrapes table* @param scrapeId ID of Scrape* @returns Scrape data or null*/-export const supabaseGetScrapeById = async (scrapeId: string) => {- const { data, error } = await supabase_rr_service- .from("scrapes")- .select("*")- .eq("id", scrapeId)- .single();-- if (error) {- return null;- }-- if (!data) {+export const supabaseGetScrapeById = async (scrapeId: string): Promise<any> => {+ try {+ const [data] = await dbRr+ .select()+ .from(schema.scrapes)+ .where(eq(schema.scrapes.id, scrapeId))+ .limit(1);+ return data ?? null;+ } catch (error) {return null;}-- return data;};/**- * Get multiple scrapes by ID from the new scrapes table+ * Get multiple scrapes by ID from the scrapes table* @param scrapeIds IDs of Scrapes* @returns Scrape data array*/-export const supabaseGetScrapesById = async (scrapeIds: string[]) => {- const { data, error } = await supabase_rr_service- .from("scrapes")- .select()- .in("id", scrapeIds);-- if (error) {+export const supabaseGetScrapesById = async (+ scrapeIds: string[],+): Promise<any[]> => {+ try {+ return await dbRr+ .select()+ .from(schema.scrapes)+ .where(inArray(schema.scrapes.id, scrapeIds));+ } catch (error) {logger.error(`Error in supabaseGetScrapesById: ${error}`);Sentry.captureException(error);return [];}-- if (!data) {- return [];- }-- return data;};/**- * Get multiple scrapes by request ID (crawl/batch scrape ID) from the new scrapes table+ * Get multiple scrapes by request ID (crawl/batch scrape ID) from the scrapes table* @param requestId ID of the parent request (crawl or batch scrape)* @returns Scrape data array*/-export const supabaseGetScrapesByRequestId = async (requestId: string) => {- const { data, error } = await supabase_rr_service- .from("scrapes")- .select()- .eq("request_id", requestId);-- if (error) {+export const supabaseGetScrapesByRequestId = async (+ requestId: string,+): Promise<any[]> => {+ try {+ return await dbRr+ .select()+ .from(schema.scrapes)+ .where(eq(schema.scrapes.request_id, requestId));+ } catch (error) {logger.error(`Error in supabaseGetScrapesByRequestId: ${error}`);Sentry.captureException(error);return [];}-- if (!data) {- return [];- }-- return data;};/**@@ -86,100 +71,86 @@ export const supabaseGetScrapesByRequestId = async (requestId: string) => {*/export const supabaseGetScrapeByIdOnlyData = async (scrapeId: string,- logger?: Logger,-) => {- const { data, error } = await supabase_rr_service- .from("scrapes")- .select("team_id")- .eq("id", scrapeId)- .single();-- if (error) {- if (logger) {- logger.error("Error in supabaseGetScrapeByIdOnlyData", { error });+ log?: Logger,+): Promise<any> => {+ try {+ const [data] = await dbRr+ .select({ team_id: schema.scrapes.team_id })+ .from(schema.scrapes)+ .where(eq(schema.scrapes.id, scrapeId))+ .limit(1);+ return data ?? null;+ } catch (error) {+ if (log) {+ log.error("Error in supabaseGetScrapeByIdOnlyData", { error });}return null;}-- if (!data) {- return null;- }-- return data;};-export const supabaseGetExtractByIdDirect = async (extractId: string) => {- const { data, error } = await supabase_service- .from("extracts")- .select("*")- .eq("id", extractId)- .single();-- if (error) {- return null;- }-- if (!data) {+export const supabaseGetExtractByIdDirect = async (+ extractId: string,+): Promise<any> => {+ try {+ const [data] = await db+ .select()+ .from(schema.extracts)+ .where(eq(schema.extracts.id, extractId))+ .limit(1);+ return data ?? null;+ } catch (error) {return null;}-- return data;};export const supabaseGetExtractRequestByIdDirect = async (extractId: string,-) => {- const { data, error } = await supabase_service- .from("requests")- .select("*")- .eq("id", extractId)- .in("kind", ["extract", "agent"])- .single();-- if (error) {+): Promise<any> => {+ try {+ const [data] = await db+ .select()+ .from(schema.requests)+ .where(+ and(+ eq(schema.requests.id, extractId),+ inArray(schema.requests.kind, ["extract", "agent"]),+ ),+ )+ .limit(1);+ return data ?? null;+ } catch (error) {return null;}-- if (!data) {- return null;- }-- return data;};-export const supabaseGetAgentRequestByIdDirect = async (agentId: string) => {- const { data, error } = await supabase_service- .from("requests")- .select("*")- .eq("id", agentId)- .eq("kind", "agent")- .single();-- if (error) {+export const supabaseGetAgentRequestByIdDirect = async (+ agentId: string,+): Promise<any> => {+ try {+ const [data] = await db+ .select()+ .from(schema.requests)+ .where(+ and(eq(schema.requests.id, agentId), eq(schema.requests.kind, "agent")),+ )+ .limit(1);+ return data ?? null;+ } catch (error) {return null;}-- if (!data) {- return null;- }-- return data;};-export const supabaseGetAgentByIdDirect = async (agentId: string) => {- const { data, error } = await supabase_service- .from("agents")- .select("*")- .eq("id", agentId)- .single();-- if (error) {+export const supabaseGetAgentByIdDirect = async (+ agentId: string,+): Promise<any> => {+ try {+ const [data] = await db+ .select()+ .from(schema.agents)+ .where(eq(schema.agents.id, agentId))+ .limit(1);+ return data ?? null;+ } catch (error) {return null;}-- if (!data) {- return null;- }-- return data;};diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.tsindex 17b9d5faa..8ab72ecf4 100644--- a/apps/api/src/lib/zdrcleaner.ts+++ b/apps/api/src/lib/zdrcleaner.ts@@ -1,5 +1,8 @@import "dotenv/config";-import { supabase_service } from "../services/supabase";+import { inArray } from "drizzle-orm";+import { db } from "../db/connection";+import * as schema from "../db/schema";+import { getZdrCleanupBatch } from "../db/rpc";import { removeJobFromGCS } from "./gcs-jobs";import { logger as _logger } from "./logger";import { config } from "../config";@@ -21,20 +24,8 @@ export async function zdrcleaner() {// Call the RPC to get all blobs (scrapes, searches, extracts, maps, llmstxts, deep_researches)// associated with requests that need cleanup.// The RPC handles the dr_clean_by filtering logic (team-specific vs scheduled)- const { data: _rows, error } = await supabase_service.rpc(- "get_zdr_cleanup_batch_2",- {- p_limit: 1000,- },- );-- if (error) {- throw new Error("Error calling get_zdr_cleanup_batch_2 RPC", {- cause: error,- });- }-- const rows: { request_id: string; ids: string[] }[] = _rows;+ const rows: { request_id: string; ids: string[] }[] =+ await getZdrCleanupBatch(1000);if (!rows || rows.length === 0) {logger.debug("zdrcleaner batch completed with no rows to process", {@@ -89,20 +80,13 @@ export async function zdrcleaner() {const updateErrors: any[] = [];if (fullyCleanedRequests.length > 0) {- // Process in batches of 100 to deal with URL limits on PostgREST- for (let i = 0; i < fullyCleanedRequests.length; i += 100) {- const batch = fullyCleanedRequests.slice(i, i + 100);- try {- await supabase_service- .from("requests")- .update({- dr_clean_by: null,- })- .in("id", batch)- .throwOnError();- } catch (error) {- updateErrors.push(error);- }+ try {+ await db+ .update(schema.requests)+ .set({ dr_clean_by: null })+ .where(inArray(schema.requests.id, fullyCleanedRequests));+ } catch (error) {+ updateErrors.push(error);}}diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 8c092e427..e4215cb82 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -23,7 +23,7 @@ import { isSelfHosted } from "../lib/deployment";import { validate as isUuid } from "uuid";import { config } from "../config";-import { supabase_service } from "../services/supabase";+import { getAgentFreeRequestsLeft } from "../db/rpc";import { autumnService } from "../services/autumn/autumn.service";export function checkCreditsMiddleware(@@ -99,22 +99,16 @@ export function checkCreditsMiddleware(if (req.path.startsWith("/agent")) {if (config.USE_DB_AUTHENTICATION) {- const { data, error: freeRequestError } = await supabase_service.rpc(- "get_agent_free_requests_left",- {- i_team_id: req.auth.team_id,- },- );-- if (freeRequestError) {+ try {+ const data = await getAgentFreeRequestsLeft(req.auth.team_id);+ if (data?.[0]?.free_requests_left !== 0) {+ return next();+ }+ } catch (freeRequestError) {logger.warn("Failed to get agent free requests left", {error: freeRequestError,teamId: req.auth.team_id,});- } else {- if (data?.[0]?.free_requests_left !== 0) {- return next();- }}}}diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex af51495ca..eabf66ed1 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -3,10 +3,8 @@ import { v7 as uuidv7 } from "uuid";import { config } from "../../../config";import { parse } from "tldts";import { TeamFlags } from "../../../controllers/v1/types";-import {- supabase_rr_service,- supabase_service,-} from "../../../services/supabase";+import { db, dbRr } from "../../../db/connection";+import * as schema from "../../../db/schema";configDotenv();@@ -39,7 +37,7 @@ async function flushHits(): Promise<void> {hitBuffer = [];if (config.USE_DB_AUTHENTICATION !== true) return;try {- await supabase_service.from("blocklist_hits").insert(batch);+ await db.insert(schema.blocklist_hits).values(batch);} catch {}}@@ -114,13 +112,13 @@ export async function initializeBlocklist() {return;}- const { data, error } = await supabase_rr_service- .from("blocklist")- .select("*")- .single();-- if (error) {- throw new Error(`Error getting blocklist: ${error.message}`);+ let data: { data: any } | undefined;+ try {+ [data] = await dbRr.select().from(schema.blocklist).limit(1);+ } catch (error) {+ throw new Error(+ `Error getting blocklist: ${error instanceof Error ? error.message : JSON.stringify(error)}`,+ );}if (!data) {diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 02577f8bd..77ad5fc40 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -6,7 +6,6 @@ import { Meta } from "../..";import {getIndexFromGCS,hashURL,- index_supabase_service,normalizeURLForIndex,saveIndexToGCS,generateURLSplits,@@ -14,6 +13,7 @@ import {generateDomainSplits,addOMCEJob,} from "../../../../services";+import { queryMaxAge, indexGetRecent4 } from "../../../../db/rpc";import {AgentIndexOnlyError,EngineError,@@ -234,23 +234,18 @@ export async function scrapeURLWithIndex(try {maxAge = await Promise.race([(async () => {- const { data, error } = await index_supabase_service.rpc(- "query_max_age",- {- i_domain_hash: domainSplitsHash[level],- },- );-- if (error || !data || data.length === 0) {- meta.logger.warn("Failed to get max age from DB", {- error,- });+ try {+ const data = await queryMaxAge(domainSplitsHash[level]);+ if (!data || data.length === 0) {+ return 2 * 24 * 60 * 60 * 1000; // 2 days+ }+ return data[0].max_age ?? 2 * 24 * 60 * 60 * 1000; // 2 days+ } catch (error) {+ meta.logger.warn("Failed to get max age from DB", { error });return 2 * 24 * 60 * 60 * 1000; // 2 days}-- return data[0].max_age ?? 2 * 24 * 60 * 60 * 1000; // 2 days})(),- new Promise(resolve =>+ new Promise<number>(resolve =>setTimeout(() => {resolve(2 * 24 * 60 * 60 * 1000); // 2 days}, 200),@@ -267,29 +262,27 @@ export async function scrapeURLWithIndex(const checkpoint1 = Date.now();- const { data, error } = await index_supabase_service.rpc(- "index_get_recent_4",- {- p_url_hash: urlHash,- p_max_age_ms: maxAge,- p_is_mobile: meta.options.mobile,- p_block_ads: meta.options.blockAds,- p_feature_screenshot: meta.featureFlags.has("screenshot"),- p_feature_screenshot_fullscreen: meta.featureFlags.has(+ let data: { id: string; created_at: string; status: number }[];+ try {+ data = await indexGetRecent4({+ url_hash: urlHash,+ max_age_ms: maxAge,+ is_mobile: meta.options.mobile,+ block_ads: meta.options.blockAds,+ feature_screenshot: meta.featureFlags.has("screenshot"),+ feature_screenshot_fullscreen: meta.featureFlags.has("screenshot@fullScreen",),- p_location_country: meta.options.location?.country ?? null,- p_location_languages:+ location_country: meta.options.location?.country ?? null,+ location_languages:(meta.options.location?.languages?.length ?? 0) > 0- ? meta.options.location?.languages+ ? (meta.options.location?.languages ?? null): null,- p_wait_time_ms: meta.options.waitFor,- p_is_stealth: meta.featureFlags.has("stealthProxy"),- p_min_age_ms: meta.options.minAge ?? null,- },- );-- if (error || !data) {+ wait_time_ms: meta.options.waitFor,+ is_stealth: meta.featureFlags.has("stealthProxy"),+ min_age_ms: meta.options.minAge ?? null,+ });+ } catch (error) {throw new EngineError("Failed to retrieve URL from DB index", {cause: error,});diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex 16850f846..a36ed8ffc 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -1,4 +1,4 @@-import { supabase_service } from "../../../services/supabase";+import { diffGetLastScrape } from "../../../db/rpc";import { Document } from "../../../controllers/v1/types";import { Meta } from "../index";import gitDiff from "git-diff";@@ -89,11 +89,20 @@ export async function deriveDiff(}const start = Date.now();- const res = await supabase_service.rpc("diff_get_last_scrape_v7", {- i_team_id: meta.internalOptions.teamId,- i_url: document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,- i_tag: changeTrackingFormat?.tag ?? null,- });+ let resData: { o_job_id: string; o_date_added: string }[];+ try {+ resData = await diffGetLastScrape(+ meta.internalOptions.teamId!,+ document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,+ changeTrackingFormat?.tag ?? null,+ );+ } catch (error) {+ meta.logger.error("Error fetching previous scrape", { error });+ document.warning =+ "Comparing failed, please try again later." ++ (document.warning ? ` ${document.warning}` : "");+ return document;+ }const end = Date.now();if (end - start > 100) {meta.logger.debug("Diffing took a while", {@@ -111,7 +120,7 @@ export async function deriveDiff(o_date_added: string;}| undefined- | null = (res.data ?? [])[0] as any;+ | null = resData[0];const rawJob = data?.o_job_id ? await getJobFromGCS(data.o_job_id) : null;const job: Document | null = rawJob?.[0] ?? null;@@ -265,7 +274,7 @@ export async function deriveDiff((document.warning ? ` ${document.warning}` : "");}}- } else if (!res.error) {+ } else {document.changeTracking = {previousScrapeAt: null,changeStatus: document.metadata.statusCode === 404 ? "removed" : "new",@@ -273,11 +282,6 @@ export async function deriveDiff(? "hidden": "visible",};- } else {- meta.logger.error("Error fetching previous scrape", { error: res.error });- document.warning =- "Comparing failed, please try again later." +- (document.warning ? ` ${document.warning}` : "");}}diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 1cc071ccd..6190778a0 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -11,7 +11,6 @@ import {performCleanContent,} from "./llmExtract";import { performQuery } from "./query";-import { uploadScreenshot } from "./uploadScreenshot";import { removeBase64Images } from "./removeBase64Images";import { performAgent } from "./agent";import { performAttributes } from "./performAttributes";@@ -574,7 +573,6 @@ const transformerStack: Transformer[] = [deriveImagesFromHTML,deriveBrandingFromActions,deriveMetadataFromRawHTML,- uploadScreenshot,...(useIndex ? [sendDocumentToIndex] : []),...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time searchperformLLMExtract,diff --git a/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts b/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.tsdeleted file mode 100644index 55cb5c9f8..000000000--- a/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts+++ /dev/null@@ -1,35 +0,0 @@-// This file is an exception to the "no supabase in scrapeURL" rule,-// and it makes me sad. - mogery--import { supabase_service } from "../../../services/supabase";-import { config } from "../../../config";-import { Meta } from "..";-import { Document } from "../../../controllers/v1/types";--export function uploadScreenshot(meta: Meta, document: Document): Document {- if (- config.USE_DB_AUTHENTICATION &&- document.screenshot !== undefined &&- document.screenshot.startsWith("data:")- ) {- meta.logger.debug("Uploading screenshot to Supabase...");-- const fileName = `screenshot-${crypto.randomUUID()}.png`;-- supabase_service.storage- .from("media")- .upload(- fileName,- Buffer.from(document.screenshot.split(",")[1], "base64"),- {- cacheControl: "3600",- upsert: false,- contentType: document.screenshot.split(":")[1].split(";")[0],- },- );-- document.screenshot = `https://service.firecrawl.dev/storage/v1/object/public/media/${encodeURIComponent(fileName)}`;- }-- return document;-}diff --git a/apps/api/src/services/agent-sponsor.ts b/apps/api/src/services/agent-sponsor.tsindex 14379e9b4..8aae62fb3 100644--- a/apps/api/src/services/agent-sponsor.ts+++ b/apps/api/src/services/agent-sponsor.ts@@ -1,10 +1,8 @@+import { eq } from "drizzle-orm";import { logger } from "../lib/logger";import { deleteKey, getValue, setValue } from "./redis";-import {- isPostgrestNoRowsError,- supabase_rr_service,- supabase_service,-} from "./supabase";+import { db, dbRr } from "../db/connection";+import * as schema from "../db/schema";type AgentSponsorStatus = {status: "pending" | "verified" | "blocked";@@ -42,37 +40,30 @@ export async function getAgentSponsorStatus({}try {- const { data, error } = await supabase_rr_service- .from("agent_sponsors")- .select("status, verification_deadline, email")- .eq("api_key_id", apiKeyId)- .single();+ const [data] = await dbRr+ .select({+ status: schema.agent_sponsors.status,+ verification_deadline: schema.agent_sponsors.verification_deadline,+ email: schema.agent_sponsors.email,+ })+ .from(schema.agent_sponsors)+ .where(eq(schema.agent_sponsors.api_key_id, apiKeyId))+ .limit(1);- if (error) {- // Only cache "no sponsor" when it's a confirmed no-rows result.- // Do not cache on other errors (e.g. connection/timeout) so we retry on next request.- if (isPostgrestNoRowsError(error)) {- await setValue(- cacheKey,- JSON.stringify(CACHE_MISS_SENTINEL),- AGENT_SPONSOR_CACHE_TTL,- );- } else {- logger.error("Failed to look up agent sponsor status", {- apiKeyId,- error,- });- }- return null;- }if (!data) {+ // Confirmed no-rows result — cache the "no sponsor" sentinel.+ await setValue(+ cacheKey,+ JSON.stringify(CACHE_MISS_SENTINEL),+ AGENT_SPONSOR_CACHE_TTL,+ );return null;}const result: AgentSponsorStatus = {- status: data.status,- verification_deadline: data.verification_deadline,- email: data.email,+ status: data.status as AgentSponsorStatus["status"],+ verification_deadline: data.verification_deadline!,+ email: data.email!,};await setValue(cacheKey, JSON.stringify(result), AGENT_SPONSOR_CACHE_TTL);@@ -110,22 +101,28 @@ async function getAgentSponsorByToken({status: string;verification_deadline: string;agent_name: string;- sandboxed_team_id: string;- api_key_id: number;+ sandboxed_team_id: string | null;+ api_key_id: number | null;} | null> {- const { data, error } = await supabase_service- .from("agent_sponsors")- .select(- "id, email, status, verification_deadline, agent_name, sandboxed_team_id, api_key_id",- )- .eq("verification_token", agent_signup_token)- .single();-- if (error || !data) {+ try {+ const [data] = await db+ .select({+ id: schema.agent_sponsors.id,+ email: schema.agent_sponsors.email,+ status: schema.agent_sponsors.status,+ verification_deadline: schema.agent_sponsors.verification_deadline,+ agent_name: schema.agent_sponsors.agent_name,+ sandboxed_team_id: schema.agent_sponsors.sandboxed_team_id,+ api_key_id: schema.agent_sponsors.api_key_id,+ })+ .from(schema.agent_sponsors)+ .where(eq(schema.agent_sponsors.verification_token, agent_signup_token))+ .limit(1);++ return data ?? null;+ } catch {return null;}-- return data;}/**@@ -136,12 +133,12 @@ async function markSponsorVerified({}: {sponsorId: number;}): Promise<void> {- const { error } = await supabase_service- .from("agent_sponsors")- .update({ status: "verified", verified_at: new Date().toISOString() })- .eq("id", sponsorId);-- if (error) {+ try {+ await db+ .update(schema.agent_sponsors)+ .set({ status: "verified", verified_at: new Date().toISOString() })+ .where(eq(schema.agent_sponsors.id, sponsorId));+ } catch (error) {logger.error("Failed to mark sponsor as verified", { sponsorId, error });throw error;}@@ -155,12 +152,12 @@ async function markSponsorBlocked({}: {sponsorId: number;}): Promise<void> {- const { error } = await supabase_service- .from("agent_sponsors")- .update({ status: "blocked" })- .eq("id", sponsorId);-- if (error) {+ try {+ await db+ .update(schema.agent_sponsors)+ .set({ status: "blocked" })+ .where(eq(schema.agent_sponsors.id, sponsorId));+ } catch (error) {logger.error("Failed to mark sponsor as blocked", { sponsorId, error });throw error;}diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex 27a343cdf..87d52316c 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -3,7 +3,7 @@** All external I/O is mocked:* - autumnClient → jest.fn() stubs on customers / entities / track- * - supabase_rr_service → stubbed Supabase query builder+ * - dbRr → stubbed Drizzle query builder*/import { jest } from "@jest/globals";@@ -44,15 +44,13 @@ jest.mock("../client", () => ({},}));-// Minimal Supabase query-builder stub: .from().select().eq().single() → resolves data/error.-const makeSupabaseStub = (data: unknown, error: unknown = null) => ({- from: () => ({- select: () => ({- eq: () => ({- single: () => Promise.resolve({ data, error }),- gte: () => Promise.resolve({ data: [], error: null }),+// Minimal Drizzle query-builder stub: .select().from().where().limit() → rows.+const makeDbStub = (data: unknown) => ({+ select: () => ({+ from: () => ({+ where: () => ({+ limit: () => Promise.resolve(data ? [data] : []),}),- gte: () => Promise.resolve({ data: [], error: null }),}),}),});@@ -62,9 +60,9 @@ let supabaseStubData: { data: unknown; error: unknown } = {error: null,};-jest.mock("../../supabase", () => ({- get supabase_rr_service() {- return makeSupabaseStub(supabaseStubData.data, supabaseStubData.error);+jest.mock("../../../db/connection", () => ({+ get dbRr() {+ return makeDbStub(supabaseStubData.data);},}));diff --git a/apps/api/src/services/autumn/__tests__/usage.test.ts b/apps/api/src/services/autumn/__tests__/usage.test.tsindex 54900d13c..857dd578a 100644--- a/apps/api/src/services/autumn/__tests__/usage.test.ts+++ b/apps/api/src/services/autumn/__tests__/usage.test.ts@@ -27,27 +27,20 @@ jest.mock("../client", () => ({},}));-jest.mock("../../supabase", () => ({- get supabase_rr_service() {+jest.mock("../../../db/connection", () => ({+ get dbRr() {return {- from: (table: string) => ({- select: () => {- if (table === "teams") {- return {- eq: () => ({- single: () => Promise.resolve(teamLookup),- }),- };- }-- if (table === "api_keys") {- return {- in: () => Promise.resolve({ data: apiKeysData, error: null }),- };- }-- return {};- },+ select: () => ({+ from: () => ({+ where: () => {+ // api_keys path awaits the builder directly; teams path calls .limit(1)+ const apiKeysPromise = Promise.resolve(apiKeysData);+ return Object.assign(apiKeysPromise, {+ limit: () =>+ Promise.resolve(teamLookup.data ? [teamLookup.data] : []),+ });+ },+ }),}),};},diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex d9e2cfb9e..1aa9ce998 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -1,7 +1,9 @@import { randomUUID } from "crypto";import { config } from "../../config";import { logger } from "../../lib/logger";-import { supabase_rr_service } from "../supabase";+import { eq } from "drizzle-orm";+import { dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { autumnClient } from "./client";import type {CreateEntityParams,@@ -87,13 +89,12 @@ export class AutumnService {}private async lookupOrgIdForTeam(teamId: string): Promise<string> {- const { data, error } = await supabase_rr_service- .from("teams")- .select("org_id")- .eq("id", teamId)- .single();+ const [data] = await dbRr+ .select({ org_id: schema.teams.org_id })+ .from(schema.teams)+ .where(eq(schema.teams.id, teamId))+ .limit(1);- if (error) throw error;if (!data?.org_id) {throw new Error(`Missing org_id for team ${teamId}`);}diff --git a/apps/api/src/services/autumn/usage.ts b/apps/api/src/services/autumn/usage.tsindex ad68d7d97..c67ccc58b 100644--- a/apps/api/src/services/autumn/usage.ts+++ b/apps/api/src/services/autumn/usage.ts@@ -1,5 +1,7 @@import { logger } from "../../lib/logger";-import { supabase_rr_service } from "../supabase";+import { eq, inArray } from "drizzle-orm";+import { dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { autumnClient } from "./client";const CREDITS_FEATURE_ID = "CREDITS";@@ -26,13 +28,12 @@ interface TeamBalance {// ---------------------------------------------------------------------------async function lookupOrgId(teamId: string): Promise<string> {- const { data, error } = await supabase_rr_service- .from("teams")- .select("org_id")- .eq("id", teamId)- .single();+ const [data] = await dbRr+ .select({ org_id: schema.teams.org_id })+ .from(schema.teams)+ .where(eq(schema.teams.id, teamId))+ .limit(1);- if (error) throw error;if (!data?.org_id) {throw new Error(`Missing org_id for team ${teamId}`);}@@ -53,15 +54,13 @@ async function lookupApiKeyNames(const nameMap: Record<string, string> = {};if (numericIds.length > 0) {- const { data } = await supabase_rr_service- .from("api_keys")- .select("id, name")- .in("id", numericIds);-- if (data) {- for (const row of data) {- nameMap[String(row.id)] = row.name;- }+ const data = await dbRr+ .select({ id: schema.api_keys.id, name: schema.api_keys.name })+ .from(schema.api_keys)+ .where(inArray(schema.api_keys.id, numericIds));++ for (const row of data) {+ nameMap[String(row.id)] = row.name ?? String(row.id);}}diff --git a/apps/api/src/services/billing/__tests__/batch_billing.test.ts b/apps/api/src/services/billing/__tests__/batch_billing.test.tsindex ce4fbb35b..3f66d1c26 100644--- a/apps/api/src/services/billing/__tests__/batch_billing.test.ts+++ b/apps/api/src/services/billing/__tests__/batch_billing.test.ts@@ -29,11 +29,9 @@ jest.mock("../../autumn/autumn.service", () => ({},}));-const rpc = jest.fn<(name: string, args: any) => Promise<any>>();-jest.mock("../../supabase", () => ({- supabase_service: {- rpc,- },+const billTeam6 = jest.fn<(params: any) => Promise<{ api_key: string }[]>>();+jest.mock("../../../db/rpc", () => ({+ billTeam6,}));const setCachedACUC = jest.fn();@@ -126,7 +124,7 @@ beforeEach(() => {queue = [];billedTeams.clear();locks.clear();- rpc.mockResolvedValue({ data: [], error: null });+ billTeam6.mockResolvedValue([]);trackCredits.mockResolvedValue(true);refundCredits.mockResolvedValue(undefined);});@@ -137,7 +135,7 @@ describe("processBillingBatch", () => {await processBillingBatch();- expect(rpc).toHaveBeenCalled();+ expect(billTeam6).toHaveBeenCalled();expect(trackCredits).toHaveBeenCalledWith({teamId: "team-1",value: 10,@@ -156,13 +154,13 @@ describe("processBillingBatch", () => {await processBillingBatch();- expect(rpc).toHaveBeenCalled();+ expect(billTeam6).toHaveBeenCalled();expect(trackCredits).not.toHaveBeenCalled();});it("continues when billing returns success false", async () => {queue = [makeOp({ autumnTrackInRequest: true })];- rpc.mockResolvedValueOnce({ data: null, error: new Error("db failed") });+ billTeam6.mockRejectedValueOnce(new Error("db failed"));await processBillingBatch();@@ -181,7 +179,7 @@ describe("processBillingBatch", () => {it("captures exceptions when billing throws", async () => {queue = [makeOp({ autumnTrackInRequest: true })];- rpc.mockRejectedValueOnce(new Error("rpc exploded"));+ billTeam6.mockRejectedValueOnce(new Error("rpc exploded"));await processBillingBatch();@@ -211,9 +209,9 @@ describe("processBillingBatch", () => {autumnTrackInRequest: false,}),];- rpc- .mockResolvedValueOnce({ data: null, error: new Error("db failed") })- .mockResolvedValueOnce({ data: [], error: null });+ billTeam6+ .mockRejectedValueOnce(new Error("db failed"))+ .mockResolvedValueOnce([]);refundCredits.mockRejectedValueOnce(new Error("refund failed"));await processBillingBatch();@@ -228,7 +226,7 @@ describe("processBillingBatch", () => {subscriptionId: "sub-1",},});- expect(rpc).toHaveBeenCalledTimes(2);+ expect(billTeam6).toHaveBeenCalledTimes(2);expect(trackCredits).toHaveBeenCalledWith({teamId: "team-2",value: 10,diff --git a/apps/api/src/services/billing/__tests__/credit_billing.test.ts b/apps/api/src/services/billing/__tests__/credit_billing.test.tsindex beb3d3606..be65d5efb 100644--- a/apps/api/src/services/billing/__tests__/credit_billing.test.ts+++ b/apps/api/src/services/billing/__tests__/credit_billing.test.ts@@ -22,10 +22,6 @@ jest.mock("../../autumn/autumn.service", () => ({jest.mock("../../notification/email_notification", () => ({sendNotification: jest.fn(),}));-jest.mock("../../supabase", () => ({- supabase_rr_service: {},- supabase_service: {},-}));jest.mock("../auto_charge", () => ({autoCharge: jest.fn(),}));diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.tsdeleted file mode 100644index 35633a254..000000000--- a/apps/api/src/services/billing/auto_charge.ts+++ /dev/null@@ -1,667 +0,0 @@-// Import necessary dependencies and types-import { AuthCreditUsageChunk } from "../../controllers/v1/types";-import { config } from "../../config";-import { clearACUC, clearACUCTeam, getACUC } from "../../controllers/auth";-import { redlock } from "../redlock";-import { supabase_rr_service, supabase_service } from "../supabase";-import {- createPaymentIntent,- createSubscription,- customerToUserId,-} from "./stripe";-import { issueCredits } from "./issue_credits";-import {- sendNotification,- sendNotificationWithCustomDays,-} from "../notification/email_notification";-import { NotificationType } from "../../types";-import { deleteKey, getValue, redisEvictConnection, setValue } from "../redis";-import { redisRateLimitClient } from "../rate-limiter";-import { sendSlackWebhook } from "../alerts/slack";-import { logger as _logger } from "../../lib/logger";--// Define the number of credits to be added during auto-recharge-const AUTO_RECHARGE_CREDITS = 1000;-const AUTO_RECHARGE_COOLDOWN = 600; // 10 minutes in seconds-const MAX_CHARGES_PER_HOUR = 5; // Maximum number of auto-charges per hour-const HOURLY_COUNTER_EXPIRY = 3600; // 1 hour in seconds--// Type definition for auto-charge response-type AutoChargeResponse = {- success: boolean;- message: string;- remainingCredits: number;- chunk: AuthCreditUsageChunk;-};--/**- * Attempt to automatically charge a user's account when their credit balance falls below a threshold- * @param chunk The user's current usage data- * @param autoRechargeThreshold The credit threshold that triggers auto-recharge- */-export async function autoCharge(- chunk: AuthCreditUsageChunk,- autoRechargeThreshold: number,-): Promise<AutoChargeResponse> {- if (chunk.price_associated_auto_recharge_price_id !== null) {- return _autoChargeScale(- chunk as AuthCreditUsageChunk & {- price_associated_auto_recharge_price_id: string;- },- autoRechargeThreshold,- );- } else {- return _autoChargeSelfServe(chunk, autoRechargeThreshold);- }-}--async function _autoChargeScale(- chunk: AuthCreditUsageChunk & {- price_associated_auto_recharge_price_id: string;- },- autoRechargeThreshold: number,-): Promise<AutoChargeResponse> {- const logger = _logger.child({- module: "auto_charge",- method: "_autoChargeScale",- team_id: chunk.team_id,- teamId: chunk.team_id,- });-- logger.info("Scale auto-recharge triggered", {});-- const resource = `auto-recharge:${chunk.team_id}`;-- try {- return await redlock.using(- [resource],- 15000,- async (signal: unknown): Promise<AutoChargeResponse> => {- logger.info("Lock acquired");-- const cooldownCheck = await redisEvictConnection.set(- "auto-recharge-cooldown:" + chunk.team_id,- "cooling",- "EX",- AUTO_RECHARGE_COOLDOWN,- "NX",- );-- if (cooldownCheck === null) {- logger.warn("Auto-recharge is on cooldown, aborting.");- return {- success: false,- message: "Auto-recharge is on cooldown",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-- const updatedChunk = await getACUC(chunk.api_key, false, false);-- if (- updatedChunk &&- updatedChunk.remaining_credits < autoRechargeThreshold- ) {- const { data: price, error: priceError } = await supabase_service- .from("prices")- .select("*")- .eq("id", chunk.price_associated_auto_recharge_price_id)- .single();- if (priceError || !price) {- logger.error("Error fetching price", {- error: priceError,- priceId:- chunk.price_associated_auto_recharge_price_id === undefined- ? "undefined"- : JSON.stringify(- chunk.price_associated_auto_recharge_price_id,- ),- });- return {- success: false,- message: "Error fetching price",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }-- const invoiceOnly = Boolean(- (price.metadata as { invoice?: boolean } | null | undefined)- ?.invoice,- );-- // Check for recharges this month-- const currentMonth = new Date();- currentMonth.setUTCDate(1);- currentMonth.setUTCHours(0, 0, 0, 0);-- const { data: rechargesThisMonth, error: rechargesThisMonthError } =- await supabase_service- .from("subscriptions")- .select("*, prices!inner(*)")- .eq("team_id", chunk.team_id)- .eq("prices.metadata->>auto_recharge", "true")- .gte("current_period_start", currentMonth.toISOString());-- if (rechargesThisMonthError || !rechargesThisMonth) {- logger.error("Error fetching recharges this month", {- error: rechargesThisMonthError,- });- return {- success: false,- message: "Error fetching recharges this month",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- } else if (- rechargesThisMonth.length >= (price.exp_pack_max_per_month ?? 4)- ) {- logger.warn("Auto-recharge failed: too many recharges this month");- return {- success: false,- message: "Auto-recharge failed: too many recharges this month",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- } else {- // Actually re-charge-- if (!chunk.sub_user_id) {- logger.error("No sub_user_id found in chunk");- return {- success: false,- message: "No sub_user_id found in chunk",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }-- const { data: customer, error: customersError } =- await supabase_rr_service- .from("customers")- .select("id, stripe_customer_id")- .eq("id", chunk.sub_user_id)- .single();-- if (customersError || !customer) {- logger.error("Error fetching customer data", {- error: customersError,- });- return {- success: false,- message: "Error fetching customer data",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }-- if (!customer.stripe_customer_id) {- logger.error("No stripe_customer_id found in customer");- return {- success: false,- message: "No stripe_customer_id found in customer",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }-- if (!chunk.sub_id) {- logger.error("No sub_id found in chunk");- return {- success: false,- message: "No sub_id found in chunk",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }-- const subscription = await createSubscription(- chunk.team_id,- customer.stripe_customer_id,- chunk.price_associated_auto_recharge_price_id,- chunk.sub_id,- invoiceOnly,- );- if (!subscription) {- logger.error("Failed to create subscription");- return {- success: false,- message: "Failed to create subscription",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }-- const userId = await customerToUserId(customer.stripe_customer_id);- if (!userId) {- logger.error("Failed to get user id from customer");- return {- success: false,- message: "Failed to get user id from customer",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }-- // Try to insert it into subscriptions ourselves in case webhook is slow- const { error: subscriptionError } = await supabase_service- .from("subscriptions")- .insert({- id: subscription.id,- user_id: userId,- metadata: subscription.metadata,- status: subscription.status,- price_id: chunk.price_associated_auto_recharge_price_id,- quantity: 1,- cancel_at_period_end: false,- cancel_at: null,- canceled_at: null,- current_period_start: subscription.current_period_start- ? new Date(- subscription.current_period_start * 1000,- ).toISOString()- : null,- current_period_end: subscription.current_period_end- ? new Date(- subscription.current_period_end * 1000,- ).toISOString()- : null,- created: subscription.created- ? new Date(subscription.created * 1000).toISOString()- : null,- ended_at: null,- trial_start: null,- trial_end: null,- team_id: chunk.team_id,- is_extract: false,- });-- if (subscriptionError) {- logger.warn(- "Failed to add subscription to supabase -- maybe we got sniped by the webhook?",- { error: subscriptionError },- );- }-- // Reset ACUC cache to reflect the new credit balance- await clearACUC(chunk.api_key);- await clearACUCTeam(chunk.team_id);-- try {- // Check for frequent auto-recharges in the past week- const weeklyAutoRechargeKey = `auto-recharge-weekly:${chunk.team_id}`;- const weeklyRecharges = await redisRateLimitClient.incr(- weeklyAutoRechargeKey,- );- // Set expiry for 7 days if not already set- await redisRateLimitClient.expire(- weeklyAutoRechargeKey,- 7 * 24 * 60 * 60,- );-- // If this is the second auto-recharge in a week, send notification- if (weeklyRecharges >= 2) {- await sendNotificationWithCustomDays(- chunk.team_id,- NotificationType.AUTO_RECHARGE_FREQUENT,- 7, // Send at most once per week- false,- );- }- } catch (error) {- logger.error(- `Error sending frequent auto-recharge notification`,- {- error,- },- );- }-- await sendNotification(- chunk.team_id,- NotificationType.AUTO_RECHARGE_SUCCESS,- chunk.sub_current_period_start,- chunk.sub_current_period_end,- chunk,- true,- false,- { autoRechargeCredits: price.credits },- );-- logger.info("Scale auto-recharge successful");-- if (config.SLACK_ADMIN_WEBHOOK_URL) {- sendSlackWebhook(- `💰 Auto-recharge successful on team ${chunk.team_id} for ${price.credits} credits (total auto-recharges this month: ${rechargesThisMonth.length + 1}).`,- false,- config.SLACK_ADMIN_WEBHOOK_URL,- ).catch(error => {- logger.debug(`Error sending slack notification: ${error}`);- });- }-- return {- success: true,- message: "Auto-recharge successful",- remainingCredits:- (updatedChunk?.remaining_credits ?? chunk.remaining_credits) +- price.credits,- chunk: {- ...(updatedChunk ?? chunk),- remaining_credits:- (updatedChunk?.remaining_credits ?? chunk.remaining_credits) +- price.credits,- },- };- }- } else {- return {- success: false,- message: "No need to auto-recharge",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }- },- );- } catch (error) {- logger.error("Auto-recharge failed", { error });- return {- success: false,- message: "Failed to acquire lock for auto-recharge",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-}--async function _autoChargeSelfServe(- chunk: AuthCreditUsageChunk,- autoRechargeThreshold: number,-): Promise<AutoChargeResponse> {- const logger = _logger.child({- module: "auto_charge",- method: "_autoChargeSelfServe",- team_id: chunk.team_id,- teamId: chunk.team_id,- });-- const resource = `auto-recharge:${chunk.team_id}`;- const cooldownKey = `auto-recharge-cooldown:${chunk.team_id}`;- const hourlyCounterKey = `auto-recharge-hourly:${chunk.team_id}`;-- if (- chunk.team_id === "285bb597-6eaf-4b96-801c-51461fc3c543" ||- chunk.team_id === "dec639a0-98ca-4995-95b5-48ac1ffab5b7"- ) {- return {- success: false,- message: "Auto-recharge failed: blocked team",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-- try {- // Check hourly rate limit first without incrementing- const currentCharges = await redisRateLimitClient.get(hourlyCounterKey);- const hourlyCharges = currentCharges ? parseInt(currentCharges) : 0;-- if (hourlyCharges >= MAX_CHARGES_PER_HOUR) {- logger.warn(- `Auto-recharge exceeded hourly limit of ${MAX_CHARGES_PER_HOUR}`,- );- return {- success: false,- message: "Auto-recharge hourly limit exceeded",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-- // Check cooldown period- const cooldownValue = await getValue(cooldownKey);- if (cooldownValue) {- logger.info(`Auto-recharge is in cooldown period`);- return {- success: false,- message: "Auto-recharge is in cooldown period",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-- // Use a distributed lock to prevent concurrent auto-charge attempts- return await redlock.using(- [resource],- 5000,- async (signal: unknown): Promise<AutoChargeResponse> => {- // Recheck all conditions inside the lock to prevent race conditions- const updatedChunk = await getACUC(chunk.api_key, false, false);-- // Recheck cooldown- const cooldownValue = await getValue(cooldownKey);- if (cooldownValue) {- logger.info(`Auto-recharge is in cooldown period`);- return {- success: false,- message: "Auto-recharge is in cooldown period",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-- // Recheck hourly limit inside lock- const currentCharges = await redisRateLimitClient.get(hourlyCounterKey);- const hourlyCharges = currentCharges ? parseInt(currentCharges) : 0;- if (hourlyCharges >= MAX_CHARGES_PER_HOUR) {- return {- success: false,- message: "Auto-recharge hourly limit exceeded",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-- if (- updatedChunk &&- updatedChunk.remaining_credits < autoRechargeThreshold- ) {- if (chunk.sub_user_id) {- // Fetch the customer's Stripe information- const { data: customer, error: customersError } =- await supabase_rr_service- .from("customers")- .select("id, stripe_customer_id")- .eq("id", chunk.sub_user_id)- .single();-- if (customersError) {- logger.error(`Error fetching customer data`, {- error: customersError,- });- return {- success: false,- message: "Error fetching customer data",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-- if (customer && customer.stripe_customer_id) {- let issueCreditsSuccess = false;-- // Set cooldown BEFORE attempting payment- await setValue(cooldownKey, "true", AUTO_RECHARGE_COOLDOWN);-- // Attempt to create a payment intent- const paymentStatus = await createPaymentIntent(- chunk.team_id,- customer.stripe_customer_id,- );-- // If payment is successful or requires further action, issue credits- if (- paymentStatus.return_status === "succeeded" ||- paymentStatus.return_status === "requires_action"- ) {- issueCreditsSuccess = await issueCredits(- chunk.team_id,- AUTO_RECHARGE_CREDITS,- );- }-- // Record the auto-recharge transaction- await supabase_service.from("auto_recharge_transactions").insert({- team_id: chunk.team_id,- initial_payment_status: paymentStatus.return_status,- credits_issued: issueCreditsSuccess ? AUTO_RECHARGE_CREDITS : 0,- stripe_charge_id: paymentStatus.charge_id,- });-- // Send a notification if credits were successfully issued- if (issueCreditsSuccess) {- // Increment hourly counter and set expiry if it doesn't exist- await redisRateLimitClient.incr(hourlyCounterKey);- await redisRateLimitClient.expire(- hourlyCounterKey,- HOURLY_COUNTER_EXPIRY,- "NX",- );-- try {- // Check for frequent auto-recharges in the past week- const weeklyAutoRechargeKey = `auto-recharge-weekly:${chunk.team_id}`;- const weeklyRecharges = await redisRateLimitClient.incr(- weeklyAutoRechargeKey,- );- // Set expiry for 7 days if not already set- await redisRateLimitClient.expire(- weeklyAutoRechargeKey,- 7 * 24 * 60 * 60,- );-- // If this is the second auto-recharge in a week, send notification- if (weeklyRecharges >= 2) {- await sendNotificationWithCustomDays(- chunk.team_id,- NotificationType.AUTO_RECHARGE_FREQUENT,- 7, // Send at most once per week- false,- );- }- } catch (error) {- logger.error(- `Error sending frequent auto-recharge notification`,- { error },- );- }-- await sendNotification(- chunk.team_id,- NotificationType.AUTO_RECHARGE_SUCCESS,- chunk.sub_current_period_start,- chunk.sub_current_period_end,- chunk,- true,- false,- { autoRechargeCredits: AUTO_RECHARGE_CREDITS },- );-- // Reset ACUC cache to reflect the new credit balance- await clearACUC(chunk.api_key);- await clearACUCTeam(chunk.team_id);-- logger.info(`Auto-recharge successful`, {- credits: AUTO_RECHARGE_CREDITS,- paymentStatus: paymentStatus.return_status,- });-- if (config.SLACK_ADMIN_WEBHOOK_URL) {- const webhookCooldownKey = `webhook_cooldown_${chunk.team_id}`;- const isInCooldown = await getValue(webhookCooldownKey);-- if (!isInCooldown) {- sendSlackWebhook(- `Auto-recharge: Team ${chunk.team_id}. ${AUTO_RECHARGE_CREDITS} credits added. Payment status: ${paymentStatus.return_status}.`,- false,- config.SLACK_ADMIN_WEBHOOK_URL,- ).catch(error => {- logger.debug(- `Error sending slack notification: ${error}`,- );- });-- // Set cooldown for 1 hour- await setValue(webhookCooldownKey, "true", 60 * 60);- }- }- return {- success: true,- message: "Auto-recharge successful",- remainingCredits:- chunk.remaining_credits + AUTO_RECHARGE_CREDITS,- chunk: {- ...chunk,- remaining_credits:- chunk.remaining_credits + AUTO_RECHARGE_CREDITS,- },- };- } else {- logger.error("No Stripe customer ID found for user");- return {- success: false,- message: "No Stripe customer ID found for user",- remainingCredits: chunk.remaining_credits,- chunk,- };- }- } else {- logger.error("No Stripe customer ID found for user");- return {- success: false,- message: "No Stripe customer ID found for user",- remainingCredits: chunk.remaining_credits,- chunk,- };- }- } else {- logger.error("No sub_user_id found in chunk");- return {- success: false,- message: "No sub_user_id found in chunk",- remainingCredits: chunk.remaining_credits,- chunk,- };- }- }- return {- success: false,- message: "No need to auto-recharge",- remainingCredits: chunk.remaining_credits,- chunk,- };- },- );- } catch (error) {- logger.error(`Failed to acquire lock for auto-recharge`, { error });- return {- success: false,- message: "Failed to acquire lock for auto-recharge",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-}diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex 00005c2bd..738ce56f6 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -1,7 +1,7 @@import { logger } from "../../lib/logger";import { config } from "../../config";import { getRedisConnection } from "../queue-service";-import { supabase_service } from "../supabase";+import { billTeam6 } from "../../db/rpc";import * as Sentry from "@sentry/node";import { withAuth } from "../../lib/withAuth";import { setCachedACUC, setCachedACUCTeam } from "../../controllers/auth";@@ -223,7 +223,6 @@ export async function processBillingBatch() {},});}-} catch (error) {await refundRequestTrackedCredits(group);logger.error(`❌ Failed to bill team ${group.team_id}`, {@@ -402,16 +401,17 @@ async function supaBillTeam(_logger.info(`Batch billing team ${team_id} for ${credits} credits`);// Perform the actual database operation- const { data, error } = await supabase_service.rpc("bill_team_6", {- _team_id: team_id,- sub_id: subscription_id ?? null,- fetch_subscription: subscription_id === undefined,- credits,- i_api_key_id: api_key_id ?? null,- is_extract_param: is_extract,- });-- if (error) {+ let data: { api_key: string }[];+ try {+ data = await billTeam6({+ team_id,+ subscription_id: subscription_id ?? null,+ fetch_subscription: subscription_id === undefined,+ credits,+ api_key_id: api_key_id ?? null,+ is_extract,+ });+ } catch (error) {Sentry.captureException(error);_logger.error("Failed to bill team.", { error });return { success: false, error };diff --git a/apps/api/src/services/billing/issue_credits.ts b/apps/api/src/services/billing/issue_credits.tsdeleted file mode 100644index 2ca057dda..000000000--- a/apps/api/src/services/billing/issue_credits.ts+++ /dev/null@@ -1,21 +0,0 @@-import { logger } from "../../lib/logger";-import { supabase_service } from "../supabase";--export async function issueCredits(team_id: string, credits: number) {- // Add an entry to supabase coupons- const { error } = await supabase_service.from("coupons").insert({- team_id: team_id,- credits: credits,- status: "active",- // indicates that this coupon was issued from auto recharge- from_auto_recharge: true,- initial_credits: credits,- });-- if (error) {- logger.error(`Error adding coupon: ${error}`);- return false;- }-- return true;-}diff --git a/apps/api/src/services/billing/stripe.ts b/apps/api/src/services/billing/stripe.tsdeleted file mode 100644index d74838f29..000000000--- a/apps/api/src/services/billing/stripe.ts+++ /dev/null@@ -1,190 +0,0 @@-import { logger } from "../../lib/logger";-import { config } from "../../config";-import Stripe from "stripe";--const stripe = new Stripe(config.STRIPE_SECRET_KEY ?? "");--async function getCustomerDefaultPaymentMethod(customerId: string) {- const paymentMethods = await stripe.customers.listPaymentMethods(customerId, {- limit: 3,- });- return paymentMethods.data[0] ?? null;-}--type ReturnStatus = "succeeded" | "requires_action" | "failed";-export async function createPaymentIntent(- team_id: string,- customer_id: string,-): Promise<{ return_status: ReturnStatus; charge_id: string }> {- try {- const defaultPaymentMethod =- await getCustomerDefaultPaymentMethod(customer_id);- if (!defaultPaymentMethod) {- logger.error(- `No default payment method found for customer: ${customer_id}`,- { team_id },- );- return { return_status: "failed", charge_id: "" };- }- const paymentIntent = await stripe.paymentIntents.create({- amount: 1100,- currency: "usd",- customer: customer_id,- description: "Firecrawl: Auto re-charge of 1000 credits",- payment_method_types: [defaultPaymentMethod?.type ?? "card"],- payment_method: defaultPaymentMethod?.id,- off_session: true,- confirm: true,- metadata: {- team_id,- auto_recharge: "true",- },- });-- if (paymentIntent.status === "succeeded") {- logger.info(`Payment succeeded for team: ${team_id}`);- return { return_status: "succeeded", charge_id: paymentIntent.id };- } else if (- paymentIntent.status === "requires_action" ||- paymentIntent.status === "processing" ||- paymentIntent.status === "requires_capture"- ) {- logger.warn(`Payment requires further action for team: ${team_id}`);- return { return_status: "requires_action", charge_id: paymentIntent.id };- } else {- logger.error(`Payment failed for team: ${team_id}`);- return { return_status: "failed", charge_id: paymentIntent.id };- }- } catch (error) {- logger.error(- `Failed to create or confirm PaymentIntent for team: ${team_id}`,- );- console.error(error);- return { return_status: "failed", charge_id: "" };- }-}--export async function createSubscription(- team_id: string,- customer_id: string,- price_id: string,- main_subscription_id: string,- invoiceOnly = false,-) {- const defaultPaymentMethod = invoiceOnly- ? null- : await getCustomerDefaultPaymentMethod(customer_id);- if (!invoiceOnly && !defaultPaymentMethod) {- logger.error("No default payment method found for customer", {- team_id,- customer_id,- });- return null;- }-- let useCoupons: string[] = [];-- const mainSub = await stripe.subscriptions.retrieve(main_subscription_id, {- expand: ["discounts"],- });- if (mainSub) {- // Clone coupons from main subscription to auto recharge expansion- useCoupons = (- await Promise.all(- ((mainSub.discounts ?? []) as Stripe.Discount[]).map(async discount => {- if (discount.coupon.duration === "once") {- return null;- }-- if (- discount.coupon.duration === "repeating" &&- new Date(discount.end! * 1000) < new Date()- ) {- return null;- }-- if (!discount.coupon.valid) {- const duration_in_months =- discount.coupon.duration === "repeating"- ? Math.round(- (new Date(discount.end! * 1000).valueOf() -- new Date(discount.start! * 1000).valueOf()) /- (1000 * 60 * 60 * 24 * 30),- )- : 0;- const coupon = await stripe.coupons.create({- name: discount.coupon.name ?? undefined,- duration: duration_in_months > 0 ? "repeating" : "once",- ...(duration_in_months > 0 ? { duration_in_months } : {}),- max_redemptions: 1,- ...(discount.coupon.amount_off- ? { amount_off: discount.coupon.amount_off }- : {}),- ...(discount.coupon.currency- ? { currency: discount.coupon.currency }- : {}),- ...(discount.coupon.currency_options- ? { currency_options: discount.coupon.currency_options }- : {}),- metadata: {- ...(discount.coupon.metadata ?? {}),- team_id,- auto_recharge: "true",- },- ...(discount.coupon.percent_off- ? { percent_off: discount.coupon.percent_off }- : {}),- ...(discount.coupon.applies_to- ? { applies_to: discount.coupon.applies_to }- : {}),- });- return coupon.id;- } else {- return discount.coupon.id;- }- }),- )- ).filter(x => x !== null);- } else {- logger.warn("No main subscription found for customer", {- team_id,- customer_id,- });- }-- const subscription = await stripe.subscriptions.create({- customer: customer_id,- items: [{ price: price_id }],- ...(invoiceOnly- ? {- collection_method: "send_invoice",- days_until_due: 30,- }- : {- off_session: true,- payment_settings: {- payment_method_types: [- (defaultPaymentMethod?.type as Stripe.SubscriptionCreateParams.PaymentSettings.PaymentMethodType) ??- "card",- ],- },- default_payment_method: defaultPaymentMethod?.id,- collection_method: "charge_automatically",- }),- metadata: {- team_id,- auto_recharge: "true",- },- ...(useCoupons.length > 0- ? { discounts: useCoupons.map(coupon => ({ coupon })) }- : {}),- });-- return subscription;-}--export async function customerToUserId(customerId: string) {- const customer = await stripe.customers.retrieve(customerId);- if (!customer || customer.deleted) return null;- return customer.metadata.supabaseUUID ?? null;-}diff --git a/apps/api/src/services/idempotency/create.ts b/apps/api/src/services/idempotency/create.tsindex 8e1ede445..21b03c361 100644--- a/apps/api/src/services/idempotency/create.ts+++ b/apps/api/src/services/idempotency/create.ts@@ -1,5 +1,6 @@import { Request } from "express";-import { supabase_service } from "../supabase";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";import { logger } from "../../../src/lib/logger";export async function createIdempotencyKey(req: Request): Promise<string> {@@ -8,11 +9,9 @@ export async function createIdempotencyKey(req: Request): Promise<string> {throw new Error("No idempotency key provided in the request headers.");}- const { data, error } = await supabase_service- .from("idempotency_keys")- .insert({ key: idempotencyKey });-- if (error) {+ try {+ await db.insert(schema.idempotency_keys).values({ key: idempotencyKey });+ } catch (error) {logger.error(`Failed to create idempotency key: ${error}`);throw error;}diff --git a/apps/api/src/services/idempotency/validate.ts b/apps/api/src/services/idempotency/validate.tsindex 54ec7bd04..94749d2f4 100644--- a/apps/api/src/services/idempotency/validate.ts+++ b/apps/api/src/services/idempotency/validate.ts@@ -1,5 +1,7 @@import { Request } from "express";-import { supabase_rr_service, supabase_service } from "../supabase";+import { eq } from "drizzle-orm";+import { dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { validate as isUuid } from "uuid";import { logger } from "../../../src/lib/logger";@@ -18,12 +20,13 @@ export async function validateIdempotencyKey(req: Request): Promise<boolean> {return false;}- const { data, error } = await supabase_rr_service- .from("idempotency_keys")- .select("key")- .eq("key", idempotencyKey);-- if (error) {+ let data: { key: string }[] = [];+ try {+ data = await dbRr+ .select({ key: schema.idempotency_keys.key })+ .from(schema.idempotency_keys)+ .where(eq(schema.idempotency_keys.key, key));+ } catch (error) {logger.error(`Error validating idempotency key: ${error}`);}diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 2711afbc4..dbdab04c3 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -1,5 +1,16 @@-import { createClient, SupabaseClient } from "@supabase/supabase-js";import { logger as _logger } from "../lib/logger";+import { dbIndex } from "../db/connection";+import * as schema from "../db/schema";+import {+ insertOmceJobIfNeeded,+ queryIndexAtSplitLevel as rpcQueryIndexAtSplitLevel,+ queryIndexAtDomainSplitLevel as rpcQueryIndexAtDomainSplitLevel,+ queryOmceSignatures as rpcQueryOmceSignatures,+ queryEngpickerVerdict as rpcQueryEngpickerVerdict,+ queryIndexAtSplitLevelWithMeta as rpcQueryIndexAtSplitLevelWithMeta,+ queryIndexAtDomainSplitLevelWithMeta as rpcQueryIndexAtDomainSplitLevelWithMeta,+ queryDomainPriority as rpcQueryDomainPriority,+} from "../db/rpc";import { configDotenv } from "dotenv";import { ApiError } from "@google-cloud/storage";import crypto from "crypto";@@ -13,51 +24,6 @@ import { withSpan, setSpanAttributes } from "../lib/otel-tracer";import { config } from "../config";configDotenv();-// SupabaseService class initializes the Supabase client conditionally based on environment variables.-class IndexSupabaseService {- private client: SupabaseClient | null = null;-- constructor() {- const supabaseUrl = config.INDEX_SUPABASE_URL;- const supabaseServiceToken = config.INDEX_SUPABASE_SERVICE_TOKEN;- // Only initialize the Supabase client if both URL and Service Token are provided.- if (!supabaseUrl || !supabaseServiceToken) {- // Warn the user that Authentication is disabled by setting the client to null- _logger.warn("Index supabase client will not be initialized.");- this.client = null;- } else {- this.client = createClient(supabaseUrl, supabaseServiceToken);- }- }-- // Provides access to the initialized Supabase client, if available.- getClient(): SupabaseClient | null {- return this.client;- }-}--const serv = new IndexSupabaseService();--// Using a Proxy to handle dynamic access to the Supabase client or service methods.-// This approach ensures that if Supabase is not configured, any attempt to use it will result in a clear error.-export const index_supabase_service: SupabaseClient = new Proxy(serv, {- get: function (target, prop, receiver) {- const client = target.getClient();- // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.- if (client === null) {- return () => {- throw new Error("Index supabase client is not configured.");- };- }- // Direct access to SupabaseService properties takes precedence.- if (prop in target) {- return Reflect.get(target, prop, receiver);- }- // Otherwise, delegate access to the Supabase client.- return Reflect.get(client, prop, receiver);- },-}) as unknown as SupabaseClient;-export async function getIndexFromGCS(url: string,logger?: Logger,@@ -208,11 +174,10 @@ export async function saveIndexToGCS(}export const useIndex =- config.INDEX_SUPABASE_URL !== "" && config.INDEX_SUPABASE_URL !== undefined;+ config.INDEX_DATABASE_URL !== "" && config.INDEX_DATABASE_URL !== undefined;export const useSearchIndex =- config.SEARCH_INDEX_SUPABASE_URL !== "" &&- config.SEARCH_INDEX_SUPABASE_URL !== undefined;+ config.SEARCH_SERVICE_URL !== "" && config.SEARCH_SERVICE_URL !== undefined;export function normalizeURLForIndex(url: string): string {const urlObj = new URL(url);@@ -254,8 +219,8 @@ export function normalizeURLForIndex(url: string): string {return urlObj.toString();}-export function hashURL(url: string): string {- return "\\x" + crypto.createHash("sha256").update(url).digest("hex");+export function hashURL(url: string): Buffer {+ return crypto.createHash("sha256").update(url).digest();}export function generateURLSplits(url: string): string[] {@@ -331,13 +296,34 @@ export async function addIndexInsertJob(data: any) {);}+function reviveBuffers(_key: string, value: any) {+ return value?.type === "Buffer" && Array.isArray(value.data)+ ? Buffer.from(value.data)+ : value;+}++function safeParseJob<T>(raw: string, queueKey: string): T | undefined {+ try {+ return JSON.parse(raw, reviveBuffers) as T;+ } catch (error) {+ _logger.error(`Failed to parse queued job, skipping`, {+ error,+ queueKey,+ raw,+ });+ return undefined;+ }+}+async function getIndexInsertJobs(): Promise<any[]> {const jobs =(await redisEvictConnection.lpop(INDEX_INSERT_QUEUE_KEY,INDEX_INSERT_BATCH_SIZE,)) ?? [];- return jobs.map(x => JSON.parse(x));+ return jobs+ .map(x => safeParseJob<any>(x, INDEX_INSERT_QUEUE_KEY))+ .filter(x => x !== undefined);}export async function processIndexInsertJobs() {@@ -349,13 +335,7 @@ export async function processIndexInsertJobs() {jobCount: jobs.length,});try {- const { error } = await index_supabase_service.from("index").insert(jobs);- if (error) {- _logger.error(`Index inserter failed to insert jobs`, {- error,- jobCount: jobs.length,- });- }+ await dbIndex.insert(schema.index).values(jobs);_logger.info(`Index inserter inserted jobs`, { jobCount: jobs.length });} catch (error) {_logger.error(`Index inserter failed to insert jobs`, {@@ -372,17 +352,19 @@ export async function getIndexInsertQueueLength(): Promise<number> {const OMCE_JOB_QUEUE_KEY = "omce-job-queue";const OMCE_JOB_QUEUE_BATCH_SIZE = 100;-export async function addOMCEJob(data: [number, string]) {+export async function addOMCEJob(data: [number, Buffer]) {await redisEvictConnection.sadd(OMCE_JOB_QUEUE_KEY, JSON.stringify(data));}-async function getOMCEJobs(): Promise<[number, string][]> {+async function getOMCEJobs(): Promise<[number, Buffer][]> {const jobs =(await redisEvictConnection.spop(OMCE_JOB_QUEUE_KEY,OMCE_JOB_QUEUE_BATCH_SIZE,)) ?? [];- return jobs.map(x => JSON.parse(x) as [number, string]);+ return jobs+ .map(x => safeParseJob<[number, Buffer]>(x, OMCE_JOB_QUEUE_KEY))+ .filter((x): x is [number, Buffer] => x !== undefined);}export async function processOMCEJobs() {@@ -396,15 +378,9 @@ export async function processOMCEJobs() {try {for (const job of jobs) {const [level, hash] = job;- const { error } = await index_supabase_service.rpc(- "insert_omce_job_if_needed",- {- i_domain_level: level,- i_domain_hash: hash,- },- );-- if (error) {+ try {+ await insertOmceJobIfNeeded(level, hash);+ } catch (error) {_logger.error(`OMCE job inserter failed to insert job`, {error,job,@@ -441,40 +417,20 @@ export async function queryIndexAtSplitLevel(const level = urlSplitsHash.length - 1;- let links: Set<string> = new Set();- let iteration = 0;-- while (true) {- // Query the index for the next set of links- const { data: _data, error } = await index_supabase_service- .rpc("query_index_at_split_level", {- i_level: level,- i_url_hash: urlSplitsHash[level],- i_newer_than: new Date(Date.now() - maxAge).toISOString(),- })- .range(iteration * 1000, (iteration + 1) * 1000);-- // If there's an error, return the links we have- if (error) {- _logger.warn("Error querying index", { error, url, limit });- return [...links].slice(0, limit);- }-- // Add the links to the set- const data = _data ?? [];+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the+ // previous .range() pagination loop is no longer needed.+ try {+ const data = await rpcQueryIndexAtSplitLevel(+ level,+ urlSplitsHash[level],+ new Date(Date.now() - maxAge).toISOString(),+ );+ const links = new Set<string>();data.forEach(x => links.add(x.resolved_url));-- // If we have enough links, return them- if (links.size >= limit) {- return [...links].slice(0, limit);- }-- // If we get less than 1000 links from the query, we're done- if (data.length < 1000) {- return [...links].slice(0, limit);- }-- iteration++;+ return [...links].slice(0, limit);+ } catch (error) {+ _logger.warn("Error querying index", { error, url, limit });+ return [];}}@@ -494,40 +450,20 @@ export async function queryIndexAtDomainSplitLevel(return [];}- let links: Set<string> = new Set();- let iteration = 0;-- while (true) {- // Query the index for the next set of links- const { data: _data, error } = await index_supabase_service- .rpc("query_index_at_domain_split_level", {- i_level: level,- i_domain_hash: domainSplitsHash[level],- i_newer_than: new Date(Date.now() - maxAge).toISOString(),- })- .range(iteration * 1000, (iteration + 1) * 1000);-- // If there's an error, return the links we have- if (error) {- _logger.warn("Error querying index", { error, hostname, limit });- return [...links].slice(0, limit);- }-- // Add the links to the set- const data = _data ?? [];+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the+ // previous .range() pagination loop is no longer needed.+ try {+ const data = await rpcQueryIndexAtDomainSplitLevel(+ level,+ domainSplitsHash[level],+ new Date(Date.now() - maxAge).toISOString(),+ );+ const links = new Set<string>();data.forEach(x => links.add(x.resolved_url));-- // If we have enough links, return them- if (links.size >= limit) {- return [...links].slice(0, limit);- }-- // If we get less than 1000 links from the query, we're done- if (data.length < 1000) {- return [...links].slice(0, limit);- }-- iteration++;+ return [...links].slice(0, limit);+ } catch (error) {+ _logger.warn("Error querying index", { error, hostname, limit });+ return [];}}@@ -546,20 +482,16 @@ export async function queryOMCESignatures(return [];}- const { data, error } = await index_supabase_service.rpc(- "query_omce_signatures",- {- i_domain_hash: domainSplitsHash[level],- i_newer_than: new Date(Date.now() - maxAge).toISOString(),- },- );-- if (error) {+ try {+ const data = await rpcQueryOmceSignatures(+ domainSplitsHash[level],+ new Date(Date.now() - maxAge).toISOString(),+ );+ return data?.[0]?.signatures ?? [];+ } catch (error) {_logger.warn("Error querying index (omce)", { error, hostname });return [];}-- return data?.[0]?.signatures ?? [];}export async function queryEngpickerVerdict(@@ -577,37 +509,26 @@ export async function queryEngpickerVerdict(}// 250ms max time taken-- const res: { data: any; error: any } = await Promise.any([- index_supabase_service.rpc("query_engpicker_verdict", {- i_domain_hash: domainSplitsHash[level],- }),- new Promise<{- data: {- verdict: "TlsClientOk" | "ChromeCdpRequired" | "Uncertain" | "Unknown";- }[];- error: any;- }>(resolve =>- setTimeout(- () =>- resolve({- data: [{ verdict: "Unknown" }],- error: "Took longer than 250ms",- }),- 250,+ try {+ const data = await Promise.race([+ rpcQueryEngpickerVerdict(domainSplitsHash[level]),+ new Promise<{ verdict: string }[]>(resolve =>+ setTimeout(() => resolve([{ verdict: "Unknown" }]), 250),),- ),- ]);+ ]);- if (res.error) {+ return (data?.[0]?.verdict ?? "Unknown") as+ | "TlsClientOk"+ | "ChromeCdpRequired"+ | "Uncertain"+ | "Unknown";+ } catch (error) {_logger.warn("Error querying index (engpicker)", {- error: res.error,+ error,hostname,});return "Unknown";}-- return res.data?.[0]?.verdict ?? "Unknown";}export async function queryIndexAtSplitLevelWithMeta(@@ -625,48 +546,23 @@ export async function queryIndexAtSplitLevelWithMeta(const level = urlSplitsHash.length - 1;- let links: MapDocument[] = [];- let iteration = 0;-- while (true) {- // Query the index for the next set of links- const { data: _data, error } = await index_supabase_service- .rpc("query_index_at_split_level_with_meta", {- i_level: level,- i_url_hash: urlSplitsHash[level],- i_newer_than: new Date(- Date.now() - 2 * 24 * 60 * 60 * 1000,- ).toISOString(),- })- .range(iteration * 1000, (iteration + 1) * 1000);-- // If there's an error, return the links we have- if (error) {- _logger.warn("Error querying index", { error, url, limit });- return links.slice(0, limit);- }-- // Add the links to the set- const data = _data ?? [];- data.forEach(x =>- links.push({- url: x.resolved_url,- title: x.title ?? undefined,- description: x.description ?? undefined,- }),+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the+ // previous .range() pagination loop is no longer needed.+ try {+ const data = await rpcQueryIndexAtSplitLevelWithMeta(+ level,+ urlSplitsHash[level],+ new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),);-- // If we have enough links, return them- if (links.length >= limit) {- return links.slice(0, limit);- }-- // If we get less than 1000 links from the query, we're done- if (data.length < 1000) {- return links.slice(0, limit);- }-- iteration++;+ const links: MapDocument[] = data.map(x => ({+ url: x.resolved_url,+ title: x.title ?? undefined,+ description: x.description ?? undefined,+ }));+ return links.slice(0, limit);+ } catch (error) {+ _logger.warn("Error querying index", { error, url, limit });+ return [];}}@@ -685,53 +581,28 @@ export async function queryIndexAtDomainSplitLevelWithMeta(return [];}- let links: MapDocument[] = [];- let iteration = 0;-- while (true) {- // Query the index for the next set of links- const { data: _data, error } = await index_supabase_service- .rpc("query_index_at_domain_split_level_with_meta", {- i_level: level,- i_domain_hash: domainSplitsHash[level],- i_newer_than: new Date(- Date.now() - 2 * 24 * 60 * 60 * 1000,- ).toISOString(),- })- .range(iteration * 1000, (iteration + 1) * 1000);-- // If there's an error, return the links we have- if (error) {- _logger.warn("Error querying index", { error, hostname, limit });- return links.slice(0, limit);- }-- // Add the links to the set- const data = _data ?? [];- data.forEach(x =>- links.push({- url: x.resolved_url,- title: x.title ?? undefined,- description: x.description ?? undefined,- }),+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the+ // previous .range() pagination loop is no longer needed.+ try {+ const data = await rpcQueryIndexAtDomainSplitLevelWithMeta(+ level,+ domainSplitsHash[level],+ new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),);-- // If we have enough links, return them- if (links.length >= limit) {- return links.slice(0, limit);- }-- // If we get less than 1000 links from the query, we're done- if (data.length < 1000) {- return links.slice(0, limit);- }-- iteration++;+ const links: MapDocument[] = data.map(x => ({+ url: x.resolved_url,+ title: x.title ?? undefined,+ description: x.description ?? undefined,+ }));+ return links.slice(0, limit);+ } catch (error) {+ _logger.warn("Error querying index", { error, hostname, limit });+ return [];}}type DomainPriority = {- domain_hash: string;+ domain_hash: Buffer;priority: number;};@@ -746,40 +617,18 @@ export async function queryDomainsForPrecrawl(return [];}- let results: DomainPriority[] = [];- let iteration = 0;-- while (true) {- const { data, error } = await index_supabase_service- .rpc("query_domain_priority", {- p_min_total: minEvents,- p_min_priority: minPriority,- p_lim: maxDomains,- p_time: date.toISOString(),- })- .range(- iteration * 1000,- Math.min((iteration + 1) * 1000, maxDomains) - 1,- );-- if (error) {- logger.error("Error getting domain priorities", {- error,- });- return results.slice(0, maxDomains);- }-- const batchData = data ?? [];- results = results.concat(batchData);-- if (results.length >= maxDomains) {- return results.slice(0, maxDomains);- }-- if (batchData.length < 1000) {- return results.slice(0, maxDomains);- }-- iteration++;+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the+ // previous .range() pagination loop is no longer needed.+ try {+ const data = await rpcQueryDomainPriority(+ minEvents,+ minPriority,+ maxDomains,+ date.toISOString(),+ );+ return data.slice(0, maxDomains);+ } catch (error) {+ logger.error("Error getting domain priorities", { error });+ return [];}}diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 52dcd4e2c..3072069d7 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -20,11 +20,11 @@ import { resolveBillingMetadata } from "../billing/types";import systemMonitor from "../system-monitor";import { v7 as uuidv7 } from "uuid";import {- index_supabase_service,processIndexInsertJobs,processOMCEJobs,queryDomainsForPrecrawl,} from "..";+import { queryTopUrlsForDomain, updateTallyTeam } from "../../db/rpc";import { getSearchIndexClient } from "../../lib/search-index-client";// Search indexing is now handled by the separate search service// import { processSearchIndexJobs } from "../../lib/search-index/queue";@@ -40,7 +40,6 @@ import { _addScrapeJobToBullMQ } from "../queue-jobs";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import { crawlGroup } from "../worker/nuq";import { getACUCTeam } from "../../controllers/auth";-import { supabase_service } from "../supabase";import { processEngpickerJob } from "../../lib/engpicker";import { logRequest } from "../logging/log_job";@@ -90,8 +89,7 @@ const processBillingJobInternal = async (token: string, job: Job) => {is_extract,api_key_id,autumnTrackInRequest,- } =- job.data;+ } = job.data;logger.info(`Adding team ${team_id} billing operation to batch queue`, {credits,@@ -270,7 +268,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {type DomainUrlResult = {url: string;- domain_hash: string;+ domain_hash: Buffer;event_count: number;rank: number;};@@ -284,13 +282,16 @@ const processPrecrawlJob = async (token: string, job: Job) => {const batch = batches[i];const batchFutures = batch.map(({ hash, urlsToFetch }) => {- return index_supabase_service- .rpc("query_top_urls_for_domain", {- p_domain_hash: hash,- p_time_window: "8 days", // increasing window can significantly slow down the query, modify with caution- p_top_n: urlsToFetch,- })- .overrideTypes<DomainUrlResult[]>();+ return queryTopUrlsForDomain<DomainUrlResult>(+ hash,+ "8 days", // increasing window can significantly slow down the query, modify with caution+ urlsToFetch,+ )+ .then(data => ({ data, error: null }))+ .catch(error => ({+ data: null as DomainUrlResult[] | null,+ error,+ }));});const startTimeNs = process.hrtime.bigint();@@ -352,10 +353,11 @@ const processPrecrawlJob = async (token: string, job: Job) => {const bucketedByDomain: Map<string, DomainUrlResult[]> = new Map();for (const item of urls) {- if (!bucketedByDomain.has(item.domain_hash)) {- bucketedByDomain.set(item.domain_hash, []);+ const key = item.domain_hash.toString("hex");+ if (!bucketedByDomain.has(key)) {+ bucketedByDomain.set(key, []);}- bucketedByDomain.get(item.domain_hash)!.push(item);+ bucketedByDomain.get(key)!.push(item);}const crawlTargets: Map<@@ -372,7 +374,9 @@ const processPrecrawlJob = async (token: string, job: Job) => {for (const domain of domains) {try {- const pages = bucketedByDomain.get(domain.domain_hash);+ const pages = bucketedByDomain.get(+ domain.domain_hash.toString("hex"),+ );// if this doesn't have any pages, do we want to locate the domain itself and add root only?if (!pages || pages.length === 0) {@@ -682,14 +686,11 @@ async function tallyBilling() {for (const teamId of billedTeams) {logger.info("Updating tally for team", { teamId });- const { error } = await supabase_service.rpc("update_tally_10_team", {- i_team_id: teamId,- });-- if (error) {- logger.warn("Failed to update tally for team", { teamId, error });- } else {+ try {+ await updateTallyTeam(teamId);logger.info("Updated tally for team", { teamId });+ } catch (error) {+ logger.warn("Failed to update tally for team", { teamId, error });}}diff --git a/apps/api/src/services/ledger/supabase-ledger.ts b/apps/api/src/services/ledger/supabase-ledger.tsdeleted file mode 100644index 7767aded2..000000000--- a/apps/api/src/services/ledger/supabase-ledger.ts+++ /dev/null@@ -1,82 +0,0 @@-import { createClient, SupabaseClient } from "@supabase/supabase-js";-import { logger } from "../../lib/logger";-import { configDotenv } from "dotenv";-import { config } from "../../config";-configDotenv();--// SupabaseLedgerService class initializes the Supabase client for the ledger schema-class SupabaseLedgerService {- private client: SupabaseClient<any, "ledger", any> | null = null;- private rrClient: SupabaseClient<any, "ledger", any> | null = null;-- constructor() {- const supabaseUrl = config.SUPABASE_URL;- const supabaseReplicaUrl = config.SUPABASE_REPLICA_URL;- const supabaseServiceToken = config.SUPABASE_SERVICE_TOKEN;- const useDbAuthentication = config.USE_DB_AUTHENTICATION;-- // Only initialize the Supabase client if both URL and Service Token are provided.- if (!useDbAuthentication) {- // Warn the user that Authentication is disabled by setting the client to null- logger.warn(- "Authentication is disabled. Supabase ledger client will not be initialized.",- );- this.client = null;- } else if (!supabaseUrl || !supabaseServiceToken || !supabaseReplicaUrl) {- logger.error(- "Supabase environment variables aren't configured correctly. Supabase ledger client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable",- );- } else {- this.client = createClient(supabaseUrl, supabaseServiceToken, {- global: {- headers: {- "sb-lb-routing-mode": "alpha-all-services",- },- },- db: {- //@ts-ignore- schema: "ledger",- },- });-- this.rrClient = createClient(supabaseReplicaUrl, supabaseServiceToken, {- db: {- //@ts-ignore- schema: "ledger",- },- });- }- }-- // Provides access to the initialized Supabase client, if available.- getClient(): SupabaseClient<any, "ledger", any> | null {- return this.client;- }-- getRRClient(): SupabaseClient<any, "ledger", any> | null {- return this.rrClient;- }-}--const ledgerServ = new SupabaseLedgerService();--// Using a Proxy to handle dynamic access to the Supabase ledger client or service methods.-// This approach ensures that if Supabase is not configured, any attempt to use it will result in a clear error.-export const supabase_ledger_service: SupabaseClient<any, "ledger", any> =- new Proxy(ledgerServ, {- get: function (target, prop, receiver) {- const client = target.getClient();- // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.- if (client === null) {- return () => {- throw new Error("Supabase ledger client is not configured.");- };- }- // Direct access to SupabaseLedgerService properties takes precedence.- if (prop in target) {- return Reflect.get(target, prop, receiver);- }- // Otherwise, delegate access to the Supabase client.- return Reflect.get(client, prop, receiver);- },- }) as unknown as SupabaseClient<any, "ledger", any>;diff --git a/apps/api/src/services/ledger/tracking.ts b/apps/api/src/services/ledger/tracking.tsindex 2c24801e4..8aba5fd7f 100644--- a/apps/api/src/services/ledger/tracking.ts+++ b/apps/api/src/services/ledger/tracking.ts@@ -1,6 +1,7 @@import { EventDataMap, EventDefinitionSlug } from "./data-schemas";-// @ts-nocheck - Schema not in types_db yet-import { supabase_ledger_service } from "./supabase-ledger";+import { eq } from "drizzle-orm";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";import { getValue, setValue } from "../redis";import { logger } from "../../lib/logger";@@ -26,14 +27,15 @@ export async function trackEvent<T extends EventDefinitionSlug>(providerDefinition = JSON.parse(cachedData);} else {// If not in cache, fetch from database- const result = await supabase_ledger_service- .from("provider_definitions")- .select("id")- .eq("slug", definitionSlug)- .single();-- definitionError = result.error;- providerDefinition = result.data;+ try {+ [providerDefinition] = await db+ .select({ id: schema.provider_definitions.id })+ .from(schema.provider_definitions)+ .where(eq(schema.provider_definitions.slug, definitionSlug))+ .limit(1);+ } catch (error) {+ definitionError = error;+ }// Cache the result for 24 hours (1440 minutes)if (!definitionError && providerDefinition) {@@ -51,23 +53,24 @@ export async function trackEvent<T extends EventDefinitionSlug>(}// Create the track- const { data: track, error: trackError } = await supabase_ledger_service- .from("tracks")- //@ts-ignore- .insert({- created_at: new Date().toISOString(),- //@ts-ignore- provider_definition_id: providerDefinition.id,- data: data,- })- .select("uuid")- .single();-- if (trackError || !track) {+ let track: { uuid: string } | undefined;+ try {+ [track] = await db+ .insert(schema.tracks)+ .values({+ created_at: new Date().toISOString(),+ provider_definition_id: providerDefinition.id,+ data: data,+ })+ .returning({ uuid: schema.tracks.uuid });+ } catch (trackError) {logger.error("Error creating track:", trackError);return null;}- //@ts-ignore++ if (!track) {+ return null;+ }return track.uuid;} catch (error) {logger.error("Error tracking event:", error);diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex 617588321..0df813e01 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -23,14 +23,10 @@ jest.mock("../../lib/logger", () => ({logger,}));-const insert = jest.fn<(data: any) => Promise<{ error: any }>>();-const from = jest.fn(() => ({- insert,-}));-jest.mock("../supabase", () => ({- supabase_service: {- from,- },+const values = jest.fn<(data: any) => Promise<void>>();+const insert = jest.fn(() => ({ values }));+jest.mock("../../db/connection", () => ({+ db: { insert },}));jest.mock("../../lib/gcs-jobs", () => ({@@ -47,6 +43,7 @@ jest.mock("../../lib/extract/extract-redis", () => ({}));import { logSearch, type LoggedSearch } from "./log_job";+import * as schema from "../../db/schema";function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {return {@@ -71,7 +68,7 @@ function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {describe("logSearch", () => {beforeEach(() => {jest.clearAllMocks();- insert.mockResolvedValue({ error: null });+ values.mockResolvedValue(undefined);});it("removes null bytes from search query log fields", async () => {@@ -85,8 +82,8 @@ describe("logSearch", () => {await logSearch(search);- expect(from).toHaveBeenCalledWith("searches");- const inserted = insert.mock.calls[0][0];+ expect(insert).toHaveBeenCalledWith(schema.searches);+ const inserted = values.mock.calls[0][0];expect(inserted.query).toBe("helloworld");expect(inserted.options.query).toBe("nestedquery");expect(inserted.options.sources[0].location).toBe("New\u0000York");@@ -94,12 +91,11 @@ describe("logSearch", () => {});it("uses sanitized data in Sentry insert failure context", async () => {- insert.mockResolvedValueOnce({- error: {+ values.mockRejectedValueOnce(+ Object.assign(new Error("unsupported Unicode escape sequence"), {code: "22P05",- message: "unsupported Unicode escape sequence",- },- });+ }),+ );await logSearch(makeSearch({diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 508ce7423..ada3d82c9 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -1,9 +1,12 @@-import { supabase_service } from "../supabase";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";+import { changeTrackingInsertScrape } from "../../db/rpc";import { config } from "../../config";import "dotenv/config";import { logger as _logger } from "../../lib/logger";import { configDotenv } from "dotenv";import * as Sentry from "@sentry/node";+import type { PgTable } from "drizzle-orm/pg-core";import {saveDeepResearchToGCS,saveExtractToGCS,@@ -33,58 +36,90 @@ function sanitizeString(value: string | null | undefined): string | null {return value.replace(nullByteRegex, "");}+const tableMap: Record<string, PgTable> = {+ requests: schema.requests,+ scrapes: schema.scrapes,+ parses: schema.parses,+ crawls: schema.crawls,+ batch_scrapes: schema.batch_scrapes,+ searches: schema.searches,+ extracts: schema.extracts,+ maps: schema.maps,+ llmstxts: schema.llmstxts,+ deep_researches: schema.deep_researches,+};+async function robustInsert(table: string,data: any,force: boolean,- logger: Logger,+ _logger: Logger,) {+ const logger = _logger.child({+ module: "log_job",+ method: "robustInsert",+ table,+ canonicalLog: "log_job/robustInsert",+ });+if (config.USE_DB_AUTHENTICATION !== true) {logger.info("Skipping database insertion due to USE_DB_AUTHENTICATION being off",- { table },);return;}+ const target = tableMap[table];++ const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];+if (force) {- let i = 0,- done = false;- let lastError: any = null;- while (i++ <= 10) {+ for (let i = 0; i < 10; i++) {+ const start = Date.now();try {- const { error } = await supabase_service.from(table).insert(data);- if (error) {- lastError = error;- logger.error(- "Error inserting into database due to Supabase error, trying again",- { error, table, attempt: i },- );- await new Promise(resolve => setTimeout(resolve, 75));- } else {- done = true;- break;- }+ await db.insert(target).values(data);+ attempts.push({+ error: null,+ timeMs: Date.now() - start,+ backoffMs: i === 0 ? 0 : 75,+ });+ break;} catch (error) {- lastError = error;- logger.error(- "Error inserting into database due to unknown error, trying again",- { error, table, attempt: i },- );+ attempts.push({+ error,+ timeMs: Date.now() - start,+ backoffMs: i === 0 ? 0 : 75,+ });await new Promise(resolve => setTimeout(resolve, 75));}}- if (done) {- logger.info("Inserted into database successfully", { table });+ if (attempts.length === 1 && attempts[0].error === null) {+ logger.debug(+ "Inserted into database successfully (" + table + ", " + data.id + ")",+ { attempts },+ );+ } else if (+ attempts.length > 1 &&+ attempts[attempts.length - 1].error === null+ ) {+ logger.warn(+ "Inserted into database successfully with retries (" ++ table ++ ", " ++ data.id ++ ")",+ { attempts },+ );} else {- logger.error("Failed to insert into database after 10 attempts", {- table,- lastError,- });+ logger.error(+ "Failed to insert into database (" + table + ", " + data.id + ")",+ { attempts },+ );// Report to Sentry with contextSentry.captureException(- lastError || new Error("Database insert failed after 10 attempts"),+ attempts[attempts.length - 1]?.error ||+ new Error("Database insert failed after 10 attempts"),{tags: {table,@@ -94,40 +129,28 @@ async function robustInsert(table,data: JSON.stringify(data).substring(0, 500), // Limit sizeattempts: 10,- lastError: lastError ? JSON.stringify(lastError) : null,+ lastError: attempts[attempts.length - 1]?.error+ ? JSON.stringify(attempts[attempts.length - 1].error)+ : null,},},);}} else {+ const start = Date.now();try {- const { error } = await supabase_service.from(table).insert(data);- if (error) {- logger.error("Error inserting into database due to Supabase error", {- error,- table,- });- // Report to Sentry- Sentry.captureException(error, {- tags: {- table,- operation: "robustInsert",- force: "false",- },- extra: {- table,- error: JSON.stringify(error),- data: JSON.stringify(data).substring(0, 500), // Limit size- },- });- } else {- logger.info("Inserted into database successfully", { table });- }+ await db.insert(target).values(data);+ attempts.push({ error: null, timeMs: Date.now() - start, backoffMs: 0 });+ logger.debug(+ "Inserted into database successfully (" + table + ", " + data.id + ")",+ { attempts },+ );} catch (error) {- logger.error("Error inserting into database due to unknown error", {- error,- table,- });+ attempts.push({ error, timeMs: Date.now() - start, backoffMs: 0 });+ logger.error(+ "Failed to insert into database (" + table + ", " + data.id + ")",+ { attempts },+ );// Report to SentrySentry.captureException(error, {tags: {@@ -295,27 +318,21 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {);if (hasMarkdown || hasChangeTracking) {- const { error } = await supabase_service.rpc(- "change_tracking_insert_scrape",- {- p_team_id: scrape.team_id,- p_url: scrape.url,- p_job_id: scrape.id,- p_change_tracking_tag: hasChangeTracking- ? hasChangeTracking.tag- : null,- p_date_added: new Date().toISOString(),- },- );-- if (error) {+ try {+ await changeTrackingInsertScrape({+ team_id: scrape.team_id,+ url: scrape.url,+ job_id: scrape.id,+ change_tracking_tag: hasChangeTracking ? hasChangeTracking.tag : null,+ date_added: new Date().toISOString(),+ });+ _logger.debug("Change tracking record inserted successfully");+ } catch (error) {_logger.warn("Error inserting into change_tracking_scrapes", {error,scrapeId: scrape.id,teamId: scrape.team_id,});- } else {- _logger.debug("Change tracking record inserted successfully");}}}diff --git a/apps/api/src/services/monitoring/email_recipients.test.ts b/apps/api/src/services/monitoring/email_recipients.test.tsindex 8e7b60e2d..a737b6cab 100644--- a/apps/api/src/services/monitoring/email_recipients.test.ts+++ b/apps/api/src/services/monitoring/email_recipients.test.ts@@ -13,10 +13,7 @@ const calls: CallRecord[] = [];// eq/neq calls and resolves at .maybeSingle() / .single() with the next// queued result. Tests stay explicit about query order without coupling to// the precise method chain shape.-type QueuedResponse = (- client: ClientKind,- op: ChainOp,-) => Promise<QueryResult>;+type QueuedResponse = (client: ClientKind, op: ChainOp) => Promise<QueryResult>;let queue: QueuedResponse[] = [];function queueResponses(responses: QueuedResponse[]): void {@@ -28,16 +25,21 @@ function makeChain(client: ClientKind): any {const setOp = (next: ChainOp) => {if (op === null) op = next;};- const resolve = (): Promise<QueryResult> => {+ // Drizzle returns row arrays and throws on error; convert the queued+ // {data, error} responses accordingly.+ const resolve = (): Promise<unknown[]> => {const finalOp = op ?? "unknown";calls.push({ client, op: finalOp });const next = queue.shift();if (!next) {throw new Error(- `No queued Supabase response for client=${client} op=${finalOp} (queue exhausted)`,+ `No queued response for client=${client} op=${finalOp} (queue exhausted)`,);}- return next(client, finalOp);+ return next(client, finalOp).then(({ data, error }) => {+ if (error) return Promise.reject(error);+ return data === null || data === undefined ? [] : [data];+ });};const builder: any = {select: () => {@@ -56,21 +58,27 @@ function makeChain(client: ClientKind): any {setOp("delete");return builder;},+ from: () => builder,+ values: () => builder,+ set: () => builder,+ where: () => builder,eq: () => builder,neq: () => builder,in: () => builder,- maybeSingle: resolve,- single: resolve,+ orderBy: () => builder,+ limit: () => resolve(),+ returning: () => resolve(),+ then: (res: any, rej: any) => resolve().then(res, rej),};return builder;}-jest.mock("../supabase", () => ({- supabase_service: {- from: (...args: unknown[]) => fromPrimary(...args),+jest.mock("../../db/connection", () => ({+ get db() {+ return makeChain("primary");},- supabase_rr_service: {- from: (...args: unknown[]) => fromRR(...args),+ get dbRr() {+ return makeChain("rr");},}));@@ -118,10 +126,14 @@ const baseEnsureInput = {},};-const ok = (data: unknown): QueuedResponse => () =>- Promise.resolve({ data, error: null });-const fail = (error: unknown): QueuedResponse => () =>- Promise.resolve({ data: null, error });+const ok =+ (data: unknown): QueuedResponse =>+ () =>+ Promise.resolve({ data, error: null });+const fail =+ (error: unknown): QueuedResponse =>+ () =>+ Promise.resolve({ data: null, error });describe("ensureMonitorEmailRecipient", () => {it("returns existing row when the recipient already exists (no insert)", async () => {diff --git a/apps/api/src/services/monitoring/email_recipients.ts b/apps/api/src/services/monitoring/email_recipients.tsindex c00a36e46..4644028c5 100644--- a/apps/api/src/services/monitoring/email_recipients.ts+++ b/apps/api/src/services/monitoring/email_recipients.ts@@ -1,6 +1,8 @@import { randomBytes } from "crypto";+import { and, eq, inArray, ne } from "drizzle-orm";+import { db, dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { logger as _logger } from "../../lib/logger";-import { supabase_rr_service, supabase_service } from "../supabase";const logger = _logger.child({ module: "monitor-email-recipients" });@@ -35,22 +37,28 @@ function generateRecipientToken(): string {return randomBytes(32).toString("base64url");}-function throwIfError(error: any, message: string): void {- if (error) {- throw new Error(`${message}: ${error.message ?? JSON.stringify(error)}`);+async function run<T>(fn: () => Promise<T>, message: string): Promise<T> {+ try {+ return await fn();+ } catch (error) {+ throw new Error(+ `${message}: ${error instanceof Error ? error.message : JSON.stringify(error)}`,+ );}}export async function listMonitorEmailRecipients(monitorId: string,): Promise<MonitorEmailRecipientRow[]> {- const { data, error } = await supabase_rr_service- .from("monitor_email_recipients")- .select("*")- .eq("monitor_id", monitorId);-- throwIfError(error, "Failed to list monitor email recipients");- return (data ?? []) as MonitorEmailRecipientRow[];+ const data = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitor_email_recipients)+ .where(eq(schema.monitor_email_recipients.monitor_id, monitorId)),+ "Failed to list monitor email recipients",+ );+ return data as MonitorEmailRecipientRow[];}async function getRecipientByToken(@@ -59,33 +67,35 @@ async function getRecipientByToken(const trimmed = token.trim();if (!trimmed) return null;- const { data, error } = await supabase_rr_service- .from("monitor_email_recipients")- .select("*")- .eq("token", trimmed)- .maybeSingle();-- throwIfError(error, "Failed to look up monitor email recipient by token");+ const [data] = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitor_email_recipients)+ .where(eq(schema.monitor_email_recipients.token, trimmed))+ .limit(1),+ "Failed to look up monitor email recipient by token",+ );return (data ?? null) as MonitorEmailRecipientRow | null;}export async function getMonitorNameById(monitorId: string,): Promise<string | null> {- const { data, error } = await supabase_rr_service- .from("monitors")- .select("name")- .eq("id", monitorId)- .maybeSingle();-- if (error) {+ try {+ const [data] = await dbRr+ .select({ name: schema.monitors.name })+ .from(schema.monitors)+ .where(eq(schema.monitors.id, monitorId))+ .limit(1);+ return data?.name ?? null;+ } catch (error) {logger.warn("Failed to load monitor name for opt-in response", {error,monitorId,});return null;}- return (data?.name as string | undefined) ?? null;}// Team members are auto-confirmed; they already have dashboard access.@@ -95,12 +105,14 @@ export async function getTeamMemberEmails(): Promise<Set<string>> {if (emails.length === 0) return new Set();- const { data, error } = await supabase_rr_service- .from("user_teams")- .select("users(email)")- .eq("team_id", teamId);-- if (error) {+ let rows: { email: string | null }[];+ try {+ rows = await dbRr+ .select({ email: schema.users.email })+ .from(schema.user_teams)+ .innerJoin(schema.users, eq(schema.user_teams.user_id, schema.users.id))+ .where(eq(schema.user_teams.team_id, teamId));+ } catch (error) {logger.warn("Failed to load team member emails for recipient sync", {error,teamId,@@ -110,10 +122,9 @@ export async function getTeamMemberEmails(const wanted = new Set(emails.map(normalizeRecipientEmail));const matches = new Set<string>();- for (const row of data ?? []) {- const email = (row as any).users?.email;- if (typeof email === "string") {- const normalized = normalizeRecipientEmail(email);+ for (const row of rows) {+ if (typeof row.email === "string") {+ const normalized = normalizeRecipientEmail(row.email);if (wanted.has(normalized)) matches.add(normalized);}}@@ -135,13 +146,20 @@ async function fetchRecipientByMonitorEmail(monitorId: string,email: string,): Promise<MonitorEmailRecipientRow | null> {- const { data, error } = await supabase_rr_service- .from("monitor_email_recipients")- .select("*")- .eq("monitor_id", monitorId)- .eq("email", email)- .maybeSingle();- throwIfError(error, "Failed to look up monitor email recipient");+ const [data] = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitor_email_recipients)+ .where(+ and(+ eq(schema.monitor_email_recipients.monitor_id, monitorId),+ eq(schema.monitor_email_recipients.email, email),+ ),+ )+ .limit(1),+ "Failed to look up monitor email recipient",+ );return (data ?? null) as MonitorEmailRecipientRow | null;}@@ -152,13 +170,20 @@ async function fetchRecipientByMonitorEmailPrimary(monitorId: string,email: string,): Promise<MonitorEmailRecipientRow | null> {- const { data, error } = await supabase_service- .from("monitor_email_recipients")- .select("*")- .eq("monitor_id", monitorId)- .eq("email", email)- .maybeSingle();- throwIfError(error, "Failed to look up monitor email recipient");+ const [data] = await run(+ () =>+ db+ .select()+ .from(schema.monitor_email_recipients)+ .where(+ and(+ eq(schema.monitor_email_recipients.monitor_id, monitorId),+ eq(schema.monitor_email_recipients.email, email),+ ),+ )+ .limit(1),+ "Failed to look up monitor email recipient",+ );return (data ?? null) as MonitorEmailRecipientRow | null;}@@ -167,12 +192,15 @@ async function fetchRecipientByMonitorEmailPrimary(async function fetchRecipientByIdPrimary(id: string,): Promise<MonitorEmailRecipientRow | null> {- const { data, error } = await supabase_service- .from("monitor_email_recipients")- .select("*")- .eq("id", id)- .maybeSingle();- throwIfError(error, "Failed to look up monitor email recipient by id");+ const [data] = await run(+ () =>+ db+ .select()+ .from(schema.monitor_email_recipients)+ .where(eq(schema.monitor_email_recipients.id, id))+ .limit(1),+ "Failed to look up monitor email recipient by id",+ );return (data ?? null) as MonitorEmailRecipientRow | null;}@@ -207,13 +235,13 @@ export async function ensureMonitorEmailRecipient(params: {updated_at: now,};- const { data, error } = await supabase_service- .from("monitor_email_recipients")- .insert(insert)- .select("*")- .maybeSingle();-- if (error) {+ let data: MonitorEmailRecipientRow | undefined;+ try {+ [data] = (await db+ .insert(schema.monitor_email_recipients)+ .values(insert)+ .returning()) as MonitorEmailRecipientRow[];+ } catch (error) {if ((error as { code?: string }).code === POSTGRES_UNIQUE_VIOLATION) {const winner = await fetchRecipientByMonitorEmailPrimary(params.monitorId,@@ -223,7 +251,9 @@ export async function ensureMonitorEmailRecipient(params: {return { row: winner, created: false };}}- throwIfError(error, "Failed to insert monitor email recipient");+ throw new Error(+ `Failed to insert monitor email recipient: ${error instanceof Error ? error.message : JSON.stringify(error)}`,+ );}return { row: data as MonitorEmailRecipientRow, created: true };@@ -231,12 +261,14 @@ export async function ensureMonitorEmailRecipient(params: {export async function markRecipientConfirmationSent(id: string): Promise<void> {const now = new Date().toISOString();- const { error } = await supabase_service- .from("monitor_email_recipients")- .update({ confirmation_sent_at: now, updated_at: now })- .eq("id", id);-- throwIfError(error, "Failed to mark recipient confirmation sent");+ await run(+ () =>+ db+ .update(schema.monitor_email_recipients)+ .set({ confirmation_sent_at: now, updated_at: now })+ .where(eq(schema.monitor_email_recipients.id, id)),+ "Failed to mark recipient confirmation sent",+ );}export async function confirmRecipientByToken(@@ -254,15 +286,20 @@ export async function confirmRecipientByToken(// affect 0 rows; re-fetch and return the actual current state so an// in-flight confirm can't silently overwrite a terminal unsubscribe.const now = new Date().toISOString();- const { data, error } = await supabase_service- .from("monitor_email_recipients")- .update({ status: "confirmed", confirmed_at: now, updated_at: now })- .eq("id", row.id)- .eq("status", "pending")- .select("*")- .maybeSingle();-- throwIfError(error, "Failed to confirm monitor email recipient");+ const [data] = await run(+ () =>+ db+ .update(schema.monitor_email_recipients)+ .set({ status: "confirmed", confirmed_at: now, updated_at: now })+ .where(+ and(+ eq(schema.monitor_email_recipients.id, row.id),+ eq(schema.monitor_email_recipients.status, "pending"),+ ),+ )+ .returning(),+ "Failed to confirm monitor email recipient",+ );if (data) return data as MonitorEmailRecipientRow;return await fetchRecipientByIdPrimary(row.id);@@ -279,15 +316,20 @@ export async function unsubscribeRecipientByToken(// so two racing unsubscribes don't double-write timestamps. 0 rows here// means a concurrent call beat us to the terminal state.const now = new Date().toISOString();- const { data, error } = await supabase_service- .from("monitor_email_recipients")- .update({ status: "unsubscribed", unsubscribed_at: now, updated_at: now })- .eq("id", row.id)- .neq("status", "unsubscribed")- .select("*")- .maybeSingle();-- throwIfError(error, "Failed to unsubscribe monitor email recipient");+ const [data] = await run(+ () =>+ db+ .update(schema.monitor_email_recipients)+ .set({ status: "unsubscribed", unsubscribed_at: now, updated_at: now })+ .where(+ and(+ eq(schema.monitor_email_recipients.id, row.id),+ ne(schema.monitor_email_recipients.status, "unsubscribed"),+ ),+ )+ .returning(),+ "Failed to unsubscribe monitor email recipient",+ );if (data) return data as MonitorEmailRecipientRow;return await fetchRecipientByIdPrimary(row.id);@@ -296,12 +338,12 @@ export async function unsubscribeRecipientByToken(export async function touchRecipientsNotified(ids: string[]): Promise<void> {if (ids.length === 0) return;const now = new Date().toISOString();- const { error } = await supabase_service- .from("monitor_email_recipients")- .update({ last_notified_at: now, updated_at: now })- .in("id", ids);-- if (error) {+ try {+ await db+ .update(schema.monitor_email_recipients)+ .set({ last_notified_at: now, updated_at: now })+ .where(inArray(schema.monitor_email_recipients.id, ids));+ } catch (error) {logger.warn("Failed to update last_notified_at on recipients", {error,ids,diff --git a/apps/api/src/services/monitoring/email_recipients_sync.test.ts b/apps/api/src/services/monitoring/email_recipients_sync.test.tsindex 4f38d06a2..84e504839 100644--- a/apps/api/src/services/monitoring/email_recipients_sync.test.ts+++ b/apps/api/src/services/monitoring/email_recipients_sync.test.ts@@ -1,8 +1,3 @@-jest.mock("../supabase", () => ({- supabase_service: {},- supabase_rr_service: {},-}));-const mockEnsureRecipient = jest.fn();const mockListRecipients = jest.fn();const mockGetTeamMemberEmails = jest.fn();@@ -52,7 +47,8 @@ function recipientRow(status,token: `tok-${email}`,source,- confirmation_sent_at: status === "pending" ? new Date().toISOString() : null,+ confirmation_sent_at:+ status === "pending" ? new Date().toISOString() : null,confirmed_at: status === "confirmed" ? new Date().toISOString() : null,unsubscribed_at:status === "unsubscribed" ? new Date().toISOString() : null,@@ -201,11 +197,7 @@ describe("syncMonitorEmailRecipients", () => {mockListRecipients.mockResolvedValue([]);mockGetTeamMemberEmails.mockResolvedValue(new Set(["[email protected]"]));mockEnsureRecipient.mockImplementation(async ({ input }) => ({- row: recipientRow(- input.email,- input.status,- input.source,- ),+ row: recipientRow(input.email, input.status, input.source),created: true,}));@@ -214,7 +206,8 @@ describe("syncMonitorEmailRecipients", () => {});expect(mockSendConfirmationEmail).toHaveBeenCalledTimes(1);- const emailedTo = mockSendConfirmationEmail.mock.calls[0][0].recipient.email;+ const emailedTo =+ mockSendConfirmationEmail.mock.calls[0][0].recipient.email;expect(emailedTo).toBe("[email protected]");const byEmail = new Map(result.recipients.map(r => [r.email, r]));diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 25c1d94d1..e4cade567 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -533,7 +533,7 @@ async function runCrawlTarget(params: {const doc = job.returnvalue ?? (await getJobFromGCS(job.id))?.[0];if (!doc) continue;const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);- seen.add(hashMonitorUrl(url));+ seen.add(hashMonitorUrl(url).toString("hex"));credits += estimateActualCredits(doc, body.scrapeOptions);pages.push(await diffAndPersistPage({@@ -554,7 +554,7 @@ async function runCrawlTarget(params: {targetId: params.target.id,});for (const previous of previousPages) {- if (seen.has(previous.url_hash)) continue;+ if (seen.has(previous.url_hash.toString("hex"))) continue;await upsertMonitorPage({monitorId: params.monitor.id,teamId: params.monitor.team_id,@@ -1054,7 +1054,7 @@ async function processRemovedPagesForCompletedCrawls(params: {const seen = new Set(checkPages.filter(page => page.target_id === target.targetId)- .map(page => page.url_hash),+ .map(page => page.url_hash.toString("hex")),);const activePages = await listActiveMonitorPages({monitorId: params.monitor.id,@@ -1063,7 +1063,7 @@ async function processRemovedPagesForCompletedCrawls(params: {const removed: MonitorCheckPageInsert[] = [];for (const previous of activePages) {- if (seen.has(previous.url_hash)) continue;+ if (seen.has(previous.url_hash.toString("hex"))) continue;await upsertMonitorPage({monitorId: params.monitor.id,teamId: params.monitor.team_id,diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 2cec0f79b..51f0f7cb9 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -1,6 +1,9 @@import { createHash } from "crypto";import { v7 as uuidv7 } from "uuid";-import { supabase_rr_service, supabase_service } from "../supabase";+import { and, asc, count, desc, eq, isNull, ne } from "drizzle-orm";+import { db, dbRr } from "../../db/connection";+import * as schema from "../../db/schema";+import { monitoringClaimDueMonitors } from "../../db/rpc";import {getNextMonitorRunAt,estimateRunsPerMonth,@@ -17,8 +20,8 @@ import type {UpdateMonitorRequest,} from "./types";-export function hashMonitorUrl(url: string): string {- return `\\x${createHash("sha256").update(url).digest("hex")}`;+export function hashMonitorUrl(url: string): Buffer {+ return createHash("sha256").update(url).digest();}function ensureTargetIds(targets: Array<Record<string, any>>): MonitorTarget[] {@@ -62,9 +65,13 @@ function toMonitorSummary(check: MonitorCheckRow): MonitorSummary {};}-function throwIfError(error: any, message: string): void {- if (error) {- throw new Error(`${message}: ${error.message ?? JSON.stringify(error)}`);+async function run<T>(fn: () => Promise<T>, message: string): Promise<T> {+ try {+ return await fn();+ } catch (error) {+ throw new Error(+ `${message}: ${error instanceof Error ? error.message : JSON.stringify(error)}`,+ );}}@@ -93,7 +100,7 @@ export async function createMonitor(params: {// Omit goal/judge_enabled keys when undefined so a pre-migration DB// doesn't reject the insert. Migration lives in a separate repo.- const insert: Record<string, unknown> = {+ const insert: typeof schema.monitors.$inferInsert = {id: uuidv7(),team_id: params.teamId,name: params.input.name,@@ -112,13 +119,11 @@ export async function createMonitor(params: {if (params.input.judgeEnabled !== undefined) {insert.judge_enabled = params.input.judgeEnabled;}- const { data, error } = await supabase_service- .from("monitors")- .insert(insert)- .select("*")- .single();+ const [data] = await run(+ () => db.insert(schema.monitors).values(insert).returning(),+ "Failed to create monitor",+ );- throwIfError(error, "Failed to create monitor");return data as MonitorRow;}@@ -127,48 +132,67 @@ export async function listMonitors(params: {limit: number;offset: number;}): Promise<MonitorRow[]> {- const { data, error } = await supabase_rr_service- .from("monitors")- .select("*")- .eq("team_id", params.teamId)- .neq("status", "deleted")- .order("created_at", { ascending: false })- .range(params.offset, params.offset + params.limit - 1);-- throwIfError(error, "Failed to list monitors");- return (data ?? []) as MonitorRow[];+ const data = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitors)+ .where(+ and(+ eq(schema.monitors.team_id, params.teamId),+ ne(schema.monitors.status, "deleted"),+ ),+ )+ .orderBy(desc(schema.monitors.created_at))+ .limit(params.limit)+ .offset(params.offset),+ "Failed to list monitors",+ );+ return data as MonitorRow[];}export async function getMonitor(teamId: string,monitorId: string,): Promise<MonitorRow | null> {- const { data, error } = await supabase_rr_service- .from("monitors")- .select("*")- .eq("id", monitorId)- .eq("team_id", teamId)- .neq("status", "deleted")- .maybeSingle();-- throwIfError(error, "Failed to get monitor");- return data as MonitorRow | null;+ const [data] = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitors)+ .where(+ and(+ eq(schema.monitors.id, monitorId),+ eq(schema.monitors.team_id, teamId),+ ne(schema.monitors.status, "deleted"),+ ),+ )+ .limit(1),+ "Failed to get monitor",+ );+ return (data ?? null) as MonitorRow | null;}export async function getMonitorForUpdate(teamId: string,monitorId: string,): Promise<MonitorRow | null> {- const { data, error } = await supabase_service- .from("monitors")- .select("*")- .eq("id", monitorId)- .eq("team_id", teamId)- .neq("status", "deleted")- .maybeSingle();-- throwIfError(error, "Failed to get monitor");- return data as MonitorRow | null;+ const [data] = await run(+ () =>+ db+ .select()+ .from(schema.monitors)+ .where(+ and(+ eq(schema.monitors.id, monitorId),+ eq(schema.monitors.team_id, teamId),+ ne(schema.monitors.status, "deleted"),+ ),+ )+ .limit(1),+ "Failed to get monitor",+ );+ return (data ?? null) as MonitorRow | null;}export async function updateMonitor(params: {@@ -178,7 +202,7 @@ export async function updateMonitor(params: {nextRunAt?: Date;intervalMs?: number;}): Promise<MonitorRow | null> {- const patch: Record<string, unknown> = {+ const patch: Partial<typeof schema.monitors.$inferInsert> = {updated_at: new Date().toISOString(),};@@ -243,39 +267,49 @@ export async function updateMonitor(params: {}}- const { data, error } = await supabase_service- .from("monitors")- .update(patch)- .eq("id", params.monitorId)- .eq("team_id", params.teamId)- .neq("status", "deleted")- .select("*")- .maybeSingle();-- throwIfError(error, "Failed to update monitor");- return data as MonitorRow | null;+ const [data] = await run(+ () =>+ db+ .update(schema.monitors)+ .set(patch)+ .where(+ and(+ eq(schema.monitors.id, params.monitorId),+ eq(schema.monitors.team_id, params.teamId),+ ne(schema.monitors.status, "deleted"),+ ),+ )+ .returning(),+ "Failed to update monitor",+ );+ return (data ?? null) as MonitorRow | null;}export async function deleteMonitor(params: {teamId: string;monitorId: string;}): Promise<boolean> {- const { data, error } = await supabase_service- .from("monitors")- .update({- status: "deleted",- deleted_at: new Date().toISOString(),- next_run_at: null,- updated_at: new Date().toISOString(),- })- .eq("id", params.monitorId)- .eq("team_id", params.teamId)- .neq("status", "deleted")- .select("id")- .maybeSingle();-- throwIfError(error, "Failed to delete monitor");- return !!data;+ const data = await run(+ () =>+ db+ .update(schema.monitors)+ .set({+ status: "deleted",+ deleted_at: new Date().toISOString(),+ next_run_at: null,+ updated_at: new Date().toISOString(),+ })+ .where(+ and(+ eq(schema.monitors.id, params.monitorId),+ eq(schema.monitors.team_id, params.teamId),+ ne(schema.monitors.status, "deleted"),+ ),+ )+ .returning({ id: schema.monitors.id }),+ "Failed to delete monitor",+ );+ return data.length > 0;}export async function createMonitorCheck(params: {@@ -288,21 +322,22 @@ export async function createMonitorCheck(params: {params.monitor.targets,Boolean(params.monitor.judge_enabled) && Boolean(params.monitor.goal),);- const { data, error } = await supabase_service- .from("monitor_checks")- .insert({- id: uuidv7(),- monitor_id: params.monitor.id,- team_id: params.monitor.team_id,- trigger: params.trigger,- status: params.status ?? "queued",- scheduled_for: params.scheduledFor ?? null,- estimated_credits: estimated,- })- .select("*")- .single();-- throwIfError(error, "Failed to create monitor check");+ const [data] = await run(+ () =>+ db+ .insert(schema.monitor_checks)+ .values({+ id: uuidv7(),+ monitor_id: params.monitor.id,+ team_id: params.monitor.team_id,+ trigger: params.trigger,+ status: params.status ?? "queued",+ scheduled_for: params.scheduledFor ?? null,+ estimated_credits: estimated,+ })+ .returning(),+ "Failed to create monitor check",+ );return data as MonitorCheckRow;}@@ -310,16 +345,22 @@ export async function markMonitorRunning(params: {monitorId: string;checkId: string;}): Promise<void> {- const { error } = await supabase_service- .from("monitors")- .update({- current_check_id: params.checkId,- updated_at: new Date().toISOString(),- })- .eq("id", params.monitorId)- .is("current_check_id", null);-- throwIfError(error, "Failed to mark monitor running");+ await run(+ () =>+ db+ .update(schema.monitors)+ .set({+ current_check_id: params.checkId,+ updated_at: new Date().toISOString(),+ })+ .where(+ and(+ eq(schema.monitors.id, params.monitorId),+ isNull(schema.monitors.current_check_id),+ ),+ ),+ "Failed to mark monitor running",+ );}export async function dispatchScheduledMonitorCheck(params: {@@ -335,22 +376,27 @@ export async function dispatchScheduledMonitorCheck(params: {).toISOString(): null;- const { data, error } = await supabase_service- .from("monitors")- .update({- current_check_id: params.checkId,- locked_at: null,- locked_until: null,- next_run_at: nextRunAt,- updated_at: new Date().toISOString(),- })- .eq("id", params.monitor.id)- .is("current_check_id", null)- .select("id")- .maybeSingle();-- throwIfError(error, "Failed to dispatch scheduled monitor check");- return !!data;+ const data = await run(+ () =>+ db+ .update(schema.monitors)+ .set({+ current_check_id: params.checkId,+ locked_at: null,+ locked_until: null,+ next_run_at: nextRunAt,+ updated_at: new Date().toISOString(),+ })+ .where(+ and(+ eq(schema.monitors.id, params.monitor.id),+ isNull(schema.monitors.current_check_id),+ ),+ )+ .returning({ id: schema.monitors.id }),+ "Failed to dispatch scheduled monitor check",+ );+ return data.length > 0;}export async function updateMonitorScheduleAfterRun(params: {@@ -366,21 +412,23 @@ export async function updateMonitorScheduleAfterRun(params: {params.monitor.schedule_timezone,).toISOString(): null;- const { error } = await supabase_service- .from("monitors")- .update({- current_check_id: null,- locked_at: null,- locked_until: null,- last_run_at: params.check.finished_at ?? new Date().toISOString(),- last_check_id: params.check.id,- next_run_at: nextRunAt,- last_check_summary: params.summary ?? toMonitorSummary(params.check),- updated_at: new Date().toISOString(),- })- .eq("id", params.monitor.id);-- throwIfError(error, "Failed to update monitor after run");+ await run(+ () =>+ db+ .update(schema.monitors)+ .set({+ current_check_id: null,+ locked_at: null,+ locked_until: null,+ last_run_at: params.check.finished_at ?? new Date().toISOString(),+ last_check_id: params.check.id,+ next_run_at: nextRunAt,+ last_check_summary: params.summary ?? toMonitorSummary(params.check),+ updated_at: new Date().toISOString(),+ })+ .where(eq(schema.monitors.id, params.monitor.id)),+ "Failed to update monitor after run",+ );}export async function advanceMonitorAfterSkippedCheck(params: {@@ -395,20 +443,22 @@ export async function advanceMonitorAfterSkippedCheck(params: {params.monitor.schedule_timezone,).toISOString(): null;- const { error } = await supabase_service- .from("monitors")- .update({- locked_at: null,- locked_until: null,- last_run_at: params.check.finished_at ?? new Date().toISOString(),- last_check_id: params.check.id,- next_run_at: nextRunAt,- last_check_summary: toMonitorSummary(params.check),- updated_at: new Date().toISOString(),- })- .eq("id", params.monitor.id);-- throwIfError(error, "Failed to advance monitor after skipped check");+ await run(+ () =>+ db+ .update(schema.monitors)+ .set({+ locked_at: null,+ locked_until: null,+ last_run_at: params.check.finished_at ?? new Date().toISOString(),+ last_check_id: params.check.id,+ next_run_at: nextRunAt,+ last_check_summary: toMonitorSummary(params.check),+ updated_at: new Date().toISOString(),+ })+ .where(eq(schema.monitors.id, params.monitor.id)),+ "Failed to advance monitor after skipped check",+ );}export async function getMonitorCheck(@@ -416,30 +466,38 @@ export async function getMonitorCheck(monitorId: string,checkId: string,): Promise<MonitorCheckRow | null> {- const { data, error } = await supabase_rr_service- .from("monitor_checks")- .select("*")- .eq("id", checkId)- .eq("monitor_id", monitorId)- .eq("team_id", teamId)- .maybeSingle();-- throwIfError(error, "Failed to get monitor check");- return data as MonitorCheckRow | null;+ const [data] = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitor_checks)+ .where(+ and(+ eq(schema.monitor_checks.id, checkId),+ eq(schema.monitor_checks.monitor_id, monitorId),+ eq(schema.monitor_checks.team_id, teamId),+ ),+ )+ .limit(1),+ "Failed to get monitor check",+ );+ return (data ?? null) as MonitorCheckRow | null;}export async function listRunningMonitorChecks(limit: number = 100,): Promise<MonitorCheckRow[]> {- const { data, error } = await supabase_service- .from("monitor_checks")- .select("*")- .eq("status", "running")- .order("created_at", { ascending: true })- .limit(limit);-- throwIfError(error, "Failed to list running monitor checks");- return (data ?? []) as MonitorCheckRow[];+ const data = await run(+ () =>+ db+ .select()+ .from(schema.monitor_checks)+ .where(eq(schema.monitor_checks.status, "running"))+ .orderBy(asc(schema.monitor_checks.created_at))+ .limit(limit),+ "Failed to list running monitor checks",+ );+ return data as MonitorCheckRow[];}export async function listMonitorChecks(params: {@@ -449,41 +507,44 @@ export async function listMonitorChecks(params: {offset: number;status?: MonitorCheckRow["status"];}): Promise<MonitorCheckRow[]> {- let query = supabase_rr_service- .from("monitor_checks")- .select("*")- .eq("monitor_id", params.monitorId)- .eq("team_id", params.teamId)- .order("created_at", { ascending: false });-+ const conditions = [+ eq(schema.monitor_checks.monitor_id, params.monitorId),+ eq(schema.monitor_checks.team_id, params.teamId),+ ];if (params.status) {- query = query.eq("status", params.status);+ conditions.push(eq(schema.monitor_checks.status, params.status));}- const { data, error } = await query.range(- params.offset,- params.offset + params.limit - 1,+ const data = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitor_checks)+ .where(and(...conditions))+ .orderBy(desc(schema.monitor_checks.created_at))+ .limit(params.limit)+ .offset(params.offset),+ "Failed to list monitor checks",);-- throwIfError(error, "Failed to list monitor checks");- return (data ?? []) as MonitorCheckRow[];+ return data as MonitorCheckRow[];}export async function updateMonitorCheck(checkId: string,patch: Partial<MonitorCheckRow>,): Promise<MonitorCheckRow> {- const { data, error } = await supabase_service- .from("monitor_checks")- .update({- ...patch,- updated_at: new Date().toISOString(),- })- .eq("id", checkId)- .select("*")- .single();-- throwIfError(error, "Failed to update monitor check");+ const [data] = await run(+ () =>+ db+ .update(schema.monitor_checks)+ .set({+ ...patch,+ updated_at: new Date().toISOString(),+ })+ .where(eq(schema.monitor_checks.id, checkId))+ .returning(),+ "Failed to update monitor check",+ );return data as MonitorCheckRow;}@@ -492,15 +553,17 @@ export async function insertMonitorCheckPages(): Promise<void> {if (pages.length === 0) return;- const { error } = await supabase_service.from("monitor_check_pages").insert(- pages.map(page => ({- id: uuidv7(),- ...page,- url_hash: page.url_hash ?? hashMonitorUrl(page.url),- })),+ await run(+ () =>+ db.insert(schema.monitor_check_pages).values(+ pages.map(page => ({+ id: uuidv7(),+ ...page,+ url_hash: page.url_hash ?? hashMonitorUrl(page.url),+ })),+ ),+ "Failed to insert monitor check pages",);-- throwIfError(error, "Failed to insert monitor check pages");}export async function listMonitorCheckPages(params: {@@ -511,25 +574,27 @@ export async function listMonitorCheckPages(params: {skip: number;status?: string;}): Promise<any[]> {- let query = supabase_rr_service- .from("monitor_check_pages")- .select("*")- .eq("check_id", params.checkId)- .eq("monitor_id", params.monitorId)- .eq("team_id", params.teamId)- .order("created_at", { ascending: true });-+ const conditions = [+ eq(schema.monitor_check_pages.check_id, params.checkId),+ eq(schema.monitor_check_pages.monitor_id, params.monitorId),+ eq(schema.monitor_check_pages.team_id, params.teamId),+ ];if (params.status) {- query = query.eq("status", params.status);+ conditions.push(eq(schema.monitor_check_pages.status, params.status));}- const { data, error } = await query.range(- params.skip,- params.skip + params.limit - 1,+ const data = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitor_check_pages)+ .where(and(...conditions))+ .orderBy(asc(schema.monitor_check_pages.created_at))+ .limit(params.limit)+ .offset(params.skip),+ "Failed to list monitor check pages",);-- throwIfError(error, "Failed to list monitor check pages");- return data ?? [];+ return data;}export async function countMonitorCheckPages(params: {@@ -537,33 +602,24 @@ export async function countMonitorCheckPages(params: {targetId?: string;status?: string;}): Promise<number> {- const pageSize = 1000;- let total = 0;- let offset = 0;-- while (true) {- let query = supabase_rr_service- .from("monitor_check_pages")- .select("id")- .eq("check_id", params.checkId);-- if (params.targetId) {- query = query.eq("target_id", params.targetId);- }- if (params.status) {- query = query.eq("status", params.status);- }-- const { data, error } = await query.range(offset, offset + pageSize - 1);- throwIfError(error, "Failed to count monitor check pages");-- const batch = data ?? [];- total += batch.length;- if (batch.length < pageSize) break;- offset += pageSize;+ const conditions = [eq(schema.monitor_check_pages.check_id, params.checkId)];+ if (params.targetId) {+ conditions.push(eq(schema.monitor_check_pages.target_id, params.targetId));}+ if (params.status) {+ conditions.push(eq(schema.monitor_check_pages.status, params.status));+ }++ const [row] = await run(+ () =>+ dbRr+ .select({ value: count() })+ .from(schema.monitor_check_pages)+ .where(and(...conditions)),+ "Failed to count monitor check pages",+ );- return total;+ return row?.value ?? 0;}export async function getMonitorPage(params: {@@ -571,16 +627,22 @@ export async function getMonitorPage(params: {targetId: string;url: string;}): Promise<MonitorPageRow | null> {- const { data, error } = await supabase_rr_service- .from("monitor_pages")- .select("*")- .eq("monitor_id", params.monitorId)- .eq("target_id", params.targetId)- .eq("url_hash", hashMonitorUrl(params.url))- .maybeSingle();-- throwIfError(error, "Failed to get monitor page");- return data as MonitorPageRow | null;+ const [data] = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitor_pages)+ .where(+ and(+ eq(schema.monitor_pages.monitor_id, params.monitorId),+ eq(schema.monitor_pages.target_id, params.targetId),+ eq(schema.monitor_pages.url_hash, hashMonitorUrl(params.url)),+ ),+ )+ .limit(1),+ "Failed to get monitor page",+ );+ return (data ?? null) as MonitorPageRow | null;}export async function upsertMonitorPage(params: {@@ -603,34 +665,36 @@ export async function upsertMonitorPage(params: {});if (!existing) {- const { error } = await supabase_service.from("monitor_pages").insert({- monitor_id: params.monitorId,- team_id: params.teamId,- target_id: params.targetId,- url: params.url,- url_hash: hashMonitorUrl(params.url),- source: params.source,- first_seen_check_id: params.checkId,- last_seen_check_id:- params.status === "removed" ? undefined : params.checkId,- last_changed_check_id:- params.status === "changed" || params.status === "new"- ? params.checkId- : undefined,- last_scrape_id: params.scrapeId,- last_status: params.status,- is_removed: params.status === "removed",- removed_at: params.status === "removed" ? now : null,- metadata: params.metadata ?? null,- created_at: now,- updated_at: now,- });-- throwIfError(error, "Failed to insert monitor page");+ await run(+ () =>+ db.insert(schema.monitor_pages).values({+ monitor_id: params.monitorId,+ team_id: params.teamId,+ target_id: params.targetId,+ url: params.url,+ url_hash: hashMonitorUrl(params.url),+ source: params.source,+ first_seen_check_id: params.checkId,+ last_seen_check_id:+ params.status === "removed" ? undefined : params.checkId,+ last_changed_check_id:+ params.status === "changed" || params.status === "new"+ ? params.checkId+ : undefined,+ last_scrape_id: params.scrapeId,+ last_status: params.status,+ is_removed: params.status === "removed",+ removed_at: params.status === "removed" ? now : null,+ metadata: params.metadata ?? null,+ created_at: now,+ updated_at: now,+ }),+ "Failed to insert monitor page",+ );return;}- const patch: Record<string, unknown> = {+ const patch: Partial<typeof schema.monitor_pages.$inferInsert> = {last_status: params.status,is_removed: params.status === "removed",removed_at: params.status === "removed" ? now : null,@@ -645,40 +709,36 @@ export async function upsertMonitorPage(params: {patch.last_changed_check_id = params.checkId;}- const { error } = await supabase_service- .from("monitor_pages")- .update(patch)- .eq("id", existing.id);-- throwIfError(error, "Failed to update monitor page");+ await run(+ () =>+ db+ .update(schema.monitor_pages)+ .set(patch)+ .where(eq(schema.monitor_pages.id, existing.id)),+ "Failed to update monitor page",+ );}export async function listActiveMonitorPages(params: {monitorId: string;targetId: string;}): Promise<MonitorPageRow[]> {- const pageSize = 1000;- const pages: MonitorPageRow[] = [];- let offset = 0;-- while (true) {- const { data, error } = await supabase_rr_service- .from("monitor_pages")- .select("*")- .eq("monitor_id", params.monitorId)- .eq("target_id", params.targetId)- .eq("is_removed", false)- .order("created_at", { ascending: true })- .range(offset, offset + pageSize - 1);-- throwIfError(error, "Failed to list active monitor pages");- const batch = (data ?? []) as MonitorPageRow[];- pages.push(...batch);- if (batch.length < pageSize) break;- offset += pageSize;- }-- return pages;+ const data = await run(+ () =>+ dbRr+ .select()+ .from(schema.monitor_pages)+ .where(+ and(+ eq(schema.monitor_pages.monitor_id, params.monitorId),+ eq(schema.monitor_pages.target_id, params.targetId),+ eq(schema.monitor_pages.is_removed, false),+ ),+ )+ .orderBy(asc(schema.monitor_pages.created_at)),+ "Failed to list active monitor pages",+ );+ return data as MonitorPageRow[];}export async function claimDueMonitors(params: {@@ -686,31 +746,27 @@ export async function claimDueMonitors(params: {limit: number;leaseSeconds: number;}): Promise<MonitorRow[]> {- const { data, error } = await supabase_service.rpc(- "monitoring_claim_due_monitors",- {- p_worker_id: params.workerId,- p_limit: params.limit,- p_lease_seconds: params.leaseSeconds,- },+ const data = await run(+ () => monitoringClaimDueMonitors<MonitorRow>(params),+ "Failed to claim due monitors",);-- throwIfError(error, "Failed to claim due monitors");- return (data ?? []) as MonitorRow[];+ return data;}export async function deferMonitorClaim(monitorId: string,until: Date,): Promise<void> {- const { error } = await supabase_service- .from("monitors")- .update({- locked_until: until.toISOString(),- locked_at: null,- updated_at: new Date().toISOString(),- })- .eq("id", monitorId);-- throwIfError(error, "Failed to defer monitor claim");+ await run(+ () =>+ db+ .update(schema.monitors)+ .set({+ locked_until: until.toISOString(),+ locked_at: null,+ updated_at: new Date().toISOString(),+ })+ .where(eq(schema.monitors.id, monitorId)),+ "Failed to defer monitor claim",+ );}diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex e043a6ef8..4f3d33c35 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -238,7 +238,7 @@ export type MonitorPageRow = {team_id: string;target_id: string;url: string;- url_hash: string;+ url_hash: Buffer;source: MonitorPageSource;first_seen_check_id: string | null;last_seen_check_id: string | null;@@ -267,7 +267,7 @@ export type MonitorCheckPageInsert = {team_id: string;target_id: string;url: string;- url_hash?: string;+ url_hash?: Buffer;status: MonitorPageStatus;previous_scrape_id?: string | null;current_scrape_id?: string | null;diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.tsindex c91b2a846..dbda0a2ec 100644--- a/apps/api/src/services/notification/email_notification.ts+++ b/apps/api/src/services/notification/email_notification.ts@@ -1,4 +1,6 @@-import { supabase_service } from "../supabase";+import { and, eq, gte, lte } from "drizzle-orm";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";import { config } from "../../config";import { withAuth } from "../../lib/withAuth";import { Resend } from "resend";@@ -104,25 +106,35 @@ async function sendEmailNotification(try {// Get user's email preferences- const { data: user, error: userError } = await supabase_service- .from("users")- .select("id")- .eq("email", email)- .single();-- if (userError) {+ let user: { id: string } | undefined;+ try {+ [user] = await db+ .select({ id: schema.users.id })+ .from(schema.users)+ .where(eq(schema.users.email, email))+ .limit(1);+ } catch (userError) {logger.debug(`Error fetching user: ${userError}`);return { success: false };}+ if (!user) {+ return { success: false };+ }// Check user's email preferences- const { data: preferences, error: prefError } = await supabase_service- .from("notification_preferences")- .select("unsubscribed_all, email_preferences")- .eq("user_id", user.id)- .single();-- if (prefError) {+ let preferences:+ | { unsubscribed_all: boolean | null; email_preferences: string[] | null }+ | undefined;+ try {+ [preferences] = await db+ .select({+ unsubscribed_all: schema.notification_preferences.unsubscribed_all,+ email_preferences: schema.notification_preferences.email_preferences,+ })+ .from(schema.notification_preferences)+ .where(eq(schema.notification_preferences.user_id, user.id))+ .limit(1);+ } catch (prefError) {logger.debug(`Error fetching preferences: ${prefError}`);return { success: false };}@@ -214,14 +226,25 @@ async function sendNotificationInternal(const fifteenDaysAgo = new Date();fifteenDaysAgo.setDate(fifteenDaysAgo.getDate() - 15);- const { data, error } = await supabase_service- .from("user_notifications")- .select("*")- .eq("team_id", team_id)- .eq("notification_type", notificationType)- .gte("sent_date", fifteenDaysAgo.toISOString());-- if (error) {+ let data: { id: string }[];+ try {+ data = await db+ .select({ id: schema.user_notifications.id })+ .from(schema.user_notifications)+ .where(+ and(+ eq(schema.user_notifications.team_id, team_id),+ eq(+ schema.user_notifications.notification_type,+ notificationType,+ ),+ gte(+ schema.user_notifications.sent_date,+ fifteenDaysAgo.toISOString(),+ ),+ ),+ );+ } catch (error) {logger.debug(`Error fetching notifications: ${error}`);return { success: false };}@@ -232,18 +255,30 @@ async function sendNotificationInternal(// TODO: observation: Free credits people are not receiving notifications- const { data: recentData, error: recentError } = await supabase_service- .from("user_notifications")- .select("*")- .eq("team_id", team_id)- .eq("notification_type", notificationType)- .gte("sent_date", startDateString)- .lte("sent_date", endDateString);-- if (recentError) {- logger.debug(- `Error fetching recent notifications: ${recentError.message}`,- );+ let recentData: { id: string }[];+ try {+ recentData = await db+ .select({ id: schema.user_notifications.id })+ .from(schema.user_notifications)+ .where(+ and(+ eq(schema.user_notifications.team_id, team_id),+ eq(+ schema.user_notifications.notification_type,+ notificationType,+ ),+ gte(+ schema.user_notifications.sent_date,+ startDateString as string,+ ),+ lte(+ schema.user_notifications.sent_date,+ endDateString as string,+ ),+ ),+ );+ } catch (recentError) {+ logger.debug(`Error fetching recent notifications: ${recentError}`);return { success: false };}@@ -266,32 +301,36 @@ async function sendNotificationInternal(});}// get the emails from the user with the team_id- const { data: emails, error: emailsError } = await supabase_service- .from("users")- .select("email")- .eq("team_id", team_id);-- if (emailsError) {+ let emails: { email: string | null }[];+ try {+ emails = await db+ .select({ email: schema.users.email })+ .from(schema.users)+ .where(eq(schema.users.team_id, team_id));+ } catch (emailsError) {logger.debug(`Error fetching emails: ${emailsError}`);return { success: false };}if (!is_ledger_enabled) {for (const email of emails) {- await sendEmailNotification(email.email, notificationType, context);+ if (email.email) {+ await sendEmailNotification(email.email, notificationType, context);+ }}}- const { error: insertError } = await supabase_service- .from("user_notifications")- .insert([- {- team_id: team_id,- notification_type: notificationType,- sent_date: new Date().toISOString(),- timestamp: new Date().toISOString(),- },- ]);+ let insertError: unknown = null;+ try {+ await db.insert(schema.user_notifications).values({+ team_id: team_id,+ notification_type: notificationType,+ sent_date: new Date().toISOString(),+ timestamp: new Date().toISOString(),+ });+ } catch (error) {+ insertError = error;+ }if (config.SLACK_ADMIN_WEBHOOK_URL && emails.length > 0) {sendSlackWebhook(@@ -352,15 +391,19 @@ export async function sendNotificationWithCustomDays(now.getTime() - daysBetweenEmails * 24 * 60 * 60 * 1000,);- const { data: recentNotifications, error: recentNotificationsError } =- await supabase_service- .from("user_notifications")- .select("*")- .eq("team_id", team_id)- .eq("notification_type", notificationType)- .gte("sent_date", pastDate.toISOString());-- if (recentNotificationsError) {+ let recentNotifications: { id: string }[];+ try {+ recentNotifications = await db+ .select({ id: schema.user_notifications.id })+ .from(schema.user_notifications)+ .where(+ and(+ eq(schema.user_notifications.team_id, team_id),+ eq(schema.user_notifications.notification_type, notificationType),+ gte(schema.user_notifications.sent_date, pastDate.toISOString()),+ ),+ );+ } catch (recentNotificationsError) {logger.debug(`Error fetching recent notifications: ${recentNotificationsError}`,);@@ -385,12 +428,13 @@ export async function sendNotificationWithCustomDays(`Sending notification for team_id: ${team_id} and notificationType: ${notificationType}`,);// get the emails from the user with the team_id- const { data: emails, error: emailsError } = await supabase_service- .from("users")- .select("email")- .eq("team_id", team_id);-- if (emailsError) {+ let emails: { email: string | null }[];+ try {+ emails = await db+ .select({ email: schema.users.email })+ .from(schema.users)+ .where(eq(schema.users.team_id, team_id));+ } catch (emailsError) {logger.debug(`Error fetching emails: ${emailsError}`);await redisEvictConnection.del(redisKey); // free up redis, let it try againreturn { success: false };@@ -408,20 +452,23 @@ export async function sendNotificationWithCustomDays(if (!is_ledger_enabled) {for (const email of emails) {- await sendEmailNotification(email.email, notificationType);+ if (email.email) {+ await sendEmailNotification(email.email, notificationType);+ }}}- const { error: insertError } = await supabase_service- .from("user_notifications")- .insert([- {- team_id: team_id,- notification_type: notificationType,- sent_date: new Date().toISOString(),- timestamp: new Date().toISOString(),- },- ]);+ let insertError: unknown = null;+ try {+ await db.insert(schema.user_notifications).values({+ team_id: team_id,+ notification_type: notificationType,+ sent_date: new Date().toISOString(),+ timestamp: new Date().toISOString(),+ });+ } catch (error) {+ insertError = error;+ }if (config.SLACK_ADMIN_WEBHOOK_URL &&diff --git a/apps/api/src/services/notification/monitoring_email.test.ts b/apps/api/src/services/notification/monitoring_email.test.tsindex a568abf0d..1757f4e60 100644--- a/apps/api/src/services/notification/monitoring_email.test.ts+++ b/apps/api/src/services/notification/monitoring_email.test.ts@@ -1,8 +1,3 @@-jest.mock("../supabase", () => ({- supabase_service: {},- supabase_rr_service: {},-}));-const mockEnsureRecipient = jest.fn();const mockListRecipients = jest.fn();const mockTouchNotified = jest.fn();@@ -70,7 +65,8 @@ function fakeRecipient(status,token: `tok-${sanitized}`,source,- confirmation_sent_at: status === "pending" ? new Date().toISOString() : null,+ confirmation_sent_at:+ status === "pending" ? new Date().toISOString() : null,confirmed_at: status === "confirmed" ? new Date().toISOString() : null,unsubscribed_at:status === "unsubscribed" ? new Date().toISOString() : null,@@ -200,7 +196,7 @@ describe("sendMonitoringConfirmationEmail", () => {expect(mockResendSend).toHaveBeenCalledTimes(1);expect(mockResendSend.mock.calls[0][0].to).toBe("[email protected]");expect(mockResendSend.mock.calls[0][0].subject).toContain(- 'Confirm subscription',+ "Confirm subscription",);expect(mockMarkConfirmationSent).toHaveBeenCalledWith("rec-newexamplecom");});diff --git a/apps/api/src/services/notification/monitoring_email.ts b/apps/api/src/services/notification/monitoring_email.tsindex 7b8a8a446..2b85e4bc9 100644--- a/apps/api/src/services/notification/monitoring_email.ts+++ b/apps/api/src/services/notification/monitoring_email.ts@@ -1,8 +1,10 @@import { Resend } from "resend";import escapeHtml from "escape-html";+import { eq } from "drizzle-orm";import { config } from "../../config";import { logger as _logger } from "../../lib/logger";-import { supabase_service } from "../supabase";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";import type { MonitorCheckRow, MonitorRow } from "../monitoring/types";import {ensureMonitorEmailRecipient,@@ -88,31 +90,39 @@ export type MonitoringEmailPayload = {};async function getTeamEmails(teamId: string): Promise<string[]> {- const { data, error } = await supabase_service- .from("user_teams")- .select(- "users(email, id, notification_preferences(unsubscribed_all, email_preferences))",- )- .eq("team_id", teamId);-- if (error) {+ let rows: {+ email: string | null;+ unsubscribed_all: boolean | null;+ email_preferences: string[] | null;+ }[];+ try {+ rows = await db+ .select({+ email: schema.users.email,+ unsubscribed_all: schema.notification_preferences.unsubscribed_all,+ email_preferences: schema.notification_preferences.email_preferences,+ })+ .from(schema.user_teams)+ .innerJoin(schema.users, eq(schema.user_teams.user_id, schema.users.id))+ .leftJoin(+ schema.notification_preferences,+ eq(schema.notification_preferences.user_id, schema.users.id),+ )+ .where(eq(schema.user_teams.team_id, teamId));+ } catch (error) {logger.warn("Failed to load team emails", { error, teamId });return [];}const emails = new Set<string>();- for (const row of data ?? []) {- const user = (row as any).users;- const email = user?.email;+ for (const row of rows) {+ const email = row.email;if (!email) continue;- const prefs = Array.isArray(user.notification_preferences)- ? user.notification_preferences[0]- : user.notification_preferences;- if (prefs?.unsubscribed_all) continue;+ if (row.unsubscribed_all) continue;if (- Array.isArray(prefs?.email_preferences) &&- !prefs.email_preferences.includes("system_alerts")+ Array.isArray(row.email_preferences) &&+ !row.email_preferences.includes("system_alerts")) {continue;}@@ -266,13 +276,10 @@ export async function sendMonitoringConfirmationEmail(params: {}): Promise<{ attempted: boolean; success: boolean; error?: string }> {const resend = getResendClient();if (!resend) {- logger.warn(- "Skipping monitor opt-in email; RESEND_API_KEY is not set",- {- monitorId: params.recipient.monitor_id,- recipientId: params.recipient.id,- },- );+ logger.warn("Skipping monitor opt-in email; RESEND_API_KEY is not set", {+ monitorId: params.recipient.monitor_id,+ recipientId: params.recipient.id,+ });return { attempted: false, success: true };}@@ -366,10 +373,13 @@ async function resolveSendableRecipients(}),);rows = legacyRows;- logger.info("Bootstrapped legacy monitor recipients without DB backfill", {- monitorId: monitor.id,- recipients: explicitConfigured,- });+ logger.info(+ "Bootstrapped legacy monitor recipients without DB backfill",+ {+ monitorId: monitor.id,+ recipients: explicitConfigured,+ },+ );}const rowsByEmail = new Map(rows.map(r => [r.email, r]));@@ -514,16 +524,13 @@ export async function sendMonitoringEmailSummary(params: {const resolved = await resolveSendableRecipients(params.monitor);if (resolved.confirmedRecipients.length === 0) {- logger.info(- "Skipping monitoring email summary; no confirmed recipients",- {- monitorId: params.monitor.id,- checkId: params.check.id,- configured: resolved.total,- pending: resolved.pending,- unsubscribed: resolved.unsubscribed,- },- );+ logger.info("Skipping monitoring email summary; no confirmed recipients", {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ configured: resolved.total,+ pending: resolved.pending,+ unsubscribed: resolved.unsubscribed,+ });return {attempted: false,success: true,@@ -589,8 +596,7 @@ export async function sendMonitoringEmailSummary(params: {return {recipient,success: false,- error:- typeof error === "string" ? error : JSON.stringify(error),+ error: typeof error === "string" ? error : JSON.stringify(error),};}return { recipient, success: true };diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 289a544e9..a96a70961 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -459,7 +459,7 @@ app.listen(workerPort, () => {initializeEngineForcing();- if (config.USE_DB_AUTHENTICATION) {+ if (config.USE_DB_AUTHENTICATION && !config.DISABLE_MONITORING) {monitorSchedulerInterval = setInterval(() => {enqueueDueMonitorChecks().catch(error => {_logger.error("Failed to enqueue due monitor checks", { error });diff --git a/apps/api/src/services/subscription/enterprise-check.ts b/apps/api/src/services/subscription/enterprise-check.tsindex 577eaab85..cfd9e6c46 100644--- a/apps/api/src/services/subscription/enterprise-check.ts+++ b/apps/api/src/services/subscription/enterprise-check.ts@@ -1,39 +1,38 @@-import { supabase_service } from "../supabase";--interface SubscriptionResponse {- prices: {- products: {- is_enterprise: boolean;- };- };-}+import { and, eq, gt } from "drizzle-orm";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";const RATE_LIMIT_CHANGE_NOTIFICATION_START_DATE = new Date("2025-03-12");export async function isEnterpriseTeamCreatedAfterRateLimitChange(team_id: string,): Promise<boolean> {- const { data, error } = (await supabase_service- .from("subscriptions")- .select("prices(products(is_enterprise))")- .eq("status", "active")- .eq("team_id", team_id)- .gt(- "created",- RATE_LIMIT_CHANGE_NOTIFICATION_START_DATE.toISOString(),- )) as {- data: SubscriptionResponse[] | null;- error: any;- };+ try {+ const data = await db+ .select({ is_enterprise: schema.products.is_enterprise })+ .from(schema.subscriptions)+ .innerJoin(+ schema.prices,+ eq(schema.subscriptions.price_id, schema.prices.id),+ )+ .innerJoin(+ schema.products,+ eq(schema.prices.product_id, schema.products.id),+ )+ .where(+ and(+ eq(schema.subscriptions.status, "active"),+ eq(schema.subscriptions.team_id, team_id),+ gt(+ schema.subscriptions.created,+ RATE_LIMIT_CHANGE_NOTIFICATION_START_DATE.toISOString(),+ ),+ ),+ );- if (error || !data) {+ return data.some(sub => sub.is_enterprise === true);+ } catch (error) {// If there's an error or no subscription found, assume non-enterprisereturn false;}-- const isEnterprise = data.find(- sub => sub.prices?.products?.is_enterprise === true,- );-- return !!isEnterprise;}diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.tsdeleted file mode 100644index 224c58c25..000000000--- a/apps/api/src/services/supabase.ts+++ /dev/null@@ -1,99 +0,0 @@-import { createClient, SupabaseClient } from "@supabase/supabase-js";-import { configDotenv } from "dotenv";-import { config } from "../config";-import { logger } from "../lib/logger";-configDotenv();--/** PostgREST code when .single() returns 0 or >1 rows. Use to distinguish "no row" from real DB errors. */-const POSTGREST_NO_ROWS_CODE = "PGRST116";--export function isPostgrestNoRowsError(- error: { code?: string } | null | undefined,-): boolean {- return error?.code === POSTGREST_NO_ROWS_CODE;-}--// SupabaseService class initializes the Supabase client conditionally based on environment variables.-class SupabaseService {- private client: SupabaseClient | null = null;- private rrClient: SupabaseClient | null = null;-- constructor() {- const supabaseUrl = config.SUPABASE_URL;- const supabaseReplicaUrl = config.SUPABASE_REPLICA_URL;- const supabaseServiceToken = config.SUPABASE_SERVICE_TOKEN;- const useDbAuthentication = config.USE_DB_AUTHENTICATION;- // Only initialize the Supabase client if both URL and Service Token are provided.- if (!useDbAuthentication) {- // Warn the user that Authentication is disabled by setting the client to null- logger.warn(- "Authentication is disabled. Supabase client will not be initialized.",- );- this.client = null;- } else if (!supabaseUrl || !supabaseServiceToken || !supabaseReplicaUrl) {- logger.error(- "Supabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable",- );- } else {- this.client = createClient(supabaseUrl, supabaseServiceToken, {- global: {- headers: {- "sb-lb-routing-mode": "alpha-all-services",- },- },- });-- this.rrClient = createClient(supabaseReplicaUrl, supabaseServiceToken);- }- }-- // Provides access to the initialized Supabase client, if available.- getClient(): SupabaseClient | null {- return this.client;- }-- getRRClient(): SupabaseClient | null {- return this.rrClient;- }-}--const serv = new SupabaseService();--// Using a Proxy to handle dynamic access to the Supabase client or service methods.-// This approach ensures that if Supabase is not configured, any attempt to use it will result in a clear error.-export const supabase_service: SupabaseClient = new Proxy(serv, {- get: function (target, prop, receiver) {- const client = target.getClient();- // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.- if (client === null) {- return () => {- throw new Error("Supabase client is not configured.");- };- }- // Direct access to SupabaseService properties takes precedence.- if (prop in target) {- return Reflect.get(target, prop, receiver);- }- // Otherwise, delegate access to the Supabase client.- return Reflect.get(client, prop, receiver);- },-}) as unknown as SupabaseClient;--export const supabase_rr_service: SupabaseClient = new Proxy(serv, {- get: function (target, prop, receiver) {- const client = target.getRRClient();- // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.- if (client === null) {- return () => {- throw new Error("Supabase RR client is not configured.");- };- }- // Direct access to SupabaseService properties takes precedence.- if (prop in target) {- return Reflect.get(target, prop, receiver);- }- // Otherwise, delegate access to the Supabase client.- return Reflect.get(client, prop, receiver);- },-}) as unknown as SupabaseClient;-diff --git a/apps/api/src/services/webhook/config.ts b/apps/api/src/services/webhook/config.tsindex f59ad3eca..3f6cda422 100644--- a/apps/api/src/services/webhook/config.ts+++ b/apps/api/src/services/webhook/config.ts@@ -1,4 +1,6 @@-import { supabase_rr_service } from "../supabase";+import { eq } from "drizzle-orm";+import { dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { WebhookConfig } from "./types";import { config } from "../../config";@@ -39,14 +41,13 @@ async function getHmacSecret(teamId: string): Promise<string | undefined> {}try {- const { data, error } = await supabase_rr_service- .from("teams")- .select("hmac_secret")- .eq("id", teamId)- .limit(1)- .single();+ const [data] = await dbRr+ .select({ hmac_secret: schema.teams.hmac_secret })+ .from(schema.teams)+ .where(eq(schema.teams.id, teamId))+ .limit(1);- return error ? undefined : data?.hmac_secret;+ return data?.hmac_secret ?? undefined;} catch {return undefined;}diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.tsindex 93c06e3ca..60afbbecb 100644--- a/apps/api/src/services/webhook/delivery.ts+++ b/apps/api/src/services/webhook/delivery.ts@@ -13,7 +13,8 @@ import type {WebhookQueueMessage,} from "./types";import { redisEvictConnection } from "../redis";-import { supabase_service } from "../supabase";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";import { webhookQueue } from "./queue";import { randomUUID } from "crypto";@@ -254,7 +255,7 @@ export async function processWebhookInsertJobs() {});try {- await supabase_service.from("webhook_logs").insert(parsedJobs);+ await db.insert(schema.webhook_logs).values(parsedJobs);_logger.info("Webhook inserter inserted jobs", {jobCount: parsedJobs.length,});diff --git a/apps/api/src/services/webhook/logs.ts b/apps/api/src/services/webhook/logs.tsindex af69ac1d1..77117afaa 100644--- a/apps/api/src/services/webhook/logs.ts+++ b/apps/api/src/services/webhook/logs.ts@@ -1,4 +1,6 @@-import { supabase_rr_service } from "../supabase";+import { and, desc, eq, inArray } from "drizzle-orm";+import { dbRr } from "../../db/connection";+import * as schema from "../../db/schema";import { logger as _logger } from "../../lib/logger";export type WebhookLogRow = {@@ -16,15 +18,29 @@ export async function getLatestWebhookLog(params: {jobId: string;event: string;}): Promise<WebhookLogRow | null> {- const { data, error } = await supabase_rr_service- .from("webhook_logs")- .select("id,success,error,status_code,latency_ms,url,event,created_at")- .eq("crawl_id", params.jobId)- .eq("event", params.event)- .order("created_at", { ascending: false })- .limit(1)- .maybeSingle();- if (error) {+ try {+ const [data] = await dbRr+ .select({+ id: schema.webhook_logs.id,+ success: schema.webhook_logs.success,+ error: schema.webhook_logs.error,+ status_code: schema.webhook_logs.status_code,+ latency_ms: schema.webhook_logs.latency_ms,+ url: schema.webhook_logs.url,+ event: schema.webhook_logs.event,+ created_at: schema.webhook_logs.created_at,+ })+ .from(schema.webhook_logs)+ .where(+ and(+ eq(schema.webhook_logs.crawl_id, params.jobId),+ eq(schema.webhook_logs.event, params.event),+ ),+ )+ .orderBy(desc(schema.webhook_logs.created_at))+ .limit(1);+ return (data ?? null) as WebhookLogRow | null;+ } catch (error) {_logger.warn("Failed to fetch latest webhook log", {module: "webhook-logs",jobId: params.jobId,@@ -33,7 +49,6 @@ export async function getLatestWebhookLog(params: {});return null;}- return data as WebhookLogRow | null;}export async function getLatestWebhookLogsByJob(params: {@@ -43,16 +58,29 @@ export async function getLatestWebhookLogsByJob(params: {const result = new Map<string, WebhookLogRow>();if (params.jobIds.length === 0) return result;- const { data, error } = await supabase_rr_service- .from("webhook_logs")- .select(- "id,success,error,status_code,latency_ms,url,event,created_at,crawl_id",- )- .in("crawl_id", params.jobIds)- .eq("event", params.event)- .order("created_at", { ascending: false });-- if (error) {+ let data: (WebhookLogRow & { crawl_id: string })[];+ try {+ data = await dbRr+ .select({+ id: schema.webhook_logs.id,+ success: schema.webhook_logs.success,+ error: schema.webhook_logs.error,+ status_code: schema.webhook_logs.status_code,+ latency_ms: schema.webhook_logs.latency_ms,+ url: schema.webhook_logs.url,+ event: schema.webhook_logs.event,+ created_at: schema.webhook_logs.created_at,+ crawl_id: schema.webhook_logs.crawl_id,+ })+ .from(schema.webhook_logs)+ .where(+ and(+ inArray(schema.webhook_logs.crawl_id, params.jobIds),+ eq(schema.webhook_logs.event, params.event),+ ),+ )+ .orderBy(desc(schema.webhook_logs.created_at));+ } catch (error) {_logger.warn("Failed to fetch webhook logs batch", {module: "webhook-logs",event: params.event,@@ -61,7 +89,7 @@ export async function getLatestWebhookLogsByJob(params: {return result;}- for (const row of (data ?? []) as (WebhookLogRow & { crawl_id: string })[]) {+ for (const row of data) {if (!result.has(row.crawl_id)) result.set(row.crawl_id, row);}return result;diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex d35926142..d3ea7aa6f 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -6,7 +6,7 @@ import {getDoneJobsOrderedLength,} from "../../lib/crawl-redis";import { getCrawl } from "../../lib/crawl-redis";-import { supabase_service } from "../supabase";+import { creditsBilledByCrawlId } from "../../db/rpc";import { getJobs } from "../../controllers/v1/crawl-status";import { logCrawl, logBatchScrape } from "../logging/log_job";import { createWebhookSender, WebhookEvent } from "../webhook/index";@@ -129,22 +129,18 @@ export async function finishCrawlSuper(job: NuQJob<any>) {} else {const num_docs = await getDoneJobsOrderedLength(crawlId);- let credits_billed = null;+ let credits_billed: number | null = null;if (config.USE_DB_AUTHENTICATION) {- const creditsRpc = await supabase_service.rpc(- "credits_billed_by_crawl_id_2",- {- i_crawl_id: crawlId,- },- );-- credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;+ try {+ const creditsRows = await creditsBilledByCrawlId(crawlId);+ credits_billed = creditsRows?.[0]?.credits_billed ?? null;+ } catch (error) {+ logger.warn("Credits billed is null", { error });+ }if (credits_billed === null) {- logger.warn("Credits billed is null", {- error: creditsRpc.error,- });+ logger.warn("Credits billed is null", {});}}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex cfe39abae..849e19aa4 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -314,6 +314,21 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {document: doc,};+ // Ensure the parent `requests` row is committed before any child+ // `scrapes`/`parses` insert, to avoid a request_id FK violation. Runs on+ // both the crawl and single-scrape paths; only single scrapes actually+ // carry a logRequestPromise.+ if (job.data.logRequestPromise) {+ const start = Date.now();+ await job.data.logRequestPromise;+ const waited = Date.now() - start;+ if (waited > 0) {+ logger.warn("Had to wait for log request promise to complete", {+ timeMs: waited,+ });+ }+ }+if (job.data.crawl_id) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 7d4775380..172171645 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -62,6 +62,8 @@ type ScrapeJobSingleUrlsUnique = {sentry?: any;is_extract?: boolean;apiKeyId: number | null;++ logRequestPromise?: Promise<any>;};export type ScrapeJobSingleUrls = ScrapeJobCommon & ScrapeJobSingleUrlsUnique;diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.jsonindex b28b89b70..4b17b82b6 100644--- a/apps/api/tsconfig.json+++ b/apps/api/tsconfig.json@@ -12,6 +12,7 @@"outDir": "./dist/src","moduleResolution": "NodeNext","strictNullChecks": true,+ "skipLibCheck": true,"inlineSources": true,diff --git a/requests.http b/requests.httpdeleted file mode 100644index e69de29bb..000000000