Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): move from Supabase SDK to Drizzle (#3698)

Gergő Móricz0566d52a6a45
diff --git a/apps/api/knip.config.ts b/apps/api/knip.config.ts
index ee1316465..9e9b4eee4 100644
--- a/apps/api/knip.config.ts
+++ b/apps/api/knip.config.ts
@@ -15,10 +15,6 @@ const config: KnipConfig = {
ignore: [
"native/**",
"src/scraper/scrapeURL/engines/fire-engine/branding-script/**",
- // Legacy auto-recharge files — kept but disabled (Autumn handles auto-recharge now)
- "src/services/billing/auto_charge.ts",
- "src/services/billing/issue_credits.ts",
- "src/services/billing/stripe.ts",
// Superseded by the integration proxy added in #3520; kept for now.
"src/controllers/v0/admin/rotate-api-key.ts",
],
diff --git a/apps/api/package.json b/apps/api/package.json
index 41f4628a9..6ef5b5c73 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -93,7 +93,6 @@
"@openrouter/ai-sdk-provider": "^0.4.5",
"@sentry/cli": "^2.58.2",
"@sentry/node": "^10.27.0",
- "@supabase/supabase-js": "^2.52.0",
"@types/ws": "^8.5.12",
"@x402/core": "^2.4.0",
"@x402/evm": "^2.4.0",
@@ -112,6 +111,7 @@
"cors": "^2.8.5",
"culori": "^4.0.2",
"dotenv": "^16.3.1",
+ "drizzle-orm": "1.0.0-rc.3",
"esbuild": "^0.27.2",
"escape-html": "^1.0.3",
"express": "4.22.0",
@@ -179,7 +179,6 @@
"msgpackr-extract",
"oxc-resolver",
"protobufjs",
- "supabase",
"utf-8-validate",
"wordpos"
],
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 50df44203..d8ded7e20 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -89,9 +89,6 @@ importers:
'@sentry/node':
specifier: ^10.27.0
version: 10.27.0
- '@supabase/supabase-js':
- specifier: ^2.52.0
'@types/ws':
specifier: ^8.5.12
version: 8.5.12
@@ -146,6 +143,9 @@ importers:
dotenv:
specifier: ^16.3.1
version: 16.4.5
+ drizzle-orm:
+ specifier: 1.0.0-rc.3
esbuild:
specifier: ^0.27.2
version: 0.27.2
@@ -2616,28 +2616,6 @@ packages:
'@standard-schema/[email protected]':
resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==}
- '@supabase/[email protected]':
- resolution: {integrity: sha512-mMIQHBRc+SKpZFRB2qtupuzulaUhFYupNyxqDj5Jp/LyPvcWvjaJzZzObv6URtL/O6lPxkanASnotGtNpS3H2Q==}
-
- '@supabase/[email protected]':
- resolution: {integrity: sha512-v5GSqb9zbosquTo6gBwIiq7W9eQ7rE5QazsK/ezNiQXdCbY+bH8D9qEaBIkhVvX4ZRW5rP03gEfw5yw9tiq4EQ==}
-
- '@supabase/[email protected]':
- resolution: {integrity: sha512-1ibVeYUacxWYi9i0cf5efil6adJ9WRyZBLivgjs+AUpewx1F3xPi7gLgaASI2SmIQxPoCEjAsLAzKPgMJVgOUQ==}
- engines: {node: 4.x || >=6.0.0}
-
- '@supabase/[email protected]':
- resolution: {integrity: sha512-O4soKqKtZIW3olqmbXXbKugUtByD2jPa8kL2m2c1oozAO11uCcGrRhkZL0kVxjBLrXHE0mdSkFsMj7jDSfyNpw==}
-
- '@supabase/[email protected]':
- resolution: {integrity: sha512-HQKRnwAqdVqJW/P9TjKVK+/ETpW4yQ8tyDPPtRMKOH4Uh3vQD74vmj353CYs8+YwVBKubeUOOEpI9CT8mT4obw==}
-
- '@supabase/[email protected]':
- resolution: {integrity: sha512-asYHcyDR1fKqrMpytAS1zjyEfvxuOIp1CIXX7ji4lHHcJKqyk+sLl/Vxgm4sN6u8zvuUtae9e4kDxQP2qrwWBA==}
-
- '@supabase/[email protected]':
- resolution: {integrity: sha512-jbs3CV1f2+ge7sgBeEduboT9v/uGjF22v0yWi/5/XFn5tbM8MfWRccsMtsDwAwu24XK8H6wt2LJDiNnZLtx/bg==}
-
resolution: {integrity: sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==}
@@ -2772,9 +2750,6 @@ packages:
resolution: {integrity: sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==}
- '@types/[email protected]':
- resolution: {integrity: sha512-PIzZZlEppgrpoT2QgbnDU+MMzuR6BbCjllj0bM70lWoejMeNJAxCchxnv7J3XFkI8MpygtRpzXrIlmWUBclP5A==}
-
resolution: {integrity: sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==}
@@ -3582,6 +3557,122 @@ packages:
resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==}
engines: {node: '>=12'}
+ resolution: {integrity: sha512-akZOa5UxapFbdBG8IDkfBRpSZJpMHaOJtGgp7oi1oHaiU8S3KN92waHo2l5aRuv1D9tMGYpv3BQFOsiGcNjLTQ==}
+ peerDependencies:
+ '@aws-sdk/client-rds-data': '>=3'
+ '@cloudflare/workers-types': '>=4'
+ '@effect/sql-pg': '>=4.0.0-beta.58 || >=4.0.0'
+ '@electric-sql/pglite': '>=0.2.0'
+ '@libsql/client': '>=0.10.0'
+ '@libsql/client-wasm': '>=0.10.0'
+ '@neondatabase/serverless': '>=0.10.0'
+ '@op-engineering/op-sqlite': '>=2'
+ '@opentelemetry/api': ^1.4.1
+ '@planetscale/database': '>=1.13'
+ '@sinclair/typebox': '>=0.34.8'
+ '@sqlitecloud/drivers': '>=1.0.653'
+ '@tidbcloud/serverless': '*'
+ '@tursodatabase/database': '>=0.2.1'
+ '@tursodatabase/database-common': '>=0.2.1'
+ '@tursodatabase/database-wasm': '>=0.2.1'
+ '@types/better-sqlite3': '*'
+ '@types/mssql': ^9.1.4
+ '@types/pg': '*'
+ '@types/sql.js': '*'
+ '@upstash/redis': '>=1.34.7'
+ '@vercel/postgres': '>=0.8.0'
+ '@xata.io/client': '*'
+ arktype: '>=2.0.0'
+ better-sqlite3: '>=9.3.0'
+ bun-types: '*'
+ effect: '>=4.0.0-beta.58 || >=4.0.0'
+ expo-sqlite: '>=14.0.0'
+ mssql: ^11.0.1
+ mysql2: '>=2'
+ pg: '>=8'
+ postgres: '>=3'
+ sql.js: '>=1'
+ sqlite3: '>=5'
+ typebox: '>=1.0.0'
+ valibot: '>=1.0.0-beta.7'
+ zod: ^3.25.0 || ^4.0.0
+ peerDependenciesMeta:
+ '@aws-sdk/client-rds-data':
+ optional: true
+ '@cloudflare/workers-types':
+ optional: true
+ '@effect/sql-pg':
+ optional: true
+ '@electric-sql/pglite':
+ optional: true
+ '@libsql/client':
+ optional: true
+ '@libsql/client-wasm':
+ optional: true
+ '@neondatabase/serverless':
+ optional: true
+ '@op-engineering/op-sqlite':
+ optional: true
+ '@opentelemetry/api':
+ optional: true
+ '@planetscale/database':
+ optional: true
+ '@sinclair/typebox':
+ optional: true
+ '@sqlitecloud/drivers':
+ optional: true
+ '@tidbcloud/serverless':
+ optional: true
+ '@tursodatabase/database':
+ optional: true
+ '@tursodatabase/database-common':
+ optional: true
+ '@tursodatabase/database-wasm':
+ optional: true
+ '@types/better-sqlite3':
+ optional: true
+ '@types/mssql':
+ optional: true
+ '@types/pg':
+ optional: true
+ '@types/sql.js':
+ optional: true
+ '@upstash/redis':
+ optional: true
+ '@vercel/postgres':
+ optional: true
+ '@xata.io/client':
+ optional: true
+ arktype:
+ optional: true
+ better-sqlite3:
+ optional: true
+ bun-types:
+ optional: true
+ effect:
+ optional: true
+ expo-sqlite:
+ optional: true
+ mssql:
+ optional: true
+ mysql2:
+ optional: true
+ pg:
+ optional: true
+ postgres:
+ optional: true
+ sql.js:
+ optional: true
+ sqlite3:
+ optional: true
+ typebox:
+ optional: true
+ valibot:
+ optional: true
+ zod:
+ optional: true
+
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
engines: {node: '>= 0.4'}
@@ -8264,49 +8355,6 @@ snapshots:
'@standard-schema/[email protected]': {}
- '@supabase/[email protected]':
- dependencies:
- '@supabase/node-fetch': 2.6.15
-
- '@supabase/[email protected]':
- dependencies:
- '@supabase/node-fetch': 2.6.15
-
- '@supabase/[email protected]':
- dependencies:
- whatwg-url: 5.0.0
-
- '@supabase/[email protected]':
- dependencies:
- '@supabase/node-fetch': 2.6.15
-
- dependencies:
- '@supabase/node-fetch': 2.6.15
- '@types/phoenix': 1.6.6
- '@types/ws': 8.18.1
- transitivePeerDependencies:
- - bufferutil
- - utf-8-validate
-
- '@supabase/[email protected]':
- dependencies:
- '@supabase/node-fetch': 2.6.15
-
- dependencies:
- '@supabase/auth-js': 2.71.1
- '@supabase/functions-js': 2.4.5
- '@supabase/node-fetch': 2.6.15
- '@supabase/postgrest-js': 1.19.4
- '@supabase/realtime-js': 2.11.15([email protected])([email protected])
- '@supabase/storage-js': 2.7.1
- transitivePeerDependencies:
- - bufferutil
- - utf-8-validate
-
dependencies:
tslib: 2.8.1
@@ -8470,8 +8518,6 @@ snapshots:
pg-protocol: 1.10.3
pg-types: 2.2.0
- '@types/[email protected]': {}
-
'@types/[email protected]': {}
'@types/[email protected]': {}
@@ -9238,6 +9284,14 @@ snapshots:
+ optionalDependencies:
+ '@opentelemetry/api': 1.9.0
+ '@sinclair/typebox': 0.34.41
+ '@types/pg': 8.15.5
+ pg: 8.16.3([email protected])
+ zod: 4.1.12
+
dependencies:
call-bind-apply-helpers: 1.0.2
diff --git a/apps/api/src/__tests__/e2e_noAuth/index.test.ts b/apps/api/src/__tests__/e2e_noAuth/index.test.ts
index 65bf979f0..2e3af0196 100644
--- a/apps/api/src/__tests__/e2e_noAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_noAuth/index.test.ts
@@ -13,9 +13,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {
beforeAll(() => {
originalConfig = {
USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,
- SUPABASE_ANON_TOKEN: config.SUPABASE_ANON_TOKEN,
- SUPABASE_URL: config.SUPABASE_URL,
- SUPABASE_SERVICE_TOKEN: config.SUPABASE_SERVICE_TOKEN,
+ DATABASE_URL: config.DATABASE_URL,
OPENAI_API_KEY: config.OPENAI_API_KEY,
BULL_AUTH_KEY: config.BULL_AUTH_KEY,
PLAYWRIGHT_MICROSERVICE_URL: config.PLAYWRIGHT_MICROSERVICE_URL,
@@ -23,9 +21,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {
TEST_API_KEY: config.TEST_API_KEY,
};
config.USE_DB_AUTHENTICATION = false;
- config.SUPABASE_ANON_TOKEN = "";
- config.SUPABASE_URL = "";
- config.SUPABASE_SERVICE_TOKEN = "";
+ config.DATABASE_URL = "";
config.OPENAI_API_KEY = "";
config.BULL_AUTH_KEY = "";
config.PLAYWRIGHT_MICROSERVICE_URL = "";
diff --git a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts
index fd6f09f18..0394bfbe0 100644
--- a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts
+++ b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts
@@ -6,10 +6,6 @@ jest.mock("../../services/autumn/autumn.service", () => ({
},
}));
-jest.mock("../../services/supabase", () => ({
- supabase_service: { rpc: jest.fn() },
-}));
-
jest.mock("../../lib/http-metrics", () => ({
httpRequestDurationSeconds: { observe: jest.fn() },
getRoutePattern: jest.fn(() => "/v1/crawl"),
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
index ce17ebb07..c50895b25 100644
--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
@@ -15,11 +15,12 @@ describe("Audio format engine routing (buildFallbackList)", () => {
let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines").buildFallbackList;
const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;
- const originalIndexUrl = process.env.INDEX_SUPABASE_URL;
+ const originalIndexUrl = process.env.INDEX_DATABASE_URL;
beforeAll(() => {
process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";
- process.env.INDEX_SUPABASE_URL = "http://test-index-supabase";
+ process.env.INDEX_DATABASE_URL =
+ "postgresql://postgres:postgres@localhost:5432/postgres";
jest.isolateModules(() => {
buildFallbackList =
@@ -34,9 +35,9 @@ describe("Audio format engine routing (buildFallbackList)", () => {
process.env.FIRE_ENGINE_BETA_URL = originalFireEngineUrl;
}
if (originalIndexUrl === undefined) {
- delete process.env.INDEX_SUPABASE_URL;
+ delete process.env.INDEX_DATABASE_URL;
} else {
- process.env.INDEX_SUPABASE_URL = originalIndexUrl;
+ process.env.INDEX_DATABASE_URL = originalIndexUrl;
}
});
diff --git a/apps/api/src/__tests__/snips/v2/parse.test.ts b/apps/api/src/__tests__/snips/v2/parse.test.ts
index 5aa990956..3d1c3613e 100644
--- a/apps/api/src/__tests__/snips/v2/parse.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parse.test.ts
@@ -6,7 +6,9 @@ import {
TEST_SUITE_WEBSITE,
} from "../lib";
import { idmux, Identity, parse, parseWithFailure, scrapeTimeout } from "./lib";
-import { supabase_service } from "../../../services/supabase";
+import { and, desc, eq } from "drizzle-orm";
+import { db } from "../../../db/connection";
+import * as schema from "../../../db/schema";
import { config } from "../../../config";
const DOCX_FIXTURE_BASE64 =
@@ -212,16 +214,19 @@ describe("/v2/parse", () => {
);
const requestLog = await waitForSingleRow<{ id: string }>(async () => {
- const { data, error } = await supabase_service
- .from("requests")
- .select("id")
- .eq("team_id", identity.teamId)
- .eq("kind", "parse")
- .eq("target_hint", filename)
- .order("created_at", { ascending: false })
+ const data = await db
+ .select({ id: schema.requests.id })
+ .from(schema.requests)
+ .where(
+ and(
+ eq(schema.requests.team_id, identity.teamId),
+ eq(schema.requests.kind, "parse"),
+ eq(schema.requests.target_hint, filename),
+ ),
+ )
+ .orderBy(desc(schema.requests.created_at))
.limit(1);
- if (error) throw error;
- return data?.[0] ?? null;
+ return data[0] ?? null;
});
expect(requestLog).not.toBeNull();
@@ -230,14 +235,16 @@ describe("/v2/parse", () => {
request_id: string;
url: string;
}>(async () => {
- const { data, error } = await supabase_service
- .from("parses")
- .select("request_id, url")
- .eq("request_id", requestLog!.id)
- .order("created_at", { ascending: false })
+ const data = await db
+ .select({
+ request_id: schema.parses.request_id,
+ url: schema.parses.url,
+ })
+ .from(schema.parses)
+ .where(eq(schema.parses.request_id, requestLog!.id))
+ .orderBy(desc(schema.parses.created_at))
.limit(1);
- if (error) throw error;
- return data?.[0] ?? null;
+ return data[0] ?? null;
});
expect(parseLog).not.toBeNull();
@@ -246,14 +253,12 @@ describe("/v2/parse", () => {
`https://parse.firecrawl.dev/uploads/${encodeURIComponent(filename)}`,
);
- const { data: scrapeRows, error: scrapeRowsError } =
- await supabase_service
- .from("scrapes")
- .select("id")
- .eq("request_id", requestLog!.id)
- .limit(1);
- expect(scrapeRowsError).toBeFalsy();
- expect(scrapeRows ?? []).toHaveLength(0);
+ const scrapeRows = await db
+ .select({ id: schema.scrapes.id })
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.request_id, requestLog!.id))
+ .limit(1);
+ expect(scrapeRows).toHaveLength(0);
},
scrapeTimeout,
);
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index df41959bf..6899a3ad1 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -1203,7 +1203,7 @@ describe("Scrape tests", () => {
identity,
);
- expect(response.markdown).toContain("| Country | United States |");
+ expect(response.markdown).toContain("| Country | United States "); // either United States or United States of America
},
scrapeTimeout,
);
@@ -1346,7 +1346,7 @@ describe("Scrape tests", () => {
);
expect(response.markdown).toContain("PDF Test File");
- expect(response.metadata.title).toBe("PDF Test Page");
+ expect(response.metadata.title).toContain("PDF Test Page");
expect(response.metadata.numPages).toBe(1);
},
scrapeTimeout,
@@ -1373,7 +1373,9 @@ describe("Scrape tests", () => {
identity,
);
- expect(response.error).toContain("Insufficient time to process PDF");
+ expect(response.error).toContain(
+ "pages, which requires more processing time than your current timeout allows.",
+ );
},
12000,
);
diff --git a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts b/apps/api/src/__tests__/snips/v2/search-feedback.test.ts
index a36288a94..df36ad307 100644
--- a/apps/api/src/__tests__/snips/v2/search-feedback.test.ts
+++ b/apps/api/src/__tests__/snips/v2/search-feedback.test.ts
@@ -7,7 +7,9 @@ import {
idmux,
Identity,
} from "./lib";
-import { supabase_service } from "../../../services/supabase";
+import { and, eq } from "drizzle-orm";
+import { db } from "../../../db/connection";
+import * as schema from "../../../db/schema";
let identity: Identity;
let secondaryIdentity: Identity;
@@ -28,425 +30,418 @@ beforeAll(async () => {
// Skipped in self-hosted mode: depends on Supabase for the `searches` row
// lookup, Autumn for credit refunds, and the per-team daily refund cap —
// none of which exist in self-hosted setups.
-describeIf(TEST_PRODUCTION)(
- "Search feedback tests",
- () => {
- it.concurrent(
- "records feedback and refunds 1 credit on first submission",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
- expect(typeof raw.body.id).toBe("string");
- expect((raw.body.data?.web ?? []).length).toBeGreaterThan(0);
-
- const result = await searchFeedback(
- raw.body.id,
- {
- rating: "good",
- valuableSources: [
- {
- url: raw.body.data.web[0].url,
- reason: "Most directly answered the question.",
- },
- ],
- missingContent: [
- {
- topic: "Enterprise pricing",
- description:
- "Pricing tier table for the Enterprise plan was not in any result.",
- },
- {
- topic: "SLA terms",
- description: "Uptime SLA and support SLAs not surfaced.",
- },
- ],
- querySuggestions:
- "Include site:firecrawl.dev when the user mentions firecrawl by name.",
- },
- identity,
- );
-
- expect(result.success).toBe(true);
- expect(result.creditsRefunded).toBe(1);
- expect(result.alreadySubmitted).toBeFalsy();
- expect(typeof result.feedbackId).toBe("string");
- },
- 90000,
- );
-
- it.concurrent(
- "is idempotent — second submission returns 0 refund",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl docs", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
- expect(typeof raw.body.id).toBe("string");
-
- const first = await searchFeedback(
- raw.body.id,
- {
- rating: "partial",
- missingContent: [
- { topic: "Recent results", description: "Nothing past 2024." },
- ],
- },
- identity,
- );
- expect(first.creditsRefunded).toBe(1);
-
- const second = await searchFeedback(
- raw.body.id,
- {
- rating: "bad",
- missingContent: [
- { topic: "Recent results", description: "Still nothing past 2024." },
- ],
- },
- identity,
- );
- expect(second.creditsRefunded).toBe(0);
- expect(second.alreadySubmitted).toBe(true);
- },
- 90000,
- );
-
- it.concurrent(
- "rejects feedback for a search owned by another team",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl api", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
- const searchId = raw.body.id;
-
- const failed = await searchFeedbackWithFailure(
- searchId,
- {
- rating: "good",
- valuableSources: [{ url: "https://firecrawl.dev/" }],
- },
- secondaryIdentity,
- );
- expect(failed.error.toLowerCase()).toContain("not found");
- expect((failed as any).feedbackErrorCode).toBe("SEARCH_NOT_FOUND");
- },
- 90000,
- );
-
- it.concurrent(
- "rejects feedback for a non-existent search id",
- async () => {
- const failed = await searchFeedbackWithFailure(
- "00000000-0000-7000-8000-000000000000",
- {
- rating: "bad",
- missingContent: [{ topic: "Anything at all" }],
- },
- identity,
- );
- expect(failed.error.toLowerCase()).toContain("not found");
- expect((failed as any).feedbackErrorCode).toBe("SEARCH_NOT_FOUND");
- },
- 30000,
- );
-
- it.concurrent(
- "rejects an invalid jobId format with 400",
- async () => {
- const raw = await searchFeedbackRaw(
- "not-a-uuid",
- {
- rating: "good",
- valuableSources: [{ url: "https://firecrawl.dev/" }],
- },
- identity,
- );
- expect(raw.statusCode).toBe(400);
- expect(raw.body.success).toBe(false);
- },
- 30000,
- );
-
- it.concurrent(
- "rejects an invalid rating value",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
-
- const failed = await searchFeedbackRaw(
- raw.body.id,
- { rating: "amazing" as any },
- identity,
- );
- expect(failed.statusCode).toBe(400);
- expect(failed.body.success).toBe(false);
- },
- 90000,
- );
-
- it.concurrent(
- "rejects feedback with a non-http URL in valuableSources",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
-
- const failed = await searchFeedbackRaw(
- raw.body.id,
- {
- rating: "good",
- valuableSources: [
- { url: "ftp://example.com/file", reason: "valuable" },
- ],
- },
- identity,
- );
- expect(failed.statusCode).toBe(400);
- expect(failed.body.success).toBe(false);
- },
- 90000,
- );
-
- it.concurrent(
- "rejects 'good' rating without any valuableSources",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
-
- const failed = await searchFeedbackRaw(
- raw.body.id,
- {
- rating: "good",
- missingContent: [
- { topic: "Irrelevant for good rating" },
- ],
- },
- identity,
- );
- expect(failed.statusCode).toBe(400);
- expect(failed.body.success).toBe(false);
- expect(String(failed.body.error).toLowerCase()).toContain(
- "substantive",
- );
- },
- 90000,
- );
-
- it.concurrent(
- "rejects 'partial' rating with no sources and no missing content",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
-
- const failed = await searchFeedbackRaw(
- raw.body.id,
- { rating: "partial" },
- identity,
- );
- expect(failed.statusCode).toBe(400);
- expect(failed.body.success).toBe(false);
- },
- 90000,
- );
-
- it.concurrent(
- "rejects 'bad' rating with no missing content or query suggestions",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
-
- const failed = await searchFeedbackRaw(
- raw.body.id,
- {
- rating: "bad",
- valuableSources: [{ url: "https://firecrawl.dev/" }],
- },
- identity,
- );
- expect(failed.statusCode).toBe(400);
- expect(failed.body.success).toBe(false);
- },
- 90000,
- );
-
- it.concurrent(
- "rejects missingContent entries without a topic",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
-
- const failed = await searchFeedbackRaw(
- raw.body.id,
- {
- rating: "bad",
- // @ts-expect-error testing invalid shape
- missingContent: [{ description: "no topic supplied" }],
- },
- identity,
- );
- expect(failed.statusCode).toBe(400);
- expect(failed.body.success).toBe(false);
- },
- 90000,
- );
-
- it.concurrent(
- "accepts a structured 'partial' rating with multiple missing topics",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl pricing", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
-
- const result = await searchFeedback(
- raw.body.id,
- {
- rating: "partial",
- missingContent: [
- {
- topic: "Enterprise pricing",
- description:
- "Pricing tier table for Enterprise was not surfaced.",
- },
- { topic: "Self-hosted pricing" },
- {
- topic: "Annual discount",
- description: "Annual vs monthly discount comparison.",
- },
- ],
- },
- identity,
+describeIf(TEST_PRODUCTION)("Search feedback tests", () => {
+ it.concurrent(
+ "records feedback and refunds 1 credit on first submission",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+ expect(typeof raw.body.id).toBe("string");
+ expect((raw.body.data?.web ?? []).length).toBeGreaterThan(0);
+
+ const result = await searchFeedback(
+ raw.body.id,
+ {
+ rating: "good",
+ valuableSources: [
+ {
+ url: raw.body.data.web[0].url,
+ reason: "Most directly answered the question.",
+ },
+ ],
+ missingContent: [
+ {
+ topic: "Enterprise pricing",
+ description:
+ "Pricing tier table for the Enterprise plan was not in any result.",
+ },
+ {
+ topic: "SLA terms",
+ description: "Uptime SLA and support SLAs not surfaced.",
+ },
+ ],
+ querySuggestions:
+ "Include site:firecrawl.dev when the user mentions firecrawl by name.",
+ },
+ identity,
+ );
+
+ expect(result.success).toBe(true);
+ expect(result.creditsRefunded).toBe(1);
+ expect(result.alreadySubmitted).toBeFalsy();
+ expect(typeof result.feedbackId).toBe("string");
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "is idempotent — second submission returns 0 refund",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl docs", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+ expect(typeof raw.body.id).toBe("string");
+
+ const first = await searchFeedback(
+ raw.body.id,
+ {
+ rating: "partial",
+ missingContent: [
+ { topic: "Recent results", description: "Nothing past 2024." },
+ ],
+ },
+ identity,
+ );
+ expect(first.creditsRefunded).toBe(1);
+
+ const second = await searchFeedback(
+ raw.body.id,
+ {
+ rating: "bad",
+ missingContent: [
+ {
+ topic: "Recent results",
+ description: "Still nothing past 2024.",
+ },
+ ],
+ },
+ identity,
+ );
+ expect(second.creditsRefunded).toBe(0);
+ expect(second.alreadySubmitted).toBe(true);
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "rejects feedback for a search owned by another team",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl api", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+ const searchId = raw.body.id;
+
+ const failed = await searchFeedbackWithFailure(
+ searchId,
+ {
+ rating: "good",
+ valuableSources: [{ url: "https://firecrawl.dev/" }],
+ },
+ secondaryIdentity,
+ );
+ expect(failed.error.toLowerCase()).toContain("not found");
+ expect((failed as any).feedbackErrorCode).toBe("SEARCH_NOT_FOUND");
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "rejects feedback for a non-existent search id",
+ async () => {
+ const failed = await searchFeedbackWithFailure(
+ "00000000-0000-7000-8000-000000000000",
+ {
+ rating: "bad",
+ missingContent: [{ topic: "Anything at all" }],
+ },
+ identity,
+ );
+ expect(failed.error.toLowerCase()).toContain("not found");
+ expect((failed as any).feedbackErrorCode).toBe("SEARCH_NOT_FOUND");
+ },
+ 30000,
+ );
+
+ it.concurrent(
+ "rejects an invalid jobId format with 400",
+ async () => {
+ const raw = await searchFeedbackRaw(
+ "not-a-uuid",
+ {
+ rating: "good",
+ valuableSources: [{ url: "https://firecrawl.dev/" }],
+ },
+ identity,
+ );
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ },
+ 30000,
+ );
+
+ it.concurrent(
+ "rejects an invalid rating value",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+
+ const failed = await searchFeedbackRaw(
+ raw.body.id,
+ { rating: "amazing" as any },
+ identity,
+ );
+ expect(failed.statusCode).toBe(400);
+ expect(failed.body.success).toBe(false);
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "rejects feedback with a non-http URL in valuableSources",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+
+ const failed = await searchFeedbackRaw(
+ raw.body.id,
+ {
+ rating: "good",
+ valuableSources: [
+ { url: "ftp://example.com/file", reason: "valuable" },
+ ],
+ },
+ identity,
+ );
+ expect(failed.statusCode).toBe(400);
+ expect(failed.body.success).toBe(false);
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "rejects 'good' rating without any valuableSources",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+
+ const failed = await searchFeedbackRaw(
+ raw.body.id,
+ {
+ rating: "good",
+ missingContent: [{ topic: "Irrelevant for good rating" }],
+ },
+ identity,
+ );
+ expect(failed.statusCode).toBe(400);
+ expect(failed.body.success).toBe(false);
+ expect(String(failed.body.error).toLowerCase()).toContain("substantive");
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "rejects 'partial' rating with no sources and no missing content",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+
+ const failed = await searchFeedbackRaw(
+ raw.body.id,
+ { rating: "partial" },
+ identity,
+ );
+ expect(failed.statusCode).toBe(400);
+ expect(failed.body.success).toBe(false);
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "rejects 'bad' rating with no missing content or query suggestions",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+
+ const failed = await searchFeedbackRaw(
+ raw.body.id,
+ {
+ rating: "bad",
+ valuableSources: [{ url: "https://firecrawl.dev/" }],
+ },
+ identity,
+ );
+ expect(failed.statusCode).toBe(400);
+ expect(failed.body.success).toBe(false);
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "rejects missingContent entries without a topic",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+
+ const failed = await searchFeedbackRaw(
+ raw.body.id,
+ {
+ rating: "bad",
+ // @ts-expect-error testing invalid shape
+ missingContent: [{ description: "no topic supplied" }],
+ },
+ identity,
+ );
+ expect(failed.statusCode).toBe(400);
+ expect(failed.body.success).toBe(false);
+ },
+ 90000,
+ );
+
+ it.concurrent(
+ "accepts a structured 'partial' rating with multiple missing topics",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl pricing", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+
+ const result = await searchFeedback(
+ raw.body.id,
+ {
+ rating: "partial",
+ missingContent: [
+ {
+ topic: "Enterprise pricing",
+ description:
+ "Pricing tier table for Enterprise was not surfaced.",
+ },
+ { topic: "Self-hosted pricing" },
+ {
+ topic: "Annual discount",
+ description: "Annual vs monthly discount comparison.",
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(result.success).toBe(true);
+ expect(result.creditsRefunded).toBe(1);
+ },
+ 90000,
+ );
+
+ // TEAM_OPTED_OUT (server-side opt-out via searchFeedbackOptOut flag)
+ // is not covered by E2E because idmux identities can't set ACUC flags.
+
+ it.concurrent(
+ "stops refunding once the team's daily refund cap is reached",
+ async () => {
+ const cappedIdentity = await idmux({
+ name: "search-feedback-cap",
+ concurrency: 100,
+ credits: 1000000,
+ });
+
+ const dailyCap = 100;
+
+ const seedSearchId =
+ "00000000-0000-7000-8000-" +
+ Math.floor(Math.random() * 1e12)
+ .toString(16)
+ .padStart(12, "0");
+ await db.insert(schema.search_feedback).values({
+ search_id: seedSearchId,
+ team_id: cappedIdentity.teamId,
+ overall_rating: "good",
+ valuable_sources: [{ url: "https://firecrawl.dev/" }],
+ missing_content: [],
+ integration: null,
+ origin: "test-seed",
+ credits_refunded: dailyCap,
+ });
+
+ // Now do a real search and submit feedback.
+ const raw = await searchRawFull(
+ { query: "firecrawl daily cap", limit: 3 },
+ cappedIdentity,
+ );
+ expect(raw.statusCode).toBe(200);
+
+ const result = await searchFeedback(
+ raw.body.id,
+ {
+ rating: "good",
+ valuableSources: [{ url: raw.body.data.web[0].url }],
+ },
+ cappedIdentity,
+ );
+
+ expect(result.success).toBe(true);
+ expect(result.creditsRefunded).toBe(0);
+ expect(result.dailyCapReached).toBe(true);
+ expect(result.creditsRefundedToday).toBeGreaterThanOrEqual(dailyCap);
+ expect(result.dailyRefundCap).toBe(dailyCap);
+ expect(String(result.warning ?? "").toLowerCase()).toContain(
+ "daily refund cap",
+ );
+
+ await db
+ .delete(schema.search_feedback)
+ .where(eq(schema.search_feedback.search_id, seedSearchId));
+ },
+ 120000,
+ );
+
+ // Back-date the searches row so we don't have to wait the full window.
+ it.concurrent(
+ "rejects feedback submitted outside the configured time window",
+ async () => {
+ const raw = await searchRawFull(
+ { query: "firecrawl windowed", limit: 3 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+ const searchId = raw.body.id;
+
+ await new Promise(r => setTimeout(r, 750));
+ const aged = new Date(Date.now() - 60 * 60 * 1000).toISOString();
+ await db
+ .update(schema.searches)
+ .set({ created_at: aged })
+ .where(
+ and(
+ eq(schema.searches.id, searchId),
+ eq(schema.searches.team_id, identity.teamId),
+ ),
);
- expect(result.success).toBe(true);
- expect(result.creditsRefunded).toBe(1);
- },
- 90000,
- );
-
- // TEAM_OPTED_OUT (server-side opt-out via searchFeedbackOptOut flag)
- // is not covered by E2E because idmux identities can't set ACUC flags.
-
- it.concurrent(
- "stops refunding once the team's daily refund cap is reached",
- async () => {
- const cappedIdentity = await idmux({
- name: "search-feedback-cap",
- concurrency: 100,
- credits: 1000000,
- });
-
- const dailyCap = 100;
-
- const seedSearchId =
- "00000000-0000-7000-8000-" +
- Math.floor(Math.random() * 1e12)
- .toString(16)
- .padStart(12, "0");
- const { error: seedErr } = await supabase_service
- .from("search_feedback")
- .insert({
- search_id: seedSearchId,
- team_id: cappedIdentity.teamId,
- overall_rating: "good",
- valuable_sources: [{ url: "https://firecrawl.dev/" }],
- missing_content: [],
- integration: null,
- origin: "test-seed",
- credits_refunded: dailyCap,
- });
- expect(seedErr).toBeFalsy();
-
- // Now do a real search and submit feedback.
- const raw = await searchRawFull(
- { query: "firecrawl daily cap", limit: 3 },
- cappedIdentity,
- );
- expect(raw.statusCode).toBe(200);
-
- const result = await searchFeedback(
- raw.body.id,
- {
- rating: "good",
- valuableSources: [{ url: raw.body.data.web[0].url }],
- },
- cappedIdentity,
- );
-
- expect(result.success).toBe(true);
- expect(result.creditsRefunded).toBe(0);
- expect(result.dailyCapReached).toBe(true);
- expect(result.creditsRefundedToday).toBeGreaterThanOrEqual(dailyCap);
- expect(result.dailyRefundCap).toBe(dailyCap);
- expect(String(result.warning ?? "").toLowerCase()).toContain(
- "daily refund cap",
- );
-
- await supabase_service
- .from("search_feedback")
- .delete()
- .eq("search_id", seedSearchId);
- },
- 120000,
- );
-
- // Back-date the searches row so we don't have to wait the full window.
- it.concurrent(
- "rejects feedback submitted outside the configured time window",
- async () => {
- const raw = await searchRawFull(
- { query: "firecrawl windowed", limit: 3 },
- identity,
- );
- expect(raw.statusCode).toBe(200);
- const searchId = raw.body.id;
-
- await new Promise(r => setTimeout(r, 750));
- const aged = new Date(Date.now() - 60 * 60 * 1000).toISOString();
- const { error: updateErr } = await supabase_service
- .from("searches")
- .update({ created_at: aged })
- .eq("id", searchId)
- .eq("team_id", identity.teamId);
- expect(updateErr).toBeFalsy();
-
- const failed = await searchFeedbackWithFailure(
- searchId,
- {
- rating: "good",
- valuableSources: [{ url: "https://firecrawl.dev/" }],
- },
- identity,
- );
- expect((failed as any).feedbackErrorCode).toBe(
- "FEEDBACK_WINDOW_EXPIRED",
- );
- },
- 90000,
- );
- },
-);
+ const failed = await searchFeedbackWithFailure(
+ searchId,
+ {
+ rating: "good",
+ valuableSources: [{ url: "https://firecrawl.dev/" }],
+ },
+ identity,
+ );
+ expect((failed as any).feedbackErrorCode).toBe("FEEDBACK_WINDOW_EXPIRED");
+ },
+ 90000,
+ );
+});
diff --git a/apps/api/src/__tests__/snips/zdr-helpers.ts b/apps/api/src/__tests__/snips/zdr-helpers.ts
index 654c1d924..35da88409 100644
--- a/apps/api/src/__tests__/snips/zdr-helpers.ts
+++ b/apps/api/src/__tests__/snips/zdr-helpers.ts
@@ -1,5 +1,7 @@
import { readFile, stat } from "node:fs/promises";
-import { supabase_service } from "../../services/supabase";
+import { eq } from "drizzle-orm";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
import { getJobFromGCS } from "../../lib/gcs-jobs";
type Identity = { apiKey: string; teamId: string };
@@ -86,16 +88,15 @@ export async function getLogs() {
}
export async function expectScrapeIsCleanedUp(scrapeId: string) {
- const { data: scrapeData, error: scrapeError } = await supabase_service
- .from("scrapes")
- .select("*")
- .eq("id", scrapeId)
+ const scrapeData = await db
+ .select()
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.id, scrapeId))
.limit(1);
- expect(scrapeError).toBeFalsy();
expect(scrapeData).toHaveLength(1);
- if (scrapeData && scrapeData.length === 1) {
+ if (scrapeData.length === 1) {
const record = scrapeData[0];
expect(record.url).not.toContain("://"); // no url stored
expect(record.options).toBeNull();
@@ -103,32 +104,29 @@ export async function expectScrapeIsCleanedUp(scrapeId: string) {
}
export async function expectCrawlIsCleanedUp(crawlId: string) {
- const { data: requestData, error: requestError } = await supabase_service
- .from("requests")
- .select("*")
- .eq("id", crawlId)
+ const requestData = await db
+ .select()
+ .from(schema.requests)
+ .where(eq(schema.requests.id, crawlId))
.limit(1);
- expect(requestError).toBeFalsy();
expect(requestData).toHaveLength(1);
-
- if (requestData && requestData.length === 1) {
+
+ if (requestData.length === 1) {
const record = requestData[0];
- expect(record.url).not.toContain("://"); // no url stored
expect(record.kind).toBe("crawl");
expect(record.dr_clean_by).not.toBeNull();
}
- const { data: crawlData, error: crawlError } = await supabase_service
- .from("crawls")
- .select("*")
- .eq("id", crawlId)
+ const crawlData = await db
+ .select()
+ .from(schema.crawls)
+ .where(eq(schema.crawls.id, crawlId))
.limit(1);
- expect(crawlError).toBeFalsy();
expect(crawlData).toHaveLength(1);
- if (crawlData && crawlData.length === 1) {
+ if (crawlData.length === 1) {
const record = crawlData[0];
expect(record.url).not.toContain("://"); // no url stored
expect(record.options).toBeNull();
@@ -136,65 +134,55 @@ export async function expectCrawlIsCleanedUp(crawlId: string) {
}
export async function expectBatchScrapeIsCleanedUp(batchScrapeId: string) {
- const { data: requestData, error: requestError } = await supabase_service
- .from("requests")
- .select("*")
- .eq("id", batchScrapeId)
+ const requestData = await db
+ .select()
+ .from(schema.requests)
+ .where(eq(schema.requests.id, batchScrapeId))
.limit(1);
- expect(requestError).toBeFalsy();
expect(requestData).toHaveLength(1);
-
- if (requestData && requestData.length === 1) {
+
+ if (requestData.length === 1) {
const record = requestData[0];
- expect(record.url).not.toContain("://"); // no url stored
expect(record.kind).toBe("batch_scrape");
expect(record.dr_clean_by).not.toBeNull();
}
- const { data: batchScrapeData, error: batchScrapeError } = await supabase_service
- .from("batch_scrapes")
- .select("*")
- .eq("id", batchScrapeId)
+ const batchScrapeData = await db
+ .select()
+ .from(schema.batch_scrapes)
+ .where(eq(schema.batch_scrapes.id, batchScrapeId))
.limit(1);
- expect(batchScrapeError).toBeFalsy();
expect(batchScrapeData).toHaveLength(1);
-
- if (batchScrapeData && batchScrapeData.length === 1) {
- const record = batchScrapeData[0];
- expect(record.url).not.toContain("://"); // no url stored
- }
}
export async function expectScrapesOfRequestAreCleanedUp(
requestId: string,
expectedScrapeCount?: number,
) {
- const { data: scrapes, error: scrapesError } = await supabase_service
- .from("scrapes")
- .select("*")
- .eq("request_id", requestId);
-
- expect(scrapesError).toBeFalsy();
+ const scrapes = await db
+ .select()
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.request_id, requestId));
if (expectedScrapeCount !== undefined) {
- expect((scrapes ?? []).length).toBe(expectedScrapeCount);
+ expect(scrapes.length).toBe(expectedScrapeCount);
} else {
- expect((scrapes ?? []).length).toBeGreaterThanOrEqual(1);
+ expect(scrapes.length).toBeGreaterThanOrEqual(1);
}
- for (const scrape of scrapes ?? []) {
+ for (const scrape of scrapes) {
expect(scrape.url).not.toContain("://"); // no url stored
expect(scrape.options).toBeNull();
- if (scrape.success) {
+ if (scrape.is_successful) {
const gcsJob = await getJobFromGCS(scrape.id);
expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry
}
}
- return scrapes ?? [];
+ return scrapes;
}
export async function expectScrapesAreFullyCleanedAfterZDRCleaner(
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index e6209ea62..850973be0 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -81,6 +81,9 @@ const configSchema = z.object({
POSTGRES_DB: z.string().default("postgres"),
POSTGRES_USER: z.string().default("postgres"),
POSTGRES_PASSWORD: z.string().default("postgres"),
+ DATABASE_URL: z.string().optional(),
+ DATABASE_REPLICA_URL: z.string().optional(),
+ INDEX_DATABASE_URL: z.string().optional(),
REDIS_URL: z.string().optional(),
REDIS_EVICT_URL: z.string().optional(),
REDIS_RATE_LIMIT_URL: z.string().optional(),
@@ -88,15 +91,6 @@ const configSchema = z.object({
NUQ_DATABASE_URL_LISTEN: z.string().optional(),
NUQ_RABBITMQ_URL: z.string().optional(),
- // Supabase
- SUPABASE_URL: z.string().optional(),
- SUPABASE_ANON_TOKEN: z.string().optional(),
- SUPABASE_SERVICE_TOKEN: z.string().optional(),
- SUPABASE_REPLICA_URL: z.string().optional(),
- INDEX_SUPABASE_URL: z.string().optional(),
- INDEX_SUPABASE_SERVICE_TOKEN: z.string().optional(),
- SEARCH_INDEX_SUPABASE_URL: z.string().optional(),
-
// Google Cloud Storage
GCS_BUCKET_NAME: z.string().optional(),
GCS_CREDENTIALS: z.string().optional(),
@@ -268,6 +262,7 @@ const configSchema = z.object({
GITHUB_REF_NAME: z.string().optional(),
RESTRICTED_COUNTRIES: delimitedList(",").optional(),
DISABLE_ENGPICKER: z.stringbool().optional(),
+ DISABLE_MONITORING: z.stringbool().default(false),
EXTRACT_V3_BETA_URL: z.string().optional(),
AGENT_INTEROP_SECRET: z.string().optional(),
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 4444268e5..86d116c29 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -10,7 +10,14 @@ import { getRedisConnection } from "../services/queue-service";
import { getRateLimiter } from "../services/rate-limiter";
import { deleteKey, getValue, setValue } from "../services/redis";
import { redlock } from "../services/redlock";
-import { supabase_rr_service, supabase_service } from "../services/supabase";
+import { eq } from "drizzle-orm";
+import { db, dbRr } from "../db/connection";
+import * as schema from "../db/schema";
+import {
+ authCreditUsageChunk,
+ authCreditUsageChunkFromTeam,
+ AuthCreditUsageChunkRow,
+} from "../db/rpc";
import { AuthResponse, RateLimiterMode } from "../types";
import { AuthCreditUsageChunk, AuthCreditUsageChunkFromTeam } from "./v1/types";
@@ -260,45 +267,38 @@ export async function getACUC(
}
if (!cacheOnly) {
- let data;
- let error;
+ let data: AuthCreditUsageChunkRow[] = [];
let retries = 0;
const maxRetries = 5;
while (retries < maxRetries) {
- const client =
- Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;
- ({ data, error } = await client.rpc(
- "auth_credit_usage_chunk_47",
- {
- input_key: api_key,
- i_is_extract: isExtract,
- tally_untallied_credits: true,
- },
- { get: true },
- ));
-
- if (!error) {
+ const database = Math.random() > 2 / 3 ? dbRr : db;
+ try {
+ data = await authCreditUsageChunk(database, api_key, isExtract);
break;
- }
-
- logger.warn(
- `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,
- { error },
- );
- retries++;
- if (retries === maxRetries) {
- throw new Error(
- "Failed to retrieve authentication and credit usage data after 3 attempts: " +
- JSON.stringify(error),
+ } catch (error) {
+ logger.warn(
+ `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,
+ { error },
);
- }
+ retries++;
+ if (retries === maxRetries) {
+ throw new Error(
+ "Failed to retrieve authentication and credit usage data after 3 attempts: " +
+ JSON.stringify(error),
+ );
+ }
- // Wait for a short time before retrying
- await new Promise(resolve => setTimeout(resolve, 200));
+ // Wait for a short time before retrying
+ await new Promise(resolve => setTimeout(resolve, 200));
+ }
}
const chunk: AuthCreditUsageChunk | null =
- data.length === 0 ? null : data[0].team_id === null ? null : data[0];
+ data.length === 0
+ ? null
+ : data[0].team_id === null
+ ? null
+ : (data[0] as any);
if (chunk) {
chunk.is_extract = isExtract;
@@ -386,46 +386,39 @@ export async function getACUCTeam(
}
if (!cacheOnly) {
- let data;
- let error;
+ let data: AuthCreditUsageChunkRow[] = [];
let retries = 0;
const maxRetries = 5;
while (retries < maxRetries) {
- const client =
- Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;
- ({ data, error } = await client.rpc(
- "auth_credit_usage_chunk_47_from_team",
- {
- input_team: team_id,
- i_is_extract: isExtract,
- tally_untallied_credits: true,
- },
- { get: true },
- ));
-
- if (!error) {
+ const database = Math.random() > 2 / 3 ? dbRr : db;
+ try {
+ data = await authCreditUsageChunkFromTeam(database, team_id, isExtract);
break;
- }
-
- logger.warn(
- `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,
- { error },
- );
- retries++;
- if (retries === maxRetries) {
- throw new Error(
- "Failed to retrieve authentication and credit usage data after 3 attempts: " +
- JSON.stringify(error),
+ } catch (error) {
+ logger.warn(
+ `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`,
+ { error },
);
- }
+ retries++;
+ if (retries === maxRetries) {
+ throw new Error(
+ "Failed to retrieve authentication and credit usage data after 3 attempts: " +
+ JSON.stringify(error),
+ );
+ }
- // Wait for a short time before retrying
- await new Promise(resolve => setTimeout(resolve, 200));
+ // Wait for a short time before retrying
+ await new Promise(resolve => setTimeout(resolve, 200));
+ }
}
const chunk: AuthCreditUsageChunk | null =
- data.length === 0 ? null : data[0].team_id === null ? null : data[0];
+ data.length === 0
+ ? null
+ : data[0].team_id === null
+ ? null
+ : (data[0] as any);
// NOTE: Should we cache null chunks? - mogery
if (chunk !== null && useCache) {
@@ -493,13 +486,14 @@ async function ensureChunkOrgId(
return chunk;
}
- const { data, error } = await supabase_rr_service
- .from("teams")
- .select("org_id")
- .eq("id", chunk.team_id)
- .single();
-
- if (error || !data?.org_id) {
+ let data: { org_id: string | null } | undefined;
+ try {
+ [data] = await dbRr
+ .select({ org_id: schema.teams.org_id })
+ .from(schema.teams)
+ .where(eq(schema.teams.id, chunk.team_id))
+ .limit(1);
+ } catch (error) {
logger.warn("Failed to backfill org_id for auth chunk", {
teamId: chunk.team_id,
error,
@@ -507,6 +501,13 @@ async function ensureChunkOrgId(
return chunk;
}
+ if (!data?.org_id) {
+ logger.warn("Failed to backfill org_id for auth chunk", {
+ teamId: chunk.team_id,
+ });
+ return chunk;
+ }
+
chunk.org_id = data.org_id;
await setCachedACUC(apiKey, !!chunk.is_extract, chunk);
return chunk;
diff --git a/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts b/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts
index bbe0d43e4..5e69f1507 100644
--- a/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts
+++ b/apps/api/src/controllers/v0/admin/acuc-cache-clear.ts
@@ -1,5 +1,7 @@
import { Request, Response } from "express";
-import { supabase_service } from "../../../services/supabase";
+import { eq } from "drizzle-orm";
+import { db } from "../../../db/connection";
+import * as schema from "../../../db/schema";
import { clearACUC, clearACUCTeam } from "../../auth";
import { logger } from "../../../lib/logger";
@@ -11,12 +13,12 @@ export async function acucCacheClearController(req: Request, res: Response) {
return res.status(400).json({ error: "team_id is required" });
}
- const keys = await supabase_service
- .from("api_keys")
- .select("*")
- .eq("team_id", team_id);
+ const keys = await db
+ .select()
+ .from(schema.api_keys)
+ .where(eq(schema.api_keys.team_id, team_id));
- await Promise.all((keys.data ?? []).map(x => clearACUC(x.key)));
+ await Promise.all(keys.map(x => clearACUC(x.key!)));
await clearACUCTeam(team_id);
logger.info(`ACUC cache cleared for team ${team_id}`);
diff --git a/apps/api/src/controllers/v0/admin/cclog.ts b/apps/api/src/controllers/v0/admin/cclog.ts
index 48a2dea4c..d77159a34 100644
--- a/apps/api/src/controllers/v0/admin/cclog.ts
+++ b/apps/api/src/controllers/v0/admin/cclog.ts
@@ -1,5 +1,6 @@
import { getRedisConnection } from "../../../services/queue-service";
-import { supabase_service } from "../../../services/supabase";
+import { db } from "../../../db/connection";
+import * as schema from "../../../db/schema";
import { logger as _logger } from "../../../lib/logger";
import { Request, Response } from "express";
@@ -46,15 +47,13 @@ async function cclog() {
}
try {
- const { error } = await supabase_service
- .from("concurrency_log")
- .insert(entries);
- if (error) {
- logger.error("Error inserting", {
- error,
- entryCount: entries.length,
- });
- }
+ await db.insert(schema.concurrency_log).values(
+ entries.map(e => ({
+ team_id: e.team_id,
+ concurrency: e.concurrency,
+ created_at: e.created_at.toISOString(),
+ })),
+ );
} catch (e) {
logger.error("Error inserting", { error: e });
}
diff --git a/apps/api/src/controllers/v1/activity.ts b/apps/api/src/controllers/v1/activity.ts
index fc0154e2f..dfd24e5be 100644
--- a/apps/api/src/controllers/v1/activity.ts
+++ b/apps/api/src/controllers/v1/activity.ts
@@ -1,6 +1,8 @@
import { Response } from "express";
+import { and, desc, eq, gte, lt, or } from "drizzle-orm";
import { RequestWithAuth, ErrorResponse } from "./types";
-import { supabase_rr_service } from "../../services/supabase";
+import { dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { logger as _logger } from "../../lib/logger";
const ACTIVITY_WINDOW_HOURS = 24;
@@ -98,30 +100,50 @@ export async function activityController(
Date.now() - ACTIVITY_WINDOW_HOURS * 60 * 60 * 1000,
).toISOString();
- let query = supabase_rr_service
- .from("requests")
- .select("id, kind, api_version, created_at, target_hint")
- .eq("team_id", req.auth.team_id)
- .gte("created_at", windowStart)
- .order("created_at", { ascending: false })
- .order("id", { ascending: false })
- .limit(limit + 1); // fetch one extra to determine has_more
+ const conditions = [
+ eq(schema.requests.team_id, req.auth.team_id),
+ gte(schema.requests.created_at, windowStart),
+ ];
if (endpoint) {
- query = query.eq("kind", endpoint);
+ conditions.push(eq(schema.requests.kind, endpoint));
}
if (cursor) {
// For keyset pagination: fetch rows where (created_at, id) < (cursor.createdAt, cursor.id)
// This translates to: created_at < cursor OR (created_at = cursor AND id < cursor.id)
- query = query.or(
- `created_at.lt.${cursor.createdAt},and(created_at.eq.${cursor.createdAt},id.lt.${cursor.id})`,
+ conditions.push(
+ or(
+ lt(schema.requests.created_at, cursor.createdAt),
+ and(
+ eq(schema.requests.created_at, cursor.createdAt),
+ lt(schema.requests.id, cursor.id),
+ ),
+ )!,
);
}
- const { data, error } = await query;
-
- if (error) {
+ let data: {
+ id: string;
+ kind: string | null;
+ api_version: string | null;
+ created_at: string | null;
+ target_hint: string | null;
+ }[];
+ try {
+ data = await dbRr
+ .select({
+ id: schema.requests.id,
+ kind: schema.requests.kind,
+ api_version: schema.requests.api_version,
+ created_at: schema.requests.created_at,
+ target_hint: schema.requests.target_hint,
+ })
+ .from(schema.requests)
+ .where(and(...conditions))
+ .orderBy(desc(schema.requests.created_at), desc(schema.requests.id))
+ .limit(limit + 1); // fetch one extra to determine has_more
+ } catch (error) {
logger.error("Failed to fetch activity", { error });
return res.status(500).json({
success: false,
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index 2067d89f0..d89fe4ce0 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -8,7 +8,9 @@ import {
import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis";
import { redisEvictConnection } from "../../../src/services/redis";
import { configDotenv } from "dotenv";
-import { supabase_rr_service } from "../../services/supabase";
+import { and, eq } from "drizzle-orm";
+import { dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { logger as _logger } from "../../lib/logger";
import { deserializeTransportableError } from "../../lib/error-serde";
import { TransportableError } from "../../lib/error";
@@ -75,14 +77,14 @@ export async function crawlErrorsController(
});
} else if (config.USE_DB_AUTHENTICATION) {
// Check the requests table for the crawl/batch scrape request
- const { data: request, error: requestError } = await supabase_rr_service
- .from("requests")
- .select("*")
- .eq("id", req.params.jobId)
- .limit(1)
- .throwOnError();
-
- if (requestError) {
+ let request: (typeof schema.requests.$inferSelect)[];
+ try {
+ request = await dbRr
+ .select()
+ .from(schema.requests)
+ .where(eq(schema.requests.id, req.params.jobId))
+ .limit(1);
+ } catch (requestError) {
_logger.error("Error getting request", { error: requestError });
throw requestError;
}
@@ -98,7 +100,7 @@ export async function crawlErrorsController(
if (
requestData &&
- new Date().valueOf() - new Date(requestData.created_at).valueOf() >
+ new Date().valueOf() - new Date(requestData.created_at!).valueOf() >
crawlTtlMs
) {
return res.status(404).json({ success: false, error: "Job expired" });
@@ -109,16 +111,19 @@ export async function crawlErrorsController(
}
// Get failed scrapes from the scrapes table
- const { data: failedScrapes, error: failedScrapesError } =
- await supabase_rr_service
- .from("scrapes")
- .select("*")
- .eq("request_id", req.params.jobId)
- .eq("team_id", req.auth.team_id)
- .eq("is_successful", false)
- .throwOnError();
-
- if (failedScrapesError) {
+ let failedScrapes: (typeof schema.scrapes.$inferSelect)[];
+ try {
+ failedScrapes = await dbRr
+ .select()
+ .from(schema.scrapes)
+ .where(
+ and(
+ eq(schema.scrapes.request_id, req.params.jobId),
+ eq(schema.scrapes.team_id, req.auth.team_id),
+ eq(schema.scrapes.is_successful, false),
+ ),
+ );
+ } catch (failedScrapesError) {
_logger.error("Error getting failed scrapes", {
error: failedScrapesError,
});
@@ -134,10 +139,9 @@ export async function crawlErrorsController(
: null;
return {
id: scrape.id,
- timestamp:
- scrape.created_at !== undefined
- ? new Date(scrape.created_at).toISOString()
- : undefined,
+ timestamp: scrape.created_at
+ ? new Date(scrape.created_at).toISOString()
+ : undefined,
url: scrape.url,
...(error
? {
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 619f67e26..9bb4cbe58 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -21,7 +21,7 @@ import {
} from "../../lib/supabase-jobs";
import { configDotenv } from "dotenv";
import { logger } from "../../lib/logger";
-import { supabase_rr_service, supabase_service } from "../../services/supabase";
+import { creditsBilledByCrawlId } from "../../db/rpc";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import {
scrapeQueue,
@@ -196,14 +196,8 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
- const creditsRpc = config.USE_DB_AUTHENTICATION
- ? await supabase_service.rpc(
- "credits_billed_by_crawl_id_2",
- {
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- )
+ const creditsBilled = config.USE_DB_AUTHENTICATION
+ ? await creditsBilledByCrawlId(req.params.jobId).catch(() => null)
: null;
// check if the crawl failed during kickoff (e.g. queue full)
@@ -226,7 +220,7 @@ export async function crawlStatusController(
(numericStats.active ?? 0) +
(numericStats.queued ?? 0) +
(numericStats.backlog ?? 0),
- creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,
+ creditsUsed: creditsBilled?.[0]?.credits_billed ?? -1,
};
// if the crawl has a stored error and no jobs were ever created, mark as failed
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index bc5b685c8..4b07896d9 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -68,7 +68,7 @@ export async function scrapeController(
account: req.account,
});
- logRequest({
+ const logRequestPromise = logRequest({
id: jobId,
kind: "scrape",
api_version: "v1",
@@ -171,6 +171,7 @@ export async function scrapeController(
zeroDataRetention: zeroDataRetention ?? false,
apiKeyId: req.acuc?.api_key_id ?? null,
concurrencyLimited: limited,
+ logRequestPromise: logRequestPromise,
},
};
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index fe044a6e1..606f2254f 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -18,7 +18,9 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
import { CostTracking } from "../../lib/cost-tracking";
-import { supabase_service } from "../../services/supabase";
+import { eq } from "drizzle-orm";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
import { fromV1ScrapeOptions } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
@@ -132,17 +134,14 @@ async function scrapeX402SearchResult(
let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
if (config.USE_DB_AUTHENTICATION) {
- const { data: costTrackingResponse, error: costTrackingError } =
- await supabase_service
- .from("scrapes")
- .select("cost_tracking")
- .eq("id", jobId);
-
- if (costTrackingError) {
- throw costTrackingError;
- }
-
- costTracking = costTrackingResponse?.[0]?.cost_tracking;
+ const costTrackingResponse = await db
+ .select({ cost_tracking: schema.scrapes.cost_tracking })
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.id, jobId));
+
+ costTracking = costTrackingResponse?.[0]?.cost_tracking as ReturnType<
+ typeof CostTracking.prototype.toJSON
+ >;
} else {
costTracking = new CostTracking().toJSON();
}
diff --git a/apps/api/src/controllers/v2/__tests__/browser-billing.test.ts b/apps/api/src/controllers/v2/__tests__/browser-billing.test.ts
index db2b07dfb..fc01717da 100644
--- a/apps/api/src/controllers/v2/__tests__/browser-billing.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/browser-billing.test.ts
@@ -16,11 +16,6 @@ jest.mock("../../../services/redis", () => ({
deleteKey: (key: string) => mockDeleteKey(key),
}));
-jest.mock("../../../services/supabase", () => ({
- supabase_service: {},
- isPostgrestNoRowsError: jest.fn(() => true),
-}));
-
jest.mock("../../../lib/logger", () => ({
logger: {
info: jest.fn(),
diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.ts
index 34136a277..e478f4128 100644
--- a/apps/api/src/controllers/v2/agent.ts
+++ b/apps/api/src/controllers/v2/agent.ts
@@ -9,7 +9,7 @@ import {
import { logger as _logger } from "../../lib/logger";
import { logRequest } from "../../services/logging/log_job";
import { config } from "../../config";
-import { supabase_service } from "../../services/supabase";
+import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";
import { getScrapeZDR } from "../../lib/zdr-helpers";
export async function agentController(
@@ -57,18 +57,7 @@ export async function agentController(
let freeRequest: any;
if (config.USE_DB_AUTHENTICATION && !highCreditRequest) {
- const { data, error: freeRequestError } = await supabase_service.rpc(
- "agent_consume_free_request_if_left",
- {
- i_team_id: req.auth.team_id,
- },
- );
-
- if (freeRequestError) {
- throw freeRequestError;
- }
-
- freeRequest = data;
+ freeRequest = await agentConsumeFreeRequestIfLeft(req.auth.team_id);
}
const isFreeRequest = highCreditRequest
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index 2067d89f0..d89fe4ce0 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -8,7 +8,9 @@ import {
import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis";
import { redisEvictConnection } from "../../../src/services/redis";
import { configDotenv } from "dotenv";
-import { supabase_rr_service } from "../../services/supabase";
+import { and, eq } from "drizzle-orm";
+import { dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { logger as _logger } from "../../lib/logger";
import { deserializeTransportableError } from "../../lib/error-serde";
import { TransportableError } from "../../lib/error";
@@ -75,14 +77,14 @@ export async function crawlErrorsController(
});
} else if (config.USE_DB_AUTHENTICATION) {
// Check the requests table for the crawl/batch scrape request
- const { data: request, error: requestError } = await supabase_rr_service
- .from("requests")
- .select("*")
- .eq("id", req.params.jobId)
- .limit(1)
- .throwOnError();
-
- if (requestError) {
+ let request: (typeof schema.requests.$inferSelect)[];
+ try {
+ request = await dbRr
+ .select()
+ .from(schema.requests)
+ .where(eq(schema.requests.id, req.params.jobId))
+ .limit(1);
+ } catch (requestError) {
_logger.error("Error getting request", { error: requestError });
throw requestError;
}
@@ -98,7 +100,7 @@ export async function crawlErrorsController(
if (
requestData &&
- new Date().valueOf() - new Date(requestData.created_at).valueOf() >
+ new Date().valueOf() - new Date(requestData.created_at!).valueOf() >
crawlTtlMs
) {
return res.status(404).json({ success: false, error: "Job expired" });
@@ -109,16 +111,19 @@ export async function crawlErrorsController(
}
// Get failed scrapes from the scrapes table
- const { data: failedScrapes, error: failedScrapesError } =
- await supabase_rr_service
- .from("scrapes")
- .select("*")
- .eq("request_id", req.params.jobId)
- .eq("team_id", req.auth.team_id)
- .eq("is_successful", false)
- .throwOnError();
-
- if (failedScrapesError) {
+ let failedScrapes: (typeof schema.scrapes.$inferSelect)[];
+ try {
+ failedScrapes = await dbRr
+ .select()
+ .from(schema.scrapes)
+ .where(
+ and(
+ eq(schema.scrapes.request_id, req.params.jobId),
+ eq(schema.scrapes.team_id, req.auth.team_id),
+ eq(schema.scrapes.is_successful, false),
+ ),
+ );
+ } catch (failedScrapesError) {
_logger.error("Error getting failed scrapes", {
error: failedScrapesError,
});
@@ -134,10 +139,9 @@ export async function crawlErrorsController(
: null;
return {
id: scrape.id,
- timestamp:
- scrape.created_at !== undefined
- ? new Date(scrape.created_at).toISOString()
- : undefined,
+ timestamp: scrape.created_at
+ ? new Date(scrape.created_at).toISOString()
+ : undefined,
url: scrape.url,
...(error
? {
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 98ec8c32e..784804e78 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -21,7 +21,7 @@ import {
} from "../../lib/supabase-jobs";
import { configDotenv } from "dotenv";
import { logger } from "../../lib/logger";
-import { supabase_rr_service, supabase_service } from "../../services/supabase";
+import { creditsBilledByCrawlId } from "../../db/rpc";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import {
scrapeQueue,
@@ -216,14 +216,8 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
- const creditsRpc = config.USE_DB_AUTHENTICATION
- ? await supabase_service.rpc(
- "credits_billed_by_crawl_id_2",
- {
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- )
+ const creditsBilled = config.USE_DB_AUTHENTICATION
+ ? await creditsBilledByCrawlId(req.params.jobId).catch(() => null)
: null;
// check if the crawl failed during kickoff (e.g. queue full)
@@ -246,7 +240,7 @@ export async function crawlStatusController(
(numericStats.active ?? 0) +
(numericStats.queued ?? 0) +
(numericStats.backlog ?? 0),
- creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,
+ creditsUsed: creditsBilled?.[0]?.credits_billed ?? -1,
};
// if the crawl has a stored error and no jobs were ever created, mark as failed
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index a96140144..2b6df4743 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -132,8 +132,10 @@ export async function scrapeController(
account: req.account,
});
+ let logRequestPromise: Promise<any> | undefined = undefined;
+
if (!agentRequestId) {
- logRequest({
+ logRequestPromise = logRequest({
id: jobId,
kind: "scrape",
api_version: "v2",
@@ -260,6 +262,7 @@ export async function scrapeController(
apiKeyId: req.acuc?.api_key_id ?? null,
concurrencyLimited: limited,
requestId: agentRequestId ?? undefined,
+ logRequestPromise: logRequestPromise,
},
};
diff --git a/apps/api/src/controllers/v2/search-feedback.ts b/apps/api/src/controllers/v2/search-feedback.ts
index bde625827..29325fe6d 100644
--- a/apps/api/src/controllers/v2/search-feedback.ts
+++ b/apps/api/src/controllers/v2/search-feedback.ts
@@ -4,11 +4,9 @@ import { z } from "zod";
import { config } from "../../config";
import { logger as _logger } from "../../lib/logger";
import { autumnService } from "../../services/autumn/autumn.service";
-import {
- isPostgrestNoRowsError,
- supabase_service,
- supabase_rr_service,
-} from "../../services/supabase";
+import { and, eq, gte } from "drizzle-orm";
+import { db, dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
import {
RequestWithAuth,
@@ -66,18 +64,24 @@ async function lookupSearchRow(
searchId: string,
dbTeamId: string,
): Promise<SearchRowForFeedback | null> {
- const { data, error } = await supabase_rr_service
- .from("searches")
- .select("id, team_id, credits_cost, created_at, is_successful")
- .eq("id", searchId)
- .eq("team_id", dbTeamId)
- .single();
-
- if (error) {
- if (isPostgrestNoRowsError(error)) return null;
- throw error;
- }
- return data as SearchRowForFeedback | null;
+ const [data] = await dbRr
+ .select({
+ id: schema.searches.id,
+ team_id: schema.searches.team_id,
+ credits_cost: schema.searches.credits_cost,
+ created_at: schema.searches.created_at,
+ is_successful: schema.searches.is_successful,
+ })
+ .from(schema.searches)
+ .where(
+ and(
+ eq(schema.searches.id, searchId),
+ eq(schema.searches.team_id, dbTeamId),
+ ),
+ )
+ .limit(1);
+
+ return (data ?? null) as SearchRowForFeedback | null;
}
function startOfUtcDay(now: Date = new Date()): Date {
@@ -94,13 +98,18 @@ async function sumTeamCreditsRefundedToday(
): Promise<number> {
const since = startOfUtcDay().toISOString();
- const { data, error } = await supabase_rr_service
- .from("search_feedback")
- .select("credits_refunded")
- .eq("team_id", dbTeamId)
- .gte("created_at", since);
-
- if (error) {
+ let data: { credits_refunded: number | null }[];
+ try {
+ data = await dbRr
+ .select({ credits_refunded: schema.search_feedback.credits_refunded })
+ .from(schema.search_feedback)
+ .where(
+ and(
+ eq(schema.search_feedback.team_id, dbTeamId),
+ gte(schema.search_feedback.created_at, since),
+ ),
+ );
+ } catch (error) {
logger.warn(
"Failed to compute today's refund total; allowing refund this call",
{ error },
@@ -108,11 +117,7 @@ async function sumTeamCreditsRefundedToday(
return 0;
}
- return (data ?? []).reduce(
- (sum, row: { credits_refunded: number | null }) =>
- sum + (row.credits_refunded ?? 0),
- 0,
- );
+ return data.reduce((sum, row) => sum + (row.credits_refunded ?? 0), 0);
}
export async function searchFeedbackController(
@@ -237,9 +242,8 @@ export async function searchFeedbackController(
}
const feedbackId = uuidv7();
- const { error: insertErr } = await supabase_service
- .from("search_feedback")
- .insert({
+ try {
+ await db.insert(schema.search_feedback).values({
id: feedbackId,
search_id: searchId,
team_id: dbTeamId,
@@ -251,14 +255,16 @@ export async function searchFeedbackController(
origin: parsedBody.origin ?? null,
credits_refunded: 0,
});
-
- if (insertErr) {
+ } catch (insertErr) {
if ((insertErr as { code?: string }).code === POSTGRES_UNIQUE_VIOLATION) {
- const { data: existing } = await supabase_rr_service
- .from("search_feedback")
- .select("id, credits_refunded")
- .eq("search_id", searchId)
- .single();
+ const [existing] = await dbRr
+ .select({
+ id: schema.search_feedback.id,
+ credits_refunded: schema.search_feedback.credits_refunded,
+ })
+ .from(schema.search_feedback)
+ .where(eq(schema.search_feedback.search_id, searchId))
+ .limit(1);
const dailyCapForResponse = config.SEARCH_FEEDBACK_DAILY_CAP_CREDITS;
const refundedTodayForResponse = await sumTeamCreditsRefundedToday(
@@ -335,11 +341,12 @@ export async function searchFeedbackController(
}
if (creditsRefunded > 0) {
- const { error: updateErr } = await supabase_service
- .from("search_feedback")
- .update({ credits_refunded: creditsRefunded })
- .eq("id", feedbackId);
- if (updateErr) {
+ try {
+ await db
+ .update(schema.search_feedback)
+ .set({ credits_refunded: creditsRefunded })
+ .where(eq(schema.search_feedback.id, feedbackId));
+ } catch (updateErr) {
logger.warn(
"Failed to persist credits_refunded on search_feedback row",
{ error: updateErr, feedbackId, creditsRefunded },
diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.ts
index 46e94532f..1f6114373 100644
--- a/apps/api/src/controllers/v2/x402-search.ts
+++ b/apps/api/src/controllers/v2/x402-search.ts
@@ -18,7 +18,9 @@ import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
import { getJobPriority } from "../../lib/job-priority";
import { CostTracking } from "../../lib/cost-tracking";
-import { supabase_service } from "../../services/supabase";
+import { eq } from "drizzle-orm";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
import { SearchV2Response } from "../../lib/entities";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
@@ -155,20 +157,14 @@ async function scrapeX420SearchResult(
let costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;
if (config.USE_DB_AUTHENTICATION) {
- const { data: costTrackingResponse, error: costTrackingError } =
- await supabase_service
- .from("scrapes")
- .select("cost_tracking")
- .eq("id", jobId);
-
- if (costTrackingError) {
- logger.error("Error getting cost tracking [x402]", {
- error: costTrackingError,
- });
- throw costTrackingError;
- }
-
- costTracking = costTrackingResponse?.[0]?.cost_tracking;
+ const costTrackingResponse = await db
+ .select({ cost_tracking: schema.scrapes.cost_tracking })
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.id, jobId));
+
+ costTracking = costTrackingResponse?.[0]?.cost_tracking as ReturnType<
+ typeof CostTracking.prototype.toJSON
+ >;
} else {
costTracking = new CostTracking().toJSON();
}
diff --git a/apps/api/src/db/connection.ts b/apps/api/src/db/connection.ts
new file mode 100644
index 000000000..9d38bd3c8
--- /dev/null
+++ b/apps/api/src/db/connection.ts
@@ -0,0 +1,94 @@
+import { Pool } from "pg";
+import { drizzle, NodePgDatabase } from "drizzle-orm/node-postgres";
+import { config } from "../config";
+import { logger } from "../lib/logger";
+
+type DB = NodePgDatabase;
+
+function makeDb(
+ connectionString: string | undefined,
+ applicationName: string,
+): DB | null {
+ if (!connectionString) {
+ return null;
+ }
+
+ const pool = new Pool({
+ connectionString,
+ application_name: applicationName,
+ min: 2,
+ keepAlive: true,
+ });
+ pool.on("error", err =>
+ logger.error("Error in idle Postgres client", {
+ err,
+ module: "db",
+ applicationName,
+ }),
+ );
+
+ let lastWarn = 0;
+ pool.on("acquire", () => {
+ const max = pool.options.max ?? 10;
+ if (
+ pool.waitingCount > 0 &&
+ pool.totalCount >= max &&
+ Date.now() - lastWarn > 1000
+ ) {
+ lastWarn = Date.now();
+ logger.warn("Postgres pool exhausted, queries are queuing", {
+ module: "db",
+ applicationName,
+ total: pool.totalCount,
+ idle: pool.idleCount,
+ waiting: pool.waitingCount,
+ max,
+ });
+ }
+ });
+
+ return drizzle({ client: pool });
+}
+
+const useDbAuthentication = config.USE_DB_AUTHENTICATION;
+
+const mainDb = useDbAuthentication
+ ? makeDb(config.DATABASE_URL, "firecrawl-api")
+ : null;
+const replicaDb = useDbAuthentication
+ ? makeDb(
+ config.DATABASE_REPLICA_URL ?? config.DATABASE_URL,
+ "firecrawl-api-rr",
+ )
+ : null;
+const indexDb = makeDb(config.INDEX_DATABASE_URL, "firecrawl-index");
+
+if (useDbAuthentication && !mainDb) {
+ logger.error(
+ "DATABASE_URL is not configured. Drizzle client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable",
+ );
+}
+
+function proxyDb(get: () => DB | null, name: string): DB {
+ return new Proxy(
+ {},
+ {
+ get(_target, prop, receiver) {
+ const client = get();
+ if (client === null) {
+ throw new Error(`${name} is not configured.`);
+ }
+ return Reflect.get(client, prop, receiver);
+ },
+ },
+ ) as DB;
+}
+
+/** Main Postgres database (writes + reads). */
+export const db: DB = proxyDb(() => mainDb, "Database client");
+
+/** Read replica. Falls back to the main connection string when no replica URL is set. */
+export const dbRr: DB = proxyDb(() => replicaDb, "Database replica client");
+
+/** Separate index project database. */
+export const dbIndex: DB = proxyDb(() => indexDb, "Index database client");
diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.ts
new file mode 100644
index 000000000..2b8397446
--- /dev/null
+++ b/apps/api/src/db/rpc.ts
@@ -0,0 +1,275 @@
+import { sql, SQL } from "drizzle-orm";
+import { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { db, dbIndex } from "./connection";
+
+type DB = NodePgDatabase;
+
+async function execRows<T = Record<string, any>>(
+ database: DB,
+ query: SQL,
+): Promise<T[]> {
+ const res = await database.execute(query);
+ return (res.rows ?? []) as T[];
+}
+
+// ============================================================================
+// Main database RPCs
+// ============================================================================
+
+export type AuthCreditUsageChunkRow = Record<string, any> & {
+ team_id: string | null;
+ api_key: string;
+};
+
+export function authCreditUsageChunk(
+ database: DB,
+ input_key: string,
+ i_is_extract: boolean,
+): Promise<AuthCreditUsageChunkRow[]> {
+ return execRows(
+ database,
+ sql`select * from auth_credit_usage_chunk_47(input_key => ${input_key}, i_is_extract => ${i_is_extract}, tally_untallied_credits => ${true})`,
+ );
+}
+
+export function authCreditUsageChunkFromTeam(
+ database: DB,
+ input_team: string,
+ i_is_extract: boolean,
+): Promise<AuthCreditUsageChunkRow[]> {
+ return execRows(
+ database,
+ sql`select * from auth_credit_usage_chunk_47_from_team(input_team => ${input_team}, i_is_extract => ${i_is_extract}, tally_untallied_credits => ${true})`,
+ );
+}
+
+export function getAgentFreeRequestsLeft(
+ i_team_id: string,
+): Promise<{ free_requests_left: number }[]> {
+ return execRows(
+ db,
+ sql`select * from get_agent_free_requests_left(i_team_id => ${i_team_id})`,
+ );
+}
+
+export function agentConsumeFreeRequestIfLeft(
+ i_team_id: string,
+): Promise<{ consumed: boolean }[]> {
+ return execRows(
+ db,
+ sql`select * from agent_consume_free_request_if_left(i_team_id => ${i_team_id})`,
+ );
+}
+
+export function billTeam6(params: {
+ team_id: string;
+ subscription_id: string | null;
+ fetch_subscription: boolean;
+ credits: number;
+ api_key_id: number | null;
+ is_extract: boolean;
+}): Promise<{ api_key: string }[]> {
+ return execRows(
+ db,
+ sql`select * from bill_team_6(_team_id => ${params.team_id}, sub_id => ${params.subscription_id}, fetch_subscription => ${params.fetch_subscription}, credits => ${params.credits}, i_api_key_id => ${params.api_key_id}, is_extract_param => ${params.is_extract})`,
+ );
+}
+
+export async function changeTrackingInsertScrape(params: {
+ team_id: string;
+ url: string;
+ job_id: string;
+ change_tracking_tag: string | null;
+ date_added: string;
+}): Promise<void> {
+ await db.execute(
+ sql`select change_tracking_insert_scrape(p_team_id => ${params.team_id}, p_url => ${params.url}, p_job_id => ${params.job_id}, p_change_tracking_tag => ${params.change_tracking_tag}, p_date_added => ${params.date_added}::timestamptz)`,
+ );
+}
+
+export function creditsBilledByCrawlId(
+ i_crawl_id: string,
+): Promise<{ credits_billed: number }[]> {
+ return execRows(
+ db,
+ sql`select * from credits_billed_by_crawl_id_2(i_crawl_id => ${i_crawl_id})`,
+ );
+}
+
+export function diffGetLastScrape(
+ i_team_id: string,
+ i_url: string,
+ i_tag: string | null,
+): Promise<{ o_job_id: string; o_date_added: string }[]> {
+ return execRows(
+ db,
+ sql`select * from diff_get_last_scrape_v7(i_team_id => ${i_team_id}, i_url => ${i_url}, i_tag => ${i_tag})`,
+ );
+}
+
+export function getZdrCleanupBatch(
+ p_limit: number,
+): Promise<{ request_id: string; ids: string[] }[]> {
+ return execRows(
+ db,
+ sql`select * from get_zdr_cleanup_batch_2(p_limit => ${p_limit})`,
+ );
+}
+
+export function monitoringClaimDueMonitors<T = Record<string, any>>(params: {
+ workerId: string;
+ limit: number;
+ leaseSeconds: number;
+}): Promise<T[]> {
+ return execRows(
+ db,
+ sql`select * from monitoring_claim_due_monitors(p_worker_id => ${params.workerId}, p_limit => ${params.limit}, p_lease_seconds => ${params.leaseSeconds})`,
+ );
+}
+
+export async function updateTallyTeam(i_team_id: string): Promise<void> {
+ await db.execute(sql`select update_tally_10_team(i_team_id => ${i_team_id})`);
+}
+
+// ============================================================================
+// Index database RPCs
+// ============================================================================
+
+export async function insertOmceJobIfNeeded(
+ i_domain_level: number,
+ i_domain_hash: Buffer,
+): Promise<void> {
+ await dbIndex.execute(
+ sql`select insert_omce_job_if_needed(i_domain_level => ${i_domain_level}, i_domain_hash => ${i_domain_hash})`,
+ );
+}
+
+export function queryIndexAtSplitLevel(
+ i_level: number,
+ i_url_hash: Buffer,
+ i_newer_than: string,
+): Promise<{ resolved_url: string }[]> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_index_at_split_level(i_level => ${i_level}, i_url_hash => ${i_url_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,
+ );
+}
+
+export function queryIndexAtDomainSplitLevel(
+ i_level: number,
+ i_domain_hash: Buffer,
+ i_newer_than: string,
+): Promise<{ resolved_url: string }[]> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_index_at_domain_split_level(i_level => ${i_level}, i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,
+ );
+}
+
+export function queryOmceSignatures(
+ i_domain_hash: Buffer,
+ i_newer_than: string,
+): Promise<{ signatures: any[] }[]> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_omce_signatures(i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,
+ );
+}
+
+export function queryEngpickerVerdict(
+ i_domain_hash: Buffer,
+): Promise<{ verdict: string }[]> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_engpicker_verdict(i_domain_hash => ${i_domain_hash})`,
+ );
+}
+
+export function queryIndexAtSplitLevelWithMeta(
+ i_level: number,
+ i_url_hash: Buffer,
+ i_newer_than: string,
+): Promise<
+ { resolved_url: string; title: string | null; description: string | null }[]
+> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_index_at_split_level_with_meta(i_level => ${i_level}, i_url_hash => ${i_url_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,
+ );
+}
+
+export function queryIndexAtDomainSplitLevelWithMeta(
+ i_level: number,
+ i_domain_hash: Buffer,
+ i_newer_than: string,
+): Promise<
+ { resolved_url: string; title: string | null; description: string | null }[]
+> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_index_at_domain_split_level_with_meta(i_level => ${i_level}, i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,
+ );
+}
+
+export function queryDomainPriority(
+ p_min_total: number,
+ p_min_priority: number,
+ p_lim: number,
+ p_time: string,
+): Promise<{ domain_hash: Buffer; priority: number }[]> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_domain_priority(p_min_total => ${p_min_total}, p_min_priority => ${p_min_priority}, p_lim => ${p_lim}, p_time => ${p_time}::timestamptz)`,
+ );
+}
+
+export function queryIndexAtDomainSplitLevelOmce<T = Record<string, any>>(
+ i_level: number,
+ i_domain_hash: Buffer,
+ i_newer_than: string,
+ limit?: number,
+): Promise<T[]> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_index_at_domain_split_level_omce(i_level => ${i_level}, i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)${limit !== undefined ? sql` limit ${limit}` : sql``}`,
+ );
+}
+
+export function queryMaxAge(
+ i_domain_hash: Buffer,
+): Promise<{ max_age: number | null }[]> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_max_age(i_domain_hash => ${i_domain_hash})`,
+ );
+}
+
+export function indexGetRecent4(params: {
+ url_hash: Buffer;
+ max_age_ms: number;
+ is_mobile: boolean;
+ block_ads: boolean;
+ feature_screenshot: boolean;
+ feature_screenshot_fullscreen: boolean;
+ location_country: string | null;
+ location_languages: string[] | null;
+ wait_time_ms: number;
+ is_stealth: boolean;
+ min_age_ms: number | null;
+}): Promise<{ id: string; created_at: string; status: number }[]> {
+ return execRows(
+ dbIndex,
+ sql`select * from index_get_recent_4(p_url_hash => ${params.url_hash}, p_max_age_ms => ${params.max_age_ms}, p_is_mobile => ${params.is_mobile}, p_block_ads => ${params.block_ads}, p_feature_screenshot => ${params.feature_screenshot}, p_feature_screenshot_fullscreen => ${params.feature_screenshot_fullscreen}, p_location_country => ${params.location_country}, p_location_languages => ${sql.param(params.location_languages)}::text[], p_wait_time_ms => ${params.wait_time_ms}, p_is_stealth => ${params.is_stealth}, p_min_age_ms => ${params.min_age_ms})`,
+ );
+}
+
+export function queryTopUrlsForDomain<T = Record<string, any>>(
+ p_domain_hash: Buffer,
+ p_time_window: string,
+ p_top_n: number,
+): Promise<T[]> {
+ return execRows(
+ dbIndex,
+ sql`select * from query_top_urls_for_domain(p_domain_hash => ${p_domain_hash}, p_time_window => ${p_time_window}::interval, p_top_n => ${p_top_n})`,
+ );
+}
diff --git a/apps/api/src/db/schema/index-db.ts b/apps/api/src/db/schema/index-db.ts
new file mode 100644
index 000000000..ec1cda32b
--- /dev/null
+++ b/apps/api/src/db/schema/index-db.ts
@@ -0,0 +1,72 @@
+import {
+ pgTable,
+ bigint,
+ text,
+ integer,
+ boolean,
+ timestamp,
+ bytea,
+ uuid,
+ varchar,
+} from "drizzle-orm/pg-core";
+
+// Tables in the separate index Postgres project (INDEX_DATABASE_URL).
+
+const ts = (name: string) =>
+ timestamp(name, { withTimezone: true, mode: "string" });
+
+const urlSplitHashes = Object.fromEntries(
+ Array.from({ length: 10 }, (_, i) => {
+ const col = bytea(`url_split_${i}_hash`);
+ return [`url_split_${i}_hash`, i === 0 ? col.notNull() : col];
+ }),
+);
+
+const domainSplitHashes = Object.fromEntries(
+ Array.from({ length: 5 }, (_, i) => [
+ `domain_splits_${i}_hash`,
+ bytea(`domain_splits_${i}_hash`),
+ ]),
+);
+
+export const index = pgTable("index", {
+ id: uuid("id").primaryKey().defaultRandom(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ url: text("url").notNull(),
+ url_hash: bytea("url_hash").notNull(),
+ original_url: text("original_url").notNull(),
+ resolved_url: text("resolved_url").notNull(),
+ has_screenshot: boolean("has_screenshot").notNull(),
+ has_screenshot_fullscreen: boolean("has_screenshot_fullscreen").notNull(),
+ is_mobile: boolean("is_mobile").notNull(),
+ block_ads: boolean("block_ads").notNull(),
+ location_country: text("location_country"),
+ location_languages: text("location_languages").array(),
+ status: integer("status").notNull(),
+ title: text("title"),
+ description: text("description"),
+ invalidated_at: ts("invalidated_at"),
+ is_precrawl: boolean("is_precrawl"),
+ wait_time_ms: bigint("wait_time_ms", { mode: "number" }),
+ is_stealth: boolean("is_stealth").notNull().default(false),
+ ...urlSplitHashes,
+ ...domainSplitHashes,
+});
+
+export const engpicker_queue = pgTable("engpicker_queue", {
+ id: bigint("id", { mode: "number" })
+ .primaryKey()
+ .generatedByDefaultAsIdentity(),
+ domain_hash: bytea("domain_hash").notNull(),
+ domain_level: integer("domain_level").notNull(),
+ picked_up_at: ts("picked_up_at"),
+ done: boolean("done").notNull().default(false),
+ created_at: ts("created_at").notNull().defaultNow(),
+});
+
+export const engpicker_verdicts = pgTable("engpicker_verdicts", {
+ id: uuid("id").primaryKey().defaultRandom(),
+ domain_hash: bytea("domain_hash").notNull(),
+ verdict: varchar("verdict").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+});
diff --git a/apps/api/src/db/schema/index.ts b/apps/api/src/db/schema/index.ts
new file mode 100644
index 000000000..402cd0bb1
--- /dev/null
+++ b/apps/api/src/db/schema/index.ts
@@ -0,0 +1,3 @@
+export * from "./public";
+export * from "./ledger";
+export * from "./index-db";
diff --git a/apps/api/src/db/schema/ledger.ts b/apps/api/src/db/schema/ledger.ts
new file mode 100644
index 000000000..cc4a4da9d
--- /dev/null
+++ b/apps/api/src/db/schema/ledger.ts
@@ -0,0 +1,40 @@
+import {
+ pgSchema,
+ bigint,
+ boolean,
+ text,
+ timestamp,
+ uuid,
+ jsonb,
+} from "drizzle-orm/pg-core";
+
+const ledger = pgSchema("ledger");
+
+const ts = (name: string) =>
+ timestamp(name, { withTimezone: true, mode: "string" });
+const bigintNum = (name: string) => bigint(name, { mode: "number" });
+
+export const provider_definitions = ledger.table("provider_definitions", {
+ id: bigintNum("id").notNull().generatedAlwaysAsIdentity(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+ is_active: boolean("is_active").notNull().default(true),
+ provider_id: bigintNum("provider_id").notNull(),
+ slug: text("slug").notNull(),
+ name: text("name").notNull(),
+ description: text("description"),
+ provider_definition_type_id: bigintNum(
+ "provider_definition_type_id",
+ ).notNull(),
+});
+
+export const tracks = ledger.table("tracks", {
+ id: bigintNum("id").notNull().generatedAlwaysAsIdentity(),
+ is_active: boolean("is_active").notNull().default(true),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+ received_at: ts("received_at").notNull().defaultNow(),
+ provider_definition_id: bigintNum("provider_definition_id").notNull(),
+ uuid: uuid("uuid").notNull().defaultRandom(),
+ data: jsonb("data").notNull().default({}),
+});
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
new file mode 100644
index 000000000..9b6c2c7e0
--- /dev/null
+++ b/apps/api/src/db/schema/public.ts
@@ -0,0 +1,542 @@
+import { sql } from "drizzle-orm";
+import {
+ pgTable,
+ uuid,
+ text,
+ varchar,
+ integer,
+ smallint,
+ bigint,
+ boolean,
+ jsonb,
+ numeric,
+ timestamp,
+ date,
+ bytea,
+} from "drizzle-orm/pg-core";
+
+const ts = (name: string) =>
+ timestamp(name, { withTimezone: true, mode: "string" });
+const bigintNum = (name: string) => bigint(name, { mode: "number" });
+const num = (name: string) => numeric(name, { mode: "number" });
+
+export const agent_sponsors = pgTable("agent_sponsors", {
+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
+ email: text("email").notNull(),
+ status: text("status").notNull().default("pending"),
+ verification_deadline: ts("verification_deadline").notNull(),
+ agent_name: text("agent_name").notNull(),
+ sandboxed_team_id: uuid("sandboxed_team_id"),
+ api_key_id: bigintNum("api_key_id"),
+ requesting_ip: text("requesting_ip"),
+ tos_version: text("tos_version"),
+ tos_hash: text("tos_hash"),
+ verification_token: text("verification_token").notNull(),
+ created_at: ts("created_at").defaultNow(),
+ verified_at: ts("verified_at"),
+ updated_at: ts("updated_at").defaultNow(),
+});
+
+export const agents = pgTable("agents", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ team_id: uuid("team_id").notNull(),
+ options: jsonb("options"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ time_taken: num("time_taken").notNull(),
+ credits_cost: integer("credits_cost").notNull(),
+ cost_tracking: jsonb("cost_tracking"),
+ is_successful: boolean("is_successful").notNull(),
+ error: text("error"),
+});
+
+export const api_keys = pgTable("api_keys", {
+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
+ created_at: ts("created_at").defaultNow(),
+ key: uuid("key").defaultRandom(),
+ name: text("name"),
+ team_id: uuid("team_id"),
+ owner_id: uuid("owner_id"),
+ agent_provisioned: boolean("agent_provisioned").default(false),
+});
+
+export const batch_scrapes = pgTable("batch_scrapes", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ team_id: uuid("team_id").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ num_docs: integer("num_docs").notNull(),
+ credits_cost: integer("credits_cost").notNull(),
+ cancelled: boolean("cancelled").notNull(),
+});
+
+export const blocklist = pgTable("blocklist", {
+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
+ data: jsonb("data").notNull(),
+});
+
+export const blocklist_hits = pgTable("blocklist_hits", {
+ id: uuid("id").notNull(),
+ domain: text("domain").notNull(),
+ url: text("url"),
+ team_id: uuid("team_id"),
+ origin: text("origin"),
+ created_at: ts("created_at").notNull().defaultNow(),
+});
+
+export const browser_session_activities = pgTable(
+ "browser_session_activities",
+ {
+ id: bigintNum("id").notNull().generatedAlwaysAsIdentity(),
+ team_id: text("team_id").notNull(),
+ session_id: text("session_id").notNull(),
+ language: text("language").notNull(),
+ timeout: integer("timeout").notNull(),
+ exit_code: integer("exit_code"),
+ killed: boolean("killed").notNull().default(false),
+ created_at: ts("created_at").notNull().defaultNow(),
+ source: text("source").default("browser"),
+ },
+);
+
+export const browser_sessions = pgTable("browser_sessions", {
+ id: uuid("id").notNull(),
+ team_id: text("team_id").notNull(),
+ browser_id: text("browser_id").notNull(),
+ workspace_id: text("workspace_id").notNull(),
+ context_id: text("context_id").notNull(),
+ cdp_url: text("cdp_url").notNull(),
+ cdp_path: text("cdp_path").notNull(),
+ stream_web_view: boolean("stream_web_view").notNull().default(false),
+ status: text("status").notNull().default("active"),
+ ttl_total: integer("ttl_total").notNull(),
+ ttl_without_activity: integer("ttl_without_activity"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+ deleted_at: ts("deleted_at"),
+ credits_used: integer("credits_used"),
+ cdp_interactive_path: text("cdp_interactive_path"),
+ scrape_id: uuid("scrape_id"),
+});
+
+export const concurrency_log = pgTable("concurrency_log", {
+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ team_id: uuid("team_id").notNull(),
+ concurrency: integer("concurrency").notNull(),
+});
+
+export const crawls = pgTable("crawls", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ url: text("url").notNull(),
+ team_id: uuid("team_id").notNull(),
+ options: jsonb("options"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ num_docs: integer("num_docs").notNull(),
+ credits_cost: integer("credits_cost").notNull(),
+ cancelled: boolean("cancelled").notNull(),
+ monitor_id: uuid("monitor_id"),
+ monitor_check_id: uuid("monitor_check_id"),
+});
+
+export const deep_researches = pgTable("deep_researches", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ query: text("query").notNull(),
+ team_id: uuid("team_id").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ time_taken: num("time_taken").notNull(),
+ credits_cost: integer("credits_cost").notNull(),
+ cost_tracking: jsonb("cost_tracking"),
+ options: jsonb("options"),
+});
+
+export const eb_sync = pgTable("eb-sync", {
+ id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ team_id: text("team_id"),
+});
+
+export const extracts = pgTable("extracts", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ urls: text("urls").array().notNull(),
+ options: jsonb("options"),
+ model_kind: text("model_kind").notNull(),
+ team_id: uuid("team_id").notNull(),
+ is_successful: boolean("is_successful").notNull(),
+ error: text("error"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ credits_cost: integer("credits_cost").notNull(),
+ cost_tracking: jsonb("cost_tracking"),
+});
+
+export const idempotency_keys = pgTable("idempotency_keys", {
+ key: uuid("key").notNull().defaultRandom(),
+ created_at: ts("created_at").notNull().defaultNow(),
+});
+
+export const llm_texts = pgTable("llm_texts", {
+ id: uuid("id").notNull().defaultRandom(),
+ origin_url: text("origin_url").notNull(),
+ llmstxt: text("llmstxt").notNull(),
+ llmstxt_full: text("llmstxt_full").notNull(),
+ max_urls: integer("max_urls").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at"),
+});
+
+export const llmstxts = pgTable("llmstxts", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ url: text("url").notNull(),
+ team_id: uuid("team_id").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ num_urls: integer("num_urls").notNull(),
+ options: jsonb("options"),
+ cost_tracking: jsonb("cost_tracking"),
+ credits_cost: integer("credits_cost").notNull(),
+});
+
+export const maps = pgTable("maps", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ url: text("url").notNull(),
+ options: jsonb("options"),
+ team_id: uuid("team_id").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ num_results: integer("num_results").notNull(),
+ credits_cost: integer("credits_cost").notNull(),
+});
+
+export const monitor_check_pages = pgTable("monitor_check_pages", {
+ id: uuid("id").notNull(),
+ check_id: uuid("check_id").notNull(),
+ monitor_id: uuid("monitor_id").notNull(),
+ team_id: uuid("team_id").notNull(),
+ target_id: text("target_id").notNull(),
+ url: text("url").notNull(),
+ url_hash: bytea("url_hash").notNull(),
+ status: text("status").notNull(),
+ previous_scrape_id: uuid("previous_scrape_id"),
+ current_scrape_id: uuid("current_scrape_id"),
+ diff_gcs_key: text("diff_gcs_key"),
+ diff_text_bytes: integer("diff_text_bytes"),
+ diff_json_bytes: integer("diff_json_bytes"),
+ status_code: integer("status_code"),
+ error: text("error"),
+ metadata: jsonb("metadata"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ judgment: jsonb("judgment"),
+});
+
+export const monitor_checks = pgTable("monitor_checks", {
+ id: uuid("id").notNull(),
+ monitor_id: uuid("monitor_id").notNull(),
+ team_id: uuid("team_id").notNull(),
+ trigger: text("trigger").notNull(),
+ status: text("status").notNull().default("queued"),
+ scheduled_for: ts("scheduled_for"),
+ started_at: ts("started_at"),
+ finished_at: ts("finished_at"),
+ estimated_credits: integer("estimated_credits"),
+ reserved_credits: integer("reserved_credits"),
+ actual_credits: integer("actual_credits"),
+ autumn_lock_id: text("autumn_lock_id"),
+ billing_status: text("billing_status").notNull().default("not_applicable"),
+ total_pages: integer("total_pages").notNull().default(0),
+ same_count: integer("same_count").notNull().default(0),
+ changed_count: integer("changed_count").notNull().default(0),
+ new_count: integer("new_count").notNull().default(0),
+ removed_count: integer("removed_count").notNull().default(0),
+ error_count: integer("error_count").notNull().default(0),
+ target_results: jsonb("target_results"),
+ webhook_payload: jsonb("webhook_payload"),
+ email_payload: jsonb("email_payload"),
+ notification_status: jsonb("notification_status"),
+ error: text("error"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+});
+
+export const monitor_email_recipients = pgTable("monitor_email_recipients", {
+ id: uuid("id").notNull().defaultRandom(),
+ monitor_id: uuid("monitor_id").notNull(),
+ team_id: uuid("team_id").notNull(),
+ email: text("email").notNull(),
+ status: text("status").notNull().default("pending"),
+ token: text("token").notNull(),
+ source: text("source").notNull().default("opt_in"),
+ confirmation_sent_at: ts("confirmation_sent_at"),
+ confirmed_at: ts("confirmed_at"),
+ unsubscribed_at: ts("unsubscribed_at"),
+ last_notified_at: ts("last_notified_at"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+});
+
+export const monitor_pages = pgTable("monitor_pages", {
+ id: uuid("id").notNull().defaultRandom(),
+ monitor_id: uuid("monitor_id").notNull(),
+ team_id: uuid("team_id").notNull(),
+ target_id: text("target_id").notNull(),
+ url: text("url").notNull(),
+ url_hash: bytea("url_hash").notNull(),
+ source: text("source").notNull(),
+ first_seen_check_id: uuid("first_seen_check_id"),
+ last_seen_check_id: uuid("last_seen_check_id"),
+ last_changed_check_id: uuid("last_changed_check_id"),
+ last_scrape_id: uuid("last_scrape_id"),
+ last_status: text("last_status").notNull(),
+ is_removed: boolean("is_removed").notNull().default(false),
+ removed_at: ts("removed_at"),
+ metadata: jsonb("metadata"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+});
+
+export const monitors = pgTable("monitors", {
+ id: uuid("id").notNull(),
+ team_id: uuid("team_id").notNull(),
+ name: text("name").notNull(),
+ status: text("status").notNull().default("active"),
+ schedule_cron: text("schedule_cron").notNull(),
+ schedule_timezone: text("schedule_timezone").notNull().default("UTC"),
+ next_run_at: ts("next_run_at"),
+ last_run_at: ts("last_run_at"),
+ last_check_id: uuid("last_check_id"),
+ current_check_id: uuid("current_check_id"),
+ locked_at: ts("locked_at"),
+ locked_until: ts("locked_until"),
+ retention_days: integer("retention_days").notNull().default(30),
+ estimated_credits_per_month: integer("estimated_credits_per_month"),
+ targets: jsonb("targets").notNull().default([]),
+ webhook: jsonb("webhook"),
+ notification: jsonb("notification"),
+ last_check_summary: jsonb("last_check_summary"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+ deleted_at: ts("deleted_at"),
+ goal: text("goal"),
+ judge_enabled: boolean("judge_enabled").notNull().default(false),
+});
+
+export const notification_preferences = pgTable("notification_preferences", {
+ id: uuid("id").notNull().defaultRandom(),
+ user_id: uuid("user_id").notNull(),
+ last_referral_notification: ts("last_referral_notification"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+ email_preferences: text("email_preferences")
+ .array()
+ .default(["rate_limit_warnings", "system_alerts"]),
+ unsubscribed_all: boolean("unsubscribed_all").default(false),
+});
+
+export const parses = pgTable("parses", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ url: text("url").notNull(),
+ is_successful: boolean("is_successful").notNull(),
+ error: text("error"),
+ time_taken: num("time_taken").notNull(),
+ team_id: uuid("team_id").notNull(),
+ options: jsonb("options"),
+ cost_tracking: jsonb("cost_tracking"),
+ pdf_num_pages: integer("pdf_num_pages"),
+ credits_cost: integer("credits_cost").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+});
+
+export const prices = pgTable("prices", {
+ id: text("id").notNull(),
+ product_id: text("product_id"),
+ active: boolean("active").notNull().default(true),
+ description: text("description"),
+ unit_amount: bigintNum("unit_amount"),
+ currency: text("currency").default("usd"),
+ type: text("type").default("recurring"),
+ interval: text("interval"),
+ interval_count: integer("interval_count").default(1),
+ trial_period_days: integer("trial_period_days").default(0),
+ metadata: jsonb("metadata").default({}),
+ credits: integer("credits").default(50),
+ is_usage: boolean("is_usage").default(false),
+ rate_limits: jsonb("rate_limits"),
+ concurrency: integer("concurrency"),
+ plan_priority: jsonb("plan_priority"),
+ upfront: boolean("upfront").default(false),
+ slug: text("slug")
+ .notNull()
+ .default(sql`gen_random_uuid()`),
+ should_be_graceful: boolean("should_be_graceful").default(false),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+ unit_amount_rollover_cap: smallint("unit_amount_rollover_cap")
+ .notNull()
+ .default(0),
+ associated_auto_recharge_price_id: text("associated_auto_recharge_price_id"),
+ exp_pack_max_per_month: integer("exp_pack_max_per_month"),
+});
+
+export const products = pgTable("products", {
+ id: text("id").notNull(),
+ active: boolean("active").notNull().default(true),
+ name: text("name"),
+ description: text("description"),
+ image: text("image"),
+ metadata: jsonb("metadata").notNull().default({}),
+ order: smallint("order"),
+ test_mode: boolean("test_mode").default(true),
+ is_enterprise: boolean("is_enterprise").default(false),
+ is_extract: boolean("is_extract").default(false),
+ is_displayed: boolean("is_displayed").default(false),
+ is_stripe_test: boolean("is_stripe_test").default(false),
+ slug: text("slug"),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+ type: text("type"),
+});
+
+export const requests = pgTable("requests", {
+ id: uuid("id").notNull(),
+ kind: text("kind").notNull(),
+ api_version: text("api_version").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ team_id: uuid("team_id").notNull(),
+ origin: text("origin").notNull(),
+ integration: text("integration"),
+ target_hint: text("target_hint").notNull(),
+ dr_clean_by: ts("dr_clean_by"),
+ api_key_id: bigintNum("api_key_id"),
+});
+
+export const scrapes = pgTable("scrapes", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ url: text("url").notNull(),
+ is_successful: boolean("is_successful").notNull(),
+ error: text("error"),
+ time_taken: num("time_taken").notNull(),
+ team_id: uuid("team_id").notNull(),
+ options: jsonb("options"),
+ cost_tracking: jsonb("cost_tracking"),
+ pdf_num_pages: integer("pdf_num_pages"),
+ credits_cost: integer("credits_cost").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ monitor_id: uuid("monitor_id"),
+ monitor_check_id: uuid("monitor_check_id"),
+});
+
+export const search_feedback = pgTable("search_feedback", {
+ id: uuid("id").notNull().defaultRandom(),
+ search_id: uuid("search_id").notNull(),
+ team_id: uuid("team_id").notNull(),
+ overall_rating: text("overall_rating").notNull(),
+ valuable_sources: jsonb("valuable_sources").notNull().default([]),
+ missing_content: jsonb("missing_content").notNull().default([]),
+ query_suggestions: text("query_suggestions"),
+ integration: text("integration"),
+ origin: text("origin"),
+ credits_refunded: integer("credits_refunded").notNull().default(0),
+ created_at: ts("created_at").notNull().defaultNow(),
+});
+
+export const searches = pgTable("searches", {
+ id: uuid("id").notNull(),
+ request_id: uuid("request_id").notNull(),
+ query: text("query").notNull(),
+ team_id: uuid("team_id").notNull(),
+ options: jsonb("options"),
+ time_taken: num("time_taken").notNull(),
+ created_at: ts("created_at").notNull().defaultNow(),
+ credits_cost: integer("credits_cost").notNull(),
+ is_successful: boolean("is_successful").notNull(),
+ error: text("error"),
+ num_results: integer("num_results").notNull(),
+});
+
+export const subscriptions = pgTable("subscriptions", {
+ id: text("id").notNull(),
+ user_id: uuid("user_id").notNull(),
+ status: text("status"),
+ metadata: jsonb("metadata"),
+ price_id: text("price_id"),
+ quantity: integer("quantity"),
+ cancel_at_period_end: boolean("cancel_at_period_end"),
+ created: ts("created").notNull().defaultNow(),
+ current_period_start: ts("current_period_start").notNull().defaultNow(),
+ current_period_end: ts("current_period_end").notNull().defaultNow(),
+ ended_at: ts("ended_at").defaultNow(),
+ cancel_at: ts("cancel_at").defaultNow(),
+ canceled_at: ts("canceled_at").defaultNow(),
+ trial_start: ts("trial_start").defaultNow(),
+ trial_end: ts("trial_end").defaultNow(),
+ team_id: uuid("team_id"),
+ is_usage: boolean("is_usage").default(false),
+ is_extract: boolean("is_extract").default(false),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+ expansion_associated_coupon: bigintNum("expansion_associated_coupon"),
+ pending_price_id: text("pending_price_id"),
+ pending_price_effective_at: ts("pending_price_effective_at"),
+ pending_schedule_id: text("pending_schedule_id"),
+});
+
+export const teams = pgTable("teams", {
+ id: uuid("id").notNull().defaultRandom(),
+ created_at: ts("created_at").defaultNow(),
+ name: text("name"),
+ banned: boolean("banned").default(false),
+ idmux_expires_at: ts("idmux_expires_at"),
+ updated_at: ts("updated_at").defaultNow(),
+ hmac_secret: text("hmac_secret")
+ .notNull()
+ .default(sql`encode(extensions.gen_random_bytes(32), 'hex'::text)`),
+ referrer_integration: varchar("referrer_integration"),
+ allocated_concurrent_browsers: integer("allocated_concurrent_browsers"),
+ org_id: uuid("org_id").notNull(),
+});
+
+export const user_notifications = pgTable("user_notifications", {
+ id: uuid("id").notNull().defaultRandom(),
+ team_id: uuid("team_id"),
+ notification_type: text("notification_type").notNull(),
+ sent_date: date("sent_date", { mode: "string" }).notNull(),
+ email_id: text("email_id"),
+ timestamp: ts("timestamp"),
+ read: boolean("read").default(false),
+ metadata: jsonb("metadata"),
+});
+
+export const user_teams = pgTable("user_teams", {
+ user_id: uuid("user_id").notNull(),
+ team_id: uuid("team_id").notNull(),
+ role: text("role").default("admin"),
+ created_at: ts("created_at").defaultNow(),
+});
+
+export const users = pgTable("users", {
+ id: uuid("id").notNull(),
+ full_name: text("full_name"),
+ avatar_url: text("avatar_url"),
+ billing_address: jsonb("billing_address"),
+ payment_method: jsonb("payment_method"),
+ email: text("email"),
+ team_id: uuid("team_id"),
+ created_on: ts("created_on").defaultNow(),
+ referrer_integration: varchar("referrer_integration"),
+});
+
+export const webhook_logs = pgTable("webhook_logs", {
+ id: uuid("id").notNull().defaultRandom(),
+ success: boolean("success").notNull(),
+ error: text("error"),
+ team_id: uuid("team_id").notNull(),
+ crawl_id: uuid("crawl_id").notNull(),
+ scrape_id: uuid("scrape_id"),
+ created_at: ts("created_at").notNull().defaultNow(),
+ url: text("url").notNull(),
+ status_code: smallint("status_code"),
+ event: text("event").notNull(),
+ latency_ms: integer("latency_ms").default(0),
+});
diff --git a/apps/api/src/lib/browser-session-activity.ts b/apps/api/src/lib/browser-session-activity.ts
index 74d7659b8..7f833b425 100644
--- a/apps/api/src/lib/browser-session-activity.ts
+++ b/apps/api/src/lib/browser-session-activity.ts
@@ -1,5 +1,6 @@
import { redisEvictConnection } from "../services/redis";
-import { supabase_service } from "../services/supabase";
+import { db } from "../db/connection";
+import * as schema from "../db/schema";
import { logger as _logger } from "./logger";
const logger = _logger.child({ module: "browser-sessions" });
@@ -36,16 +37,7 @@ export async function processBrowserSessionActivityJobs() {
const rows: BrowserSessionActivityEvent[] = raw.map(x => JSON.parse(x));
try {
- const { error } = await supabase_service
- .from("browser_session_activities")
- .insert(rows);
-
- if (error) {
- logger.error("Failed to insert browser session activities", {
- error,
- count: rows.length,
- });
- }
+ await db.insert(schema.browser_session_activities).values(rows);
} catch (err) {
logger.error("Error inserting browser session activities", {
err,
diff --git a/apps/api/src/lib/browser-sessions.ts b/apps/api/src/lib/browser-sessions.ts
index d3b27a3cc..05aa3dcd2 100644
--- a/apps/api/src/lib/browser-sessions.ts
+++ b/apps/api/src/lib/browser-sessions.ts
@@ -1,5 +1,7 @@
+import { and, desc, eq } from "drizzle-orm";
import { deleteKey, getValue, setValue } from "../services/redis";
-import { isPostgrestNoRowsError, supabase_service } from "../services/supabase";
+import { db } from "../db/connection";
+import * as schema from "../db/schema";
import { logger as _logger } from "./logger";
const logger = _logger.child({ module: "browser-sessions" });
@@ -29,8 +31,6 @@ interface BrowserSessionRow {
updated_at: string; // ISO timestamp
}
-const TABLE = "browser_sessions";
-
// ---------------------------------------------------------------------------
// CRUD helpers
// ---------------------------------------------------------------------------
@@ -49,29 +49,19 @@ export async function insertBrowserSession(
let lastError: any = null;
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
try {
- const { data, error } = await supabase_service
- .from(TABLE)
- .insert(full)
- .select()
- .single();
-
- if (error) {
- lastError = error;
- logger.error(
- "Error inserting browser session due to Supabase error, trying again",
- { error, id: row.id, attempt },
- );
- await new Promise(resolve => setTimeout(resolve, 75));
- continue;
- }
+ const [data] = await db
+ .insert(schema.browser_sessions)
+ .values(full)
+ .returning();
return data as BrowserSessionRow;
} catch (error) {
lastError = error;
- logger.error(
- "Error inserting browser session due to unknown error, trying again",
- { error, id: row.id, attempt },
- );
+ logger.error("Error inserting browser session, trying again", {
+ error,
+ id: row.id,
+ attempt,
+ });
await new Promise(resolve => setTimeout(resolve, 75));
}
}
@@ -89,72 +79,70 @@ export async function insertBrowserSession(
export async function getBrowserSession(
id: string,
): Promise<BrowserSessionRow | null> {
- const { data, error } = await supabase_service
- .from(TABLE)
- .select("*")
- .eq("id", id)
- .single();
-
- if (error) {
- if (isPostgrestNoRowsError(error)) return null;
+ try {
+ const [data] = await db
+ .select()
+ .from(schema.browser_sessions)
+ .where(eq(schema.browser_sessions.id, id))
+ .limit(1);
+ return (data ?? null) as BrowserSessionRow | null;
+ } catch (error) {
logger.error("Failed to get browser session", { error, id });
- throw new Error(`Failed to get browser session: ${error.message}`);
+ throw new Error(
+ `Failed to get browser session: ${error instanceof Error ? error.message : JSON.stringify(error)}`,
+ );
}
-
- return data as BrowserSessionRow;
}
export async function getBrowserSessionFromScrape(
id: string,
): Promise<BrowserSessionRow | null> {
- const { data, error } = await supabase_service
- .from(TABLE)
- .select("*")
- .eq("scrape_id", id)
- .single();
-
- if (error) {
- if (isPostgrestNoRowsError(error)) return null;
+ try {
+ const [data] = await db
+ .select()
+ .from(schema.browser_sessions)
+ .where(eq(schema.browser_sessions.scrape_id, id))
+ .limit(1);
+ return (data ?? null) as BrowserSessionRow | null;
+ } catch (error) {
logger.error("Failed to get browser session from scrape", { error, id });
throw new Error(
- `Failed to get browser session from scrape: ${error.message}`,
+ `Failed to get browser session from scrape: ${error instanceof Error ? error.message : JSON.stringify(error)}`,
);
}
-
- return data as BrowserSessionRow;
}
export async function listBrowserSessions(
teamId: string,
opts?: { status?: BrowserSessionStatus },
): Promise<BrowserSessionRow[]> {
- let query = supabase_service
- .from(TABLE)
- .select("*")
- .eq("team_id", teamId)
- .order("created_at", { ascending: false });
-
+ const conditions = [eq(schema.browser_sessions.team_id, teamId)];
if (opts?.status) {
- query = query.eq("status", opts.status);
+ conditions.push(eq(schema.browser_sessions.status, opts.status));
}
- const { data, error } = await query;
-
- if (error) {
+ try {
+ const data = await db
+ .select()
+ .from(schema.browser_sessions)
+ .where(and(...conditions))
+ .orderBy(desc(schema.browser_sessions.created_at));
+ return data as BrowserSessionRow[];
+ } catch (error) {
logger.error("Failed to list browser sessions", { error, teamId });
- throw new Error(`Failed to list browser sessions: ${error.message}`);
+ throw new Error(
+ `Failed to list browser sessions: ${error instanceof Error ? error.message : JSON.stringify(error)}`,
+ );
}
-
- return (data ?? []) as BrowserSessionRow[];
}
export async function updateBrowserSessionActivity(id: string): Promise<void> {
- const { error } = await supabase_service
- .from(TABLE)
- .update({ updated_at: new Date().toISOString() })
- .eq("id", id);
-
- if (error) {
+ try {
+ await db
+ .update(schema.browser_sessions)
+ .set({ updated_at: new Date().toISOString() })
+ .where(eq(schema.browser_sessions.id, id));
+ } catch (error) {
logger.warn("Failed to update browser session activity", { error, id });
}
}
@@ -162,40 +150,38 @@ export async function updateBrowserSessionActivity(id: string): Promise<void> {
export async function getBrowserSessionByBrowserId(
browserId: string,
): Promise<BrowserSessionRow | null> {
- const { data, error } = await supabase_service
- .from(TABLE)
- .select("*")
- .eq("browser_id", browserId)
- .single();
-
- if (error) {
- if (isPostgrestNoRowsError(error)) return null;
+ try {
+ const [data] = await db
+ .select()
+ .from(schema.browser_sessions)
+ .where(eq(schema.browser_sessions.browser_id, browserId))
+ .limit(1);
+ return (data ?? null) as BrowserSessionRow | null;
+ } catch (error) {
logger.error("Failed to get browser session by browser_id", {
error,
browserId,
});
throw new Error(
- `Failed to get browser session by browser_id: ${error.message}`,
+ `Failed to get browser session by browser_id: ${error instanceof Error ? error.message : JSON.stringify(error)}`,
);
}
-
- return data as BrowserSessionRow;
}
export async function updateBrowserSessionStatus(
id: string,
status: BrowserSessionStatus,
): Promise<void> {
- const { error } = await supabase_service
- .from(TABLE)
- .update({
- status,
- updated_at: new Date().toISOString(),
- deleted_at: status === "destroyed" ? new Date().toISOString() : null,
- })
- .eq("id", id);
-
- if (error) {
+ try {
+ await db
+ .update(schema.browser_sessions)
+ .set({
+ status,
+ updated_at: new Date().toISOString(),
+ deleted_at: status === "destroyed" ? new Date().toISOString() : null,
+ })
+ .where(eq(schema.browser_sessions.id, id));
+ } catch (error) {
logger.warn("Failed to update browser session status", { error, id });
}
}
@@ -204,35 +190,38 @@ export async function claimBrowserSessionDestroyed(
id: string,
): Promise<boolean> {
const now = new Date().toISOString();
- const { data, error } = await supabase_service
- .from(TABLE)
- .update({
- status: "destroyed" as BrowserSessionStatus,
- updated_at: now,
- deleted_at: now,
- })
- .eq("id", id)
- .eq("status", "active")
- .select("id");
-
- if (error) {
+ try {
+ const data = await db
+ .update(schema.browser_sessions)
+ .set({
+ status: "destroyed" as BrowserSessionStatus,
+ updated_at: now,
+ deleted_at: now,
+ })
+ .where(
+ and(
+ eq(schema.browser_sessions.id, id),
+ eq(schema.browser_sessions.status, "active"),
+ ),
+ )
+ .returning({ id: schema.browser_sessions.id });
+ return data.length > 0;
+ } catch (error) {
logger.warn("Failed to claim browser session destroyed", { error, id });
return false;
}
-
- return (data?.length ?? 0) > 0;
}
export async function updateBrowserSessionScrapeId(
id: string,
scrapeId: string,
): Promise<void> {
- const { error } = await supabase_service
- .from(TABLE)
- .update({ scrape_id: scrapeId, updated_at: new Date().toISOString() })
- .eq("id", id);
-
- if (error) {
+ try {
+ await db
+ .update(schema.browser_sessions)
+ .set({ scrape_id: scrapeId, updated_at: new Date().toISOString() })
+ .where(eq(schema.browser_sessions.id, id));
+ } catch (error) {
logger.warn("Failed to update browser session scrape_id", {
error,
id,
@@ -245,12 +234,15 @@ export async function updateBrowserSessionCreditsUsed(
id: string,
creditsUsed: number,
): Promise<void> {
- const { error } = await supabase_service
- .from(TABLE)
- .update({ credits_used: creditsUsed, updated_at: new Date().toISOString() })
- .eq("id", id);
-
- if (error) {
+ try {
+ await db
+ .update(schema.browser_sessions)
+ .set({
+ credits_used: creditsUsed,
+ updated_at: new Date().toISOString(),
+ })
+ .where(eq(schema.browser_sessions.id, id));
+ } catch (error) {
logger.warn("Failed to update browser session credits_used", {
error,
id,
diff --git a/apps/api/src/lib/engpicker.ts b/apps/api/src/lib/engpicker.ts
index dc9104686..74fddfdb8 100644
--- a/apps/api/src/lib/engpicker.ts
+++ b/apps/api/src/lib/engpicker.ts
@@ -3,7 +3,10 @@ import { z } from "zod";
import { scrapeOptions } from "../controllers/v2/types";
import { scrapeURL } from "../scraper/scrapeURL";
import type { Engine } from "../scraper/scrapeURL/engines";
-import { index_supabase_service } from "../services";
+import { and, desc, eq, inArray, isNull } from "drizzle-orm";
+import { dbIndex } from "../db/connection";
+import * as schema from "../db/schema";
+import { queryIndexAtDomainSplitLevelOmce } from "../db/rpc";
import { CostTracking } from "./cost-tracking";
import { getModel } from "./generic-ai";
import { logger as _logger } from "./logger";
@@ -15,7 +18,7 @@ import {
type EngpickerJob = {
id: number;
- domain_hash: string;
+ domain_hash: Buffer;
domain_level: number;
picked_up_at: string | null;
done: boolean;
@@ -115,17 +118,26 @@ export async function processEngpickerJob() {
method: "processEngpickerJob",
});
- const { data: jobData, error: getJobError } = await index_supabase_service
- .from("engpicker_queue")
- .update({
- picked_up_at: new Date().toISOString(),
- })
- .is("picked_up_at", null)
- .order("created_at", { ascending: false })
- .limit(1)
- .select("*");
-
- if (getJobError) {
+ let jobData: EngpickerJob[];
+ try {
+ const candidate = dbIndex
+ .select({ id: schema.engpicker_queue.id })
+ .from(schema.engpicker_queue)
+ .where(isNull(schema.engpicker_queue.picked_up_at))
+ .orderBy(desc(schema.engpicker_queue.created_at))
+ .limit(1);
+
+ jobData = await dbIndex
+ .update(schema.engpicker_queue)
+ .set({ picked_up_at: new Date().toISOString() })
+ .where(
+ and(
+ inArray(schema.engpicker_queue.id, candidate),
+ isNull(schema.engpicker_queue.picked_up_at),
+ ),
+ )
+ .returning();
+ } catch (getJobError) {
logger.error("Error picking up engpicker job", { getJobError });
await new Promise(resolve => setTimeout(resolve, 1000));
return;
@@ -146,18 +158,17 @@ export async function processEngpickerJob() {
// main shit here
- const { data: indexRows, error: indexRowsError } =
- await index_supabase_service
- .rpc("query_index_at_domain_split_level_omce", {
- i_level: job.domain_level,
- i_domain_hash: job.domain_hash,
- i_newer_than: new Date(
- new Date(job.created_at).valueOf() - 1000 * 60 * 60 * 24,
- ).toISOString(),
- })
- .limit(100);
-
- if (indexRowsError || !indexRows) {
+ let indexRows: { url: string }[];
+ try {
+ indexRows = await queryIndexAtDomainSplitLevelOmce<{ url: string }>(
+ job.domain_level,
+ job.domain_hash,
+ new Date(
+ new Date(job.created_at).valueOf() - 1000 * 60 * 60 * 24,
+ ).toISOString(),
+ 100,
+ );
+ } catch (indexRowsError) {
logger.error("Error querying index rows", { indexRowsError });
await new Promise(resolve => setTimeout(resolve, 1000));
return;
@@ -287,27 +298,23 @@ export async function processEngpickerJob() {
// This is the verdict - "TlsClientOk", "ChromeCdpRequired", or "Uncertain"
const verdict = verdictResult.verdict;
- const { error: insertVerdictError } = await index_supabase_service
- .from("engpicker_verdicts")
- .insert({
+ try {
+ await dbIndex.insert(schema.engpicker_verdicts).values({
domain_hash: job.domain_hash,
verdict,
});
-
- if (insertVerdictError) {
+ } catch (insertVerdictError) {
logger.error("Error inserting engpicker verdict", { insertVerdictError });
await new Promise(resolve => setTimeout(resolve, 1000));
return;
}
- const { error: updateJobError } = await index_supabase_service
- .from("engpicker_queue")
- .update({
- done: true,
- })
- .eq("id", job.id);
-
- if (updateJobError) {
+ try {
+ await dbIndex
+ .update(schema.engpicker_queue)
+ .set({ done: true })
+ .where(eq(schema.engpicker_queue.id, job.id));
+ } catch (updateJobError) {
logger.error("Error updating engpicker job", { updateJobError });
await new Promise(resolve => setTimeout(resolve, 1000));
return;
diff --git a/apps/api/src/lib/extract/team-id-sync.ts b/apps/api/src/lib/extract/team-id-sync.ts
index e477583ab..2c0c079b0 100644
--- a/apps/api/src/lib/extract/team-id-sync.ts
+++ b/apps/api/src/lib/extract/team-id-sync.ts
@@ -1,18 +1,17 @@
-import { supabase_rr_service, supabase_service } from "../../services/supabase";
+import { eq } from "drizzle-orm";
+import { dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { logger } from "../logger";
import { withAuth } from "../withAuth";
async function getTeamIdSyncBOriginal(teamId: string) {
try {
- const { data, error } = await supabase_rr_service
- .from("eb-sync")
- .select("team_id")
- .eq("team_id", teamId)
+ const data = await dbRr
+ .select({ team_id: schema.eb_sync.team_id })
+ .from(schema.eb_sync)
+ .where(eq(schema.eb_sync.team_id, teamId))
.limit(1);
- if (error) {
- throw new Error("Error getting team id (sync b)");
- }
return data[0] ?? null;
} catch (error) {
logger.error("Error getting team id (sync b)", error);
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts
index a7372e3fe..5943fc929 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts
@@ -1,4 +1,6 @@
-import { supabase_service } from "../../services/supabase";
+import { and, desc, eq, gte } from "drizzle-orm";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
import { config } from "../../config";
import { logger } from "../logger";
import { normalizeUrl, normalizeUrlOnlyHostname } from "../canonical-url";
@@ -21,28 +23,27 @@ export async function getLlmsTextFromCache(
const originUrl = normalizeUrlOnlyHostname(url);
try {
- const { data, error } = await supabase_service
- .from("llm_texts")
- .select("*")
- .eq("origin_url", originUrl)
- .gte("max_urls", maxUrls) // Changed to gte since we want cached results with more URLs than requested
- .order("updated_at", { ascending: false })
- .limit(1)
- .single();
-
- if (error) {
- return null;
- }
+ const [data] = await db
+ .select()
+ .from(schema.llm_texts)
+ .where(
+ and(
+ eq(schema.llm_texts.origin_url, originUrl),
+ gte(schema.llm_texts.max_urls, maxUrls), // gte since we want cached results with more URLs than requested
+ ),
+ )
+ .orderBy(desc(schema.llm_texts.updated_at))
+ .limit(1);
// Check if data is older than 1 week
const oneWeekAgo = new Date();
oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);
- if (!data || new Date(data.updated_at) < oneWeekAgo) {
+ if (!data || !data.updated_at || new Date(data.updated_at) < oneWeekAgo) {
return null;
}
- return data;
+ return data as LlmsTextCache;
} catch (error) {
logger.error("Failed to fetch LLMs text from cache", { error, originUrl });
return null;
@@ -63,52 +64,50 @@ export async function saveLlmsTextToCache(
try {
// First check if there's an existing entry
- const { data: existingData } = await supabase_service
- .from("llm_texts")
- .select("*")
- .eq("origin_url", originUrl)
- .single();
+ const [existingData] = await db
+ .select({ id: schema.llm_texts.id })
+ .from(schema.llm_texts)
+ .where(eq(schema.llm_texts.origin_url, originUrl))
+ .limit(1);
if (existingData) {
// Update existing entry
- const { error } = await supabase_service
- .from("llm_texts")
- .update({
- llmstxt,
- llmstxt_full,
- max_urls: maxUrls,
- updated_at: new Date().toISOString(),
- })
- .eq("origin_url", originUrl);
-
- if (error) {
- logger.error("Error updating LLMs text in cache", { error, originUrl });
- } else {
+ try {
+ await db
+ .update(schema.llm_texts)
+ .set({
+ llmstxt,
+ llmstxt_full,
+ max_urls: maxUrls,
+ updated_at: new Date().toISOString(),
+ })
+ .where(eq(schema.llm_texts.origin_url, originUrl));
logger.debug("Successfully updated cached LLMs text", {
originUrl,
maxUrls,
});
+ } catch (error) {
+ logger.error("Error updating LLMs text in cache", { error, originUrl });
}
} else {
// Insert new entry
- const { error } = await supabase_service.from("llm_texts").insert({
- origin_url: originUrl,
- llmstxt,
- llmstxt_full,
- max_urls: maxUrls,
- updated_at: new Date().toISOString(),
- });
-
- if (error) {
- logger.error("Error inserting LLMs text to cache", {
- error,
- originUrl,
+ try {
+ await db.insert(schema.llm_texts).values({
+ origin_url: originUrl,
+ llmstxt,
+ llmstxt_full,
+ max_urls: maxUrls,
+ updated_at: new Date().toISOString(),
});
- } else {
logger.debug("Successfully inserted new cached LLMs text", {
originUrl,
maxUrls,
});
+ } catch (error) {
+ logger.error("Error inserting LLMs text to cache", {
+ error,
+ originUrl,
+ });
}
}
} catch (error) {
diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.ts
index b9741e6ee..0c9d8db59 100644
--- a/apps/api/src/lib/supabase-jobs.ts
+++ b/apps/api/src/lib/supabase-jobs.ts
@@ -1,81 +1,66 @@
import type { Logger } from "winston";
-import { supabase_rr_service, supabase_service } from "../services/supabase";
+import { eq, inArray, and } from "drizzle-orm";
+import { db, dbRr } from "../db/connection";
+import * as schema from "../db/schema";
import { logger } from "./logger";
import * as Sentry from "@sentry/node";
-// ============================================================================
-// NEW TABLES: scrapes, requests, crawls, etc.
-// ============================================================================
-
/**
- * Get a single scrape by ID from the new scrapes table
+ * Get a single scrape by ID from the scrapes table
* @param scrapeId ID of Scrape
* @returns Scrape data or null
*/
-export const supabaseGetScrapeById = async (scrapeId: string) => {
- const { data, error } = await supabase_rr_service
- .from("scrapes")
- .select("*")
- .eq("id", scrapeId)
- .single();
-
- if (error) {
- return null;
- }
-
- if (!data) {
+export const supabaseGetScrapeById = async (scrapeId: string): Promise<any> => {
+ try {
+ const [data] = await dbRr
+ .select()
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.id, scrapeId))
+ .limit(1);
+ return data ?? null;
+ } catch (error) {
return null;
}
-
- return data;
};
/**
- * Get multiple scrapes by ID from the new scrapes table
+ * Get multiple scrapes by ID from the scrapes table
* @param scrapeIds IDs of Scrapes
* @returns Scrape data array
*/
-export const supabaseGetScrapesById = async (scrapeIds: string[]) => {
- const { data, error } = await supabase_rr_service
- .from("scrapes")
- .select()
- .in("id", scrapeIds);
-
- if (error) {
+export const supabaseGetScrapesById = async (
+ scrapeIds: string[],
+): Promise<any[]> => {
+ try {
+ return await dbRr
+ .select()
+ .from(schema.scrapes)
+ .where(inArray(schema.scrapes.id, scrapeIds));
+ } catch (error) {
logger.error(`Error in supabaseGetScrapesById: ${error}`);
Sentry.captureException(error);
return [];
}
-
- if (!data) {
- return [];
- }
-
- return data;
};
/**
- * Get multiple scrapes by request ID (crawl/batch scrape ID) from the new scrapes table
+ * Get multiple scrapes by request ID (crawl/batch scrape ID) from the scrapes table
* @param requestId ID of the parent request (crawl or batch scrape)
* @returns Scrape data array
*/
-export const supabaseGetScrapesByRequestId = async (requestId: string) => {
- const { data, error } = await supabase_rr_service
- .from("scrapes")
- .select()
- .eq("request_id", requestId);
-
- if (error) {
+export const supabaseGetScrapesByRequestId = async (
+ requestId: string,
+): Promise<any[]> => {
+ try {
+ return await dbRr
+ .select()
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.request_id, requestId));
+ } catch (error) {
logger.error(`Error in supabaseGetScrapesByRequestId: ${error}`);
Sentry.captureException(error);
return [];
}
-
- if (!data) {
- return [];
- }
-
- return data;
};
/**
@@ -86,100 +71,86 @@ export const supabaseGetScrapesByRequestId = async (requestId: string) => {
*/
export const supabaseGetScrapeByIdOnlyData = async (
scrapeId: string,
- logger?: Logger,
-) => {
- const { data, error } = await supabase_rr_service
- .from("scrapes")
- .select("team_id")
- .eq("id", scrapeId)
- .single();
-
- if (error) {
- if (logger) {
- logger.error("Error in supabaseGetScrapeByIdOnlyData", { error });
+ log?: Logger,
+): Promise<any> => {
+ try {
+ const [data] = await dbRr
+ .select({ team_id: schema.scrapes.team_id })
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.id, scrapeId))
+ .limit(1);
+ return data ?? null;
+ } catch (error) {
+ if (log) {
+ log.error("Error in supabaseGetScrapeByIdOnlyData", { error });
}
return null;
}
-
- if (!data) {
- return null;
- }
-
- return data;
};
-export const supabaseGetExtractByIdDirect = async (extractId: string) => {
- const { data, error } = await supabase_service
- .from("extracts")
- .select("*")
- .eq("id", extractId)
- .single();
-
- if (error) {
- return null;
- }
-
- if (!data) {
+export const supabaseGetExtractByIdDirect = async (
+ extractId: string,
+): Promise<any> => {
+ try {
+ const [data] = await db
+ .select()
+ .from(schema.extracts)
+ .where(eq(schema.extracts.id, extractId))
+ .limit(1);
+ return data ?? null;
+ } catch (error) {
return null;
}
-
- return data;
};
export const supabaseGetExtractRequestByIdDirect = async (
extractId: string,
-) => {
- const { data, error } = await supabase_service
- .from("requests")
- .select("*")
- .eq("id", extractId)
- .in("kind", ["extract", "agent"])
- .single();
-
- if (error) {
+): Promise<any> => {
+ try {
+ const [data] = await db
+ .select()
+ .from(schema.requests)
+ .where(
+ and(
+ eq(schema.requests.id, extractId),
+ inArray(schema.requests.kind, ["extract", "agent"]),
+ ),
+ )
+ .limit(1);
+ return data ?? null;
+ } catch (error) {
return null;
}
-
- if (!data) {
- return null;
- }
-
- return data;
};
-export const supabaseGetAgentRequestByIdDirect = async (agentId: string) => {
- const { data, error } = await supabase_service
- .from("requests")
- .select("*")
- .eq("id", agentId)
- .eq("kind", "agent")
- .single();
-
- if (error) {
+export const supabaseGetAgentRequestByIdDirect = async (
+ agentId: string,
+): Promise<any> => {
+ try {
+ const [data] = await db
+ .select()
+ .from(schema.requests)
+ .where(
+ and(eq(schema.requests.id, agentId), eq(schema.requests.kind, "agent")),
+ )
+ .limit(1);
+ return data ?? null;
+ } catch (error) {
return null;
}
-
- if (!data) {
- return null;
- }
-
- return data;
};
-export const supabaseGetAgentByIdDirect = async (agentId: string) => {
- const { data, error } = await supabase_service
- .from("agents")
- .select("*")
- .eq("id", agentId)
- .single();
-
- if (error) {
+export const supabaseGetAgentByIdDirect = async (
+ agentId: string,
+): Promise<any> => {
+ try {
+ const [data] = await db
+ .select()
+ .from(schema.agents)
+ .where(eq(schema.agents.id, agentId))
+ .limit(1);
+ return data ?? null;
+ } catch (error) {
return null;
}
-
- if (!data) {
- return null;
- }
-
- return data;
};
diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.ts
index 17b9d5faa..8ab72ecf4 100644
--- a/apps/api/src/lib/zdrcleaner.ts
+++ b/apps/api/src/lib/zdrcleaner.ts
@@ -1,5 +1,8 @@
import "dotenv/config";
-import { supabase_service } from "../services/supabase";
+import { inArray } from "drizzle-orm";
+import { db } from "../db/connection";
+import * as schema from "../db/schema";
+import { getZdrCleanupBatch } from "../db/rpc";
import { removeJobFromGCS } from "./gcs-jobs";
import { logger as _logger } from "./logger";
import { config } from "../config";
@@ -21,20 +24,8 @@ export async function zdrcleaner() {
// Call the RPC to get all blobs (scrapes, searches, extracts, maps, llmstxts, deep_researches)
// associated with requests that need cleanup.
// The RPC handles the dr_clean_by filtering logic (team-specific vs scheduled)
- const { data: _rows, error } = await supabase_service.rpc(
- "get_zdr_cleanup_batch_2",
- {
- p_limit: 1000,
- },
- );
-
- if (error) {
- throw new Error("Error calling get_zdr_cleanup_batch_2 RPC", {
- cause: error,
- });
- }
-
- const rows: { request_id: string; ids: string[] }[] = _rows;
+ const rows: { request_id: string; ids: string[] }[] =
+ await getZdrCleanupBatch(1000);
if (!rows || rows.length === 0) {
logger.debug("zdrcleaner batch completed with no rows to process", {
@@ -89,20 +80,13 @@ export async function zdrcleaner() {
const updateErrors: any[] = [];
if (fullyCleanedRequests.length > 0) {
- // Process in batches of 100 to deal with URL limits on PostgREST
- for (let i = 0; i < fullyCleanedRequests.length; i += 100) {
- const batch = fullyCleanedRequests.slice(i, i + 100);
- try {
- await supabase_service
- .from("requests")
- .update({
- dr_clean_by: null,
- })
- .in("id", batch)
- .throwOnError();
- } catch (error) {
- updateErrors.push(error);
- }
+ try {
+ await db
+ .update(schema.requests)
+ .set({ dr_clean_by: null })
+ .where(inArray(schema.requests.id, fullyCleanedRequests));
+ } catch (error) {
+ updateErrors.push(error);
}
}
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 8c092e427..e4215cb82 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -23,7 +23,7 @@ import { isSelfHosted } from "../lib/deployment";
import { validate as isUuid } from "uuid";
import { config } from "../config";
-import { supabase_service } from "../services/supabase";
+import { getAgentFreeRequestsLeft } from "../db/rpc";
import { autumnService } from "../services/autumn/autumn.service";
export function checkCreditsMiddleware(
@@ -99,22 +99,16 @@ export function checkCreditsMiddleware(
if (req.path.startsWith("/agent")) {
if (config.USE_DB_AUTHENTICATION) {
- const { data, error: freeRequestError } = await supabase_service.rpc(
- "get_agent_free_requests_left",
- {
- i_team_id: req.auth.team_id,
- },
- );
-
- if (freeRequestError) {
+ try {
+ const data = await getAgentFreeRequestsLeft(req.auth.team_id);
+ if (data?.[0]?.free_requests_left !== 0) {
+ return next();
+ }
+ } catch (freeRequestError) {
logger.warn("Failed to get agent free requests left", {
error: freeRequestError,
teamId: req.auth.team_id,
});
- } else {
- if (data?.[0]?.free_requests_left !== 0) {
- return next();
- }
}
}
}
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index af51495ca..eabf66ed1 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -3,10 +3,8 @@ import { v7 as uuidv7 } from "uuid";
import { config } from "../../../config";
import { parse } from "tldts";
import { TeamFlags } from "../../../controllers/v1/types";
-import {
- supabase_rr_service,
- supabase_service,
-} from "../../../services/supabase";
+import { db, dbRr } from "../../../db/connection";
+import * as schema from "../../../db/schema";
configDotenv();
@@ -39,7 +37,7 @@ async function flushHits(): Promise<void> {
hitBuffer = [];
if (config.USE_DB_AUTHENTICATION !== true) return;
try {
- await supabase_service.from("blocklist_hits").insert(batch);
+ await db.insert(schema.blocklist_hits).values(batch);
} catch {}
}
@@ -114,13 +112,13 @@ export async function initializeBlocklist() {
return;
}
- const { data, error } = await supabase_rr_service
- .from("blocklist")
- .select("*")
- .single();
-
- if (error) {
- throw new Error(`Error getting blocklist: ${error.message}`);
+ let data: { data: any } | undefined;
+ try {
+ [data] = await dbRr.select().from(schema.blocklist).limit(1);
+ } catch (error) {
+ throw new Error(
+ `Error getting blocklist: ${error instanceof Error ? error.message : JSON.stringify(error)}`,
+ );
}
if (!data) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 02577f8bd..77ad5fc40 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -6,7 +6,6 @@ import { Meta } from "../..";
import {
getIndexFromGCS,
hashURL,
- index_supabase_service,
normalizeURLForIndex,
saveIndexToGCS,
generateURLSplits,
@@ -14,6 +13,7 @@ import {
generateDomainSplits,
addOMCEJob,
} from "../../../../services";
+import { queryMaxAge, indexGetRecent4 } from "../../../../db/rpc";
import {
AgentIndexOnlyError,
EngineError,
@@ -234,23 +234,18 @@ export async function scrapeURLWithIndex(
try {
maxAge = await Promise.race([
(async () => {
- const { data, error } = await index_supabase_service.rpc(
- "query_max_age",
- {
- i_domain_hash: domainSplitsHash[level],
- },
- );
-
- if (error || !data || data.length === 0) {
- meta.logger.warn("Failed to get max age from DB", {
- error,
- });
+ try {
+ const data = await queryMaxAge(domainSplitsHash[level]);
+ if (!data || data.length === 0) {
+ return 2 * 24 * 60 * 60 * 1000; // 2 days
+ }
+ return data[0].max_age ?? 2 * 24 * 60 * 60 * 1000; // 2 days
+ } catch (error) {
+ meta.logger.warn("Failed to get max age from DB", { error });
return 2 * 24 * 60 * 60 * 1000; // 2 days
}
-
- return data[0].max_age ?? 2 * 24 * 60 * 60 * 1000; // 2 days
})(),
- new Promise(resolve =>
+ new Promise<number>(resolve =>
setTimeout(() => {
resolve(2 * 24 * 60 * 60 * 1000); // 2 days
}, 200),
@@ -267,29 +262,27 @@ export async function scrapeURLWithIndex(
const checkpoint1 = Date.now();
- const { data, error } = await index_supabase_service.rpc(
- "index_get_recent_4",
- {
- p_url_hash: urlHash,
- p_max_age_ms: maxAge,
- p_is_mobile: meta.options.mobile,
- p_block_ads: meta.options.blockAds,
- p_feature_screenshot: meta.featureFlags.has("screenshot"),
- p_feature_screenshot_fullscreen: meta.featureFlags.has(
+ let data: { id: string; created_at: string; status: number }[];
+ try {
+ data = await indexGetRecent4({
+ url_hash: urlHash,
+ max_age_ms: maxAge,
+ is_mobile: meta.options.mobile,
+ block_ads: meta.options.blockAds,
+ feature_screenshot: meta.featureFlags.has("screenshot"),
+ feature_screenshot_fullscreen: meta.featureFlags.has(
"screenshot@fullScreen",
),
- p_location_country: meta.options.location?.country ?? null,
- p_location_languages:
+ location_country: meta.options.location?.country ?? null,
+ location_languages:
(meta.options.location?.languages?.length ?? 0) > 0
- ? meta.options.location?.languages
+ ? (meta.options.location?.languages ?? null)
: null,
- p_wait_time_ms: meta.options.waitFor,
- p_is_stealth: meta.featureFlags.has("stealthProxy"),
- p_min_age_ms: meta.options.minAge ?? null,
- },
- );
-
- if (error || !data) {
+ wait_time_ms: meta.options.waitFor,
+ is_stealth: meta.featureFlags.has("stealthProxy"),
+ min_age_ms: meta.options.minAge ?? null,
+ });
+ } catch (error) {
throw new EngineError("Failed to retrieve URL from DB index", {
cause: error,
});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
index 16850f846..a36ed8ffc 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
@@ -1,4 +1,4 @@
-import { supabase_service } from "../../../services/supabase";
+import { diffGetLastScrape } from "../../../db/rpc";
import { Document } from "../../../controllers/v1/types";
import { Meta } from "../index";
import gitDiff from "git-diff";
@@ -89,11 +89,20 @@ export async function deriveDiff(
}
const start = Date.now();
- const res = await supabase_service.rpc("diff_get_last_scrape_v7", {
- i_team_id: meta.internalOptions.teamId,
- i_url: document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,
- i_tag: changeTrackingFormat?.tag ?? null,
- });
+ let resData: { o_job_id: string; o_date_added: string }[];
+ try {
+ resData = await diffGetLastScrape(
+ meta.internalOptions.teamId!,
+ document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,
+ changeTrackingFormat?.tag ?? null,
+ );
+ } catch (error) {
+ meta.logger.error("Error fetching previous scrape", { error });
+ document.warning =
+ "Comparing failed, please try again later." +
+ (document.warning ? ` ${document.warning}` : "");
+ return document;
+ }
const end = Date.now();
if (end - start > 100) {
meta.logger.debug("Diffing took a while", {
@@ -111,7 +120,7 @@ export async function deriveDiff(
o_date_added: string;
}
| undefined
- | null = (res.data ?? [])[0] as any;
+ | null = resData[0];
const rawJob = data?.o_job_id ? await getJobFromGCS(data.o_job_id) : null;
const job: Document | null = rawJob?.[0] ?? null;
@@ -265,7 +274,7 @@ export async function deriveDiff(
(document.warning ? ` ${document.warning}` : "");
}
}
- } else if (!res.error) {
+ } else {
document.changeTracking = {
previousScrapeAt: null,
changeStatus: document.metadata.statusCode === 404 ? "removed" : "new",
@@ -273,11 +282,6 @@ export async function deriveDiff(
? "hidden"
: "visible",
};
- } else {
- meta.logger.error("Error fetching previous scrape", { error: res.error });
- document.warning =
- "Comparing failed, please try again later." +
- (document.warning ? ` ${document.warning}` : "");
}
}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 1cc071ccd..6190778a0 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -11,7 +11,6 @@ import {
performCleanContent,
} from "./llmExtract";
import { performQuery } from "./query";
-import { uploadScreenshot } from "./uploadScreenshot";
import { removeBase64Images } from "./removeBase64Images";
import { performAgent } from "./agent";
import { performAttributes } from "./performAttributes";
@@ -574,7 +573,6 @@ const transformerStack: Transformer[] = [
deriveImagesFromHTML,
deriveBrandingFromActions,
deriveMetadataFromRawHTML,
- uploadScreenshot,
...(useIndex ? [sendDocumentToIndex] : []),
...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time search
performLLMExtract,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts b/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts
deleted file mode 100644
index 55cb5c9f8..000000000
--- a/apps/api/src/scraper/scrapeURL/transformers/uploadScreenshot.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-// This file is an exception to the "no supabase in scrapeURL" rule,
-// and it makes me sad. - mogery
-
-import { supabase_service } from "../../../services/supabase";
-import { config } from "../../../config";
-import { Meta } from "..";
-import { Document } from "../../../controllers/v1/types";
-
-export function uploadScreenshot(meta: Meta, document: Document): Document {
- if (
- config.USE_DB_AUTHENTICATION &&
- document.screenshot !== undefined &&
- document.screenshot.startsWith("data:")
- ) {
- meta.logger.debug("Uploading screenshot to Supabase...");
-
- const fileName = `screenshot-${crypto.randomUUID()}.png`;
-
- supabase_service.storage
- .from("media")
- .upload(
- fileName,
- Buffer.from(document.screenshot.split(",")[1], "base64"),
- {
- cacheControl: "3600",
- upsert: false,
- contentType: document.screenshot.split(":")[1].split(";")[0],
- },
- );
-
- document.screenshot = `https://service.firecrawl.dev/storage/v1/object/public/media/${encodeURIComponent(fileName)}`;
- }
-
- return document;
-}
diff --git a/apps/api/src/services/agent-sponsor.ts b/apps/api/src/services/agent-sponsor.ts
index 14379e9b4..8aae62fb3 100644
--- a/apps/api/src/services/agent-sponsor.ts
+++ b/apps/api/src/services/agent-sponsor.ts
@@ -1,10 +1,8 @@
+import { eq } from "drizzle-orm";
import { logger } from "../lib/logger";
import { deleteKey, getValue, setValue } from "./redis";
-import {
- isPostgrestNoRowsError,
- supabase_rr_service,
- supabase_service,
-} from "./supabase";
+import { db, dbRr } from "../db/connection";
+import * as schema from "../db/schema";
type AgentSponsorStatus = {
status: "pending" | "verified" | "blocked";
@@ -42,37 +40,30 @@ export async function getAgentSponsorStatus({
}
try {
- const { data, error } = await supabase_rr_service
- .from("agent_sponsors")
- .select("status, verification_deadline, email")
- .eq("api_key_id", apiKeyId)
- .single();
+ const [data] = await dbRr
+ .select({
+ status: schema.agent_sponsors.status,
+ verification_deadline: schema.agent_sponsors.verification_deadline,
+ email: schema.agent_sponsors.email,
+ })
+ .from(schema.agent_sponsors)
+ .where(eq(schema.agent_sponsors.api_key_id, apiKeyId))
+ .limit(1);
- if (error) {
- // Only cache "no sponsor" when it's a confirmed no-rows result.
- // Do not cache on other errors (e.g. connection/timeout) so we retry on next request.
- if (isPostgrestNoRowsError(error)) {
- await setValue(
- cacheKey,
- JSON.stringify(CACHE_MISS_SENTINEL),
- AGENT_SPONSOR_CACHE_TTL,
- );
- } else {
- logger.error("Failed to look up agent sponsor status", {
- apiKeyId,
- error,
- });
- }
- return null;
- }
if (!data) {
+ // Confirmed no-rows result — cache the "no sponsor" sentinel.
+ await setValue(
+ cacheKey,
+ JSON.stringify(CACHE_MISS_SENTINEL),
+ AGENT_SPONSOR_CACHE_TTL,
+ );
return null;
}
const result: AgentSponsorStatus = {
- status: data.status,
- verification_deadline: data.verification_deadline,
- email: data.email,
+ status: data.status as AgentSponsorStatus["status"],
+ verification_deadline: data.verification_deadline!,
+ email: data.email!,
};
await setValue(cacheKey, JSON.stringify(result), AGENT_SPONSOR_CACHE_TTL);
@@ -110,22 +101,28 @@ async function getAgentSponsorByToken({
status: string;
verification_deadline: string;
agent_name: string;
- sandboxed_team_id: string;
- api_key_id: number;
+ sandboxed_team_id: string | null;
+ api_key_id: number | null;
} | null> {
- const { data, error } = await supabase_service
- .from("agent_sponsors")
- .select(
- "id, email, status, verification_deadline, agent_name, sandboxed_team_id, api_key_id",
- )
- .eq("verification_token", agent_signup_token)
- .single();
-
- if (error || !data) {
+ try {
+ const [data] = await db
+ .select({
+ id: schema.agent_sponsors.id,
+ email: schema.agent_sponsors.email,
+ status: schema.agent_sponsors.status,
+ verification_deadline: schema.agent_sponsors.verification_deadline,
+ agent_name: schema.agent_sponsors.agent_name,
+ sandboxed_team_id: schema.agent_sponsors.sandboxed_team_id,
+ api_key_id: schema.agent_sponsors.api_key_id,
+ })
+ .from(schema.agent_sponsors)
+ .where(eq(schema.agent_sponsors.verification_token, agent_signup_token))
+ .limit(1);
+
+ return data ?? null;
+ } catch {
return null;
}
-
- return data;
}
/**
@@ -136,12 +133,12 @@ async function markSponsorVerified({
}: {
sponsorId: number;
}): Promise<void> {
- const { error } = await supabase_service
- .from("agent_sponsors")
- .update({ status: "verified", verified_at: new Date().toISOString() })
- .eq("id", sponsorId);
-
- if (error) {
+ try {
+ await db
+ .update(schema.agent_sponsors)
+ .set({ status: "verified", verified_at: new Date().toISOString() })
+ .where(eq(schema.agent_sponsors.id, sponsorId));
+ } catch (error) {
logger.error("Failed to mark sponsor as verified", { sponsorId, error });
throw error;
}
@@ -155,12 +152,12 @@ async function markSponsorBlocked({
}: {
sponsorId: number;
}): Promise<void> {
- const { error } = await supabase_service
- .from("agent_sponsors")
- .update({ status: "blocked" })
- .eq("id", sponsorId);
-
- if (error) {
+ try {
+ await db
+ .update(schema.agent_sponsors)
+ .set({ status: "blocked" })
+ .where(eq(schema.agent_sponsors.id, sponsorId));
+ } catch (error) {
logger.error("Failed to mark sponsor as blocked", { sponsorId, error });
throw error;
}
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index 27a343cdf..87d52316c 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -3,7 +3,7 @@
*
* All external I/O is mocked:
* - autumnClient → jest.fn() stubs on customers / entities / track
- * - supabase_rr_service → stubbed Supabase query builder
+ * - dbRr → stubbed Drizzle query builder
*/
import { jest } from "@jest/globals";
@@ -44,15 +44,13 @@ jest.mock("../client", () => ({
},
}));
-// Minimal Supabase query-builder stub: .from().select().eq().single() → resolves data/error.
-const makeSupabaseStub = (data: unknown, error: unknown = null) => ({
- from: () => ({
- select: () => ({
- eq: () => ({
- single: () => Promise.resolve({ data, error }),
- gte: () => Promise.resolve({ data: [], error: null }),
+// Minimal Drizzle query-builder stub: .select().from().where().limit() → rows.
+const makeDbStub = (data: unknown) => ({
+ select: () => ({
+ from: () => ({
+ where: () => ({
+ limit: () => Promise.resolve(data ? [data] : []),
}),
- gte: () => Promise.resolve({ data: [], error: null }),
}),
}),
});
@@ -62,9 +60,9 @@ let supabaseStubData: { data: unknown; error: unknown } = {
error: null,
};
-jest.mock("../../supabase", () => ({
- get supabase_rr_service() {
- return makeSupabaseStub(supabaseStubData.data, supabaseStubData.error);
+jest.mock("../../../db/connection", () => ({
+ get dbRr() {
+ return makeDbStub(supabaseStubData.data);
},
}));
diff --git a/apps/api/src/services/autumn/__tests__/usage.test.ts b/apps/api/src/services/autumn/__tests__/usage.test.ts
index 54900d13c..857dd578a 100644
--- a/apps/api/src/services/autumn/__tests__/usage.test.ts
+++ b/apps/api/src/services/autumn/__tests__/usage.test.ts
@@ -27,27 +27,20 @@ jest.mock("../client", () => ({
},
}));
-jest.mock("../../supabase", () => ({
- get supabase_rr_service() {
+jest.mock("../../../db/connection", () => ({
+ get dbRr() {
return {
- from: (table: string) => ({
- select: () => {
- if (table === "teams") {
- return {
- eq: () => ({
- single: () => Promise.resolve(teamLookup),
- }),
- };
- }
-
- if (table === "api_keys") {
- return {
- in: () => Promise.resolve({ data: apiKeysData, error: null }),
- };
- }
-
- return {};
- },
+ select: () => ({
+ from: () => ({
+ where: () => {
+ // api_keys path awaits the builder directly; teams path calls .limit(1)
+ const apiKeysPromise = Promise.resolve(apiKeysData);
+ return Object.assign(apiKeysPromise, {
+ limit: () =>
+ Promise.resolve(teamLookup.data ? [teamLookup.data] : []),
+ });
+ },
+ }),
}),
};
},
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index d9e2cfb9e..1aa9ce998 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -1,7 +1,9 @@
import { randomUUID } from "crypto";
import { config } from "../../config";
import { logger } from "../../lib/logger";
-import { supabase_rr_service } from "../supabase";
+import { eq } from "drizzle-orm";
+import { dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { autumnClient } from "./client";
import type {
CreateEntityParams,
@@ -87,13 +89,12 @@ export class AutumnService {
}
private async lookupOrgIdForTeam(teamId: string): Promise<string> {
- const { data, error } = await supabase_rr_service
- .from("teams")
- .select("org_id")
- .eq("id", teamId)
- .single();
+ const [data] = await dbRr
+ .select({ org_id: schema.teams.org_id })
+ .from(schema.teams)
+ .where(eq(schema.teams.id, teamId))
+ .limit(1);
- if (error) throw error;
if (!data?.org_id) {
throw new Error(`Missing org_id for team ${teamId}`);
}
diff --git a/apps/api/src/services/autumn/usage.ts b/apps/api/src/services/autumn/usage.ts
index ad68d7d97..c67ccc58b 100644
--- a/apps/api/src/services/autumn/usage.ts
+++ b/apps/api/src/services/autumn/usage.ts
@@ -1,5 +1,7 @@
import { logger } from "../../lib/logger";
-import { supabase_rr_service } from "../supabase";
+import { eq, inArray } from "drizzle-orm";
+import { dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { autumnClient } from "./client";
const CREDITS_FEATURE_ID = "CREDITS";
@@ -26,13 +28,12 @@ interface TeamBalance {
// ---------------------------------------------------------------------------
async function lookupOrgId(teamId: string): Promise<string> {
- const { data, error } = await supabase_rr_service
- .from("teams")
- .select("org_id")
- .eq("id", teamId)
- .single();
+ const [data] = await dbRr
+ .select({ org_id: schema.teams.org_id })
+ .from(schema.teams)
+ .where(eq(schema.teams.id, teamId))
+ .limit(1);
- if (error) throw error;
if (!data?.org_id) {
throw new Error(`Missing org_id for team ${teamId}`);
}
@@ -53,15 +54,13 @@ async function lookupApiKeyNames(
const nameMap: Record<string, string> = {};
if (numericIds.length > 0) {
- const { data } = await supabase_rr_service
- .from("api_keys")
- .select("id, name")
- .in("id", numericIds);
-
- if (data) {
- for (const row of data) {
- nameMap[String(row.id)] = row.name;
- }
+ const data = await dbRr
+ .select({ id: schema.api_keys.id, name: schema.api_keys.name })
+ .from(schema.api_keys)
+ .where(inArray(schema.api_keys.id, numericIds));
+
+ for (const row of data) {
+ nameMap[String(row.id)] = row.name ?? String(row.id);
}
}
diff --git a/apps/api/src/services/billing/__tests__/batch_billing.test.ts b/apps/api/src/services/billing/__tests__/batch_billing.test.ts
index ce4fbb35b..3f66d1c26 100644
--- a/apps/api/src/services/billing/__tests__/batch_billing.test.ts
+++ b/apps/api/src/services/billing/__tests__/batch_billing.test.ts
@@ -29,11 +29,9 @@ jest.mock("../../autumn/autumn.service", () => ({
},
}));
-const rpc = jest.fn<(name: string, args: any) => Promise<any>>();
-jest.mock("../../supabase", () => ({
- supabase_service: {
- rpc,
- },
+const billTeam6 = jest.fn<(params: any) => Promise<{ api_key: string }[]>>();
+jest.mock("../../../db/rpc", () => ({
+ billTeam6,
}));
const setCachedACUC = jest.fn();
@@ -126,7 +124,7 @@ beforeEach(() => {
queue = [];
billedTeams.clear();
locks.clear();
- rpc.mockResolvedValue({ data: [], error: null });
+ billTeam6.mockResolvedValue([]);
trackCredits.mockResolvedValue(true);
refundCredits.mockResolvedValue(undefined);
});
@@ -137,7 +135,7 @@ describe("processBillingBatch", () => {
await processBillingBatch();
- expect(rpc).toHaveBeenCalled();
+ expect(billTeam6).toHaveBeenCalled();
expect(trackCredits).toHaveBeenCalledWith({
teamId: "team-1",
value: 10,
@@ -156,13 +154,13 @@ describe("processBillingBatch", () => {
await processBillingBatch();
- expect(rpc).toHaveBeenCalled();
+ expect(billTeam6).toHaveBeenCalled();
expect(trackCredits).not.toHaveBeenCalled();
});
it("continues when billing returns success false", async () => {
queue = [makeOp({ autumnTrackInRequest: true })];
- rpc.mockResolvedValueOnce({ data: null, error: new Error("db failed") });
+ billTeam6.mockRejectedValueOnce(new Error("db failed"));
await processBillingBatch();
@@ -181,7 +179,7 @@ describe("processBillingBatch", () => {
it("captures exceptions when billing throws", async () => {
queue = [makeOp({ autumnTrackInRequest: true })];
- rpc.mockRejectedValueOnce(new Error("rpc exploded"));
+ billTeam6.mockRejectedValueOnce(new Error("rpc exploded"));
await processBillingBatch();
@@ -211,9 +209,9 @@ describe("processBillingBatch", () => {
autumnTrackInRequest: false,
}),
];
- rpc
- .mockResolvedValueOnce({ data: null, error: new Error("db failed") })
- .mockResolvedValueOnce({ data: [], error: null });
+ billTeam6
+ .mockRejectedValueOnce(new Error("db failed"))
+ .mockResolvedValueOnce([]);
refundCredits.mockRejectedValueOnce(new Error("refund failed"));
await processBillingBatch();
@@ -228,7 +226,7 @@ describe("processBillingBatch", () => {
subscriptionId: "sub-1",
},
});
- expect(rpc).toHaveBeenCalledTimes(2);
+ expect(billTeam6).toHaveBeenCalledTimes(2);
expect(trackCredits).toHaveBeenCalledWith({
teamId: "team-2",
value: 10,
diff --git a/apps/api/src/services/billing/__tests__/credit_billing.test.ts b/apps/api/src/services/billing/__tests__/credit_billing.test.ts
index beb3d3606..be65d5efb 100644
--- a/apps/api/src/services/billing/__tests__/credit_billing.test.ts
+++ b/apps/api/src/services/billing/__tests__/credit_billing.test.ts
@@ -22,10 +22,6 @@ jest.mock("../../autumn/autumn.service", () => ({
jest.mock("../../notification/email_notification", () => ({
sendNotification: jest.fn(),
}));
-jest.mock("../../supabase", () => ({
- supabase_rr_service: {},
- supabase_service: {},
-}));
jest.mock("../auto_charge", () => ({
autoCharge: jest.fn(),
}));
diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.ts
deleted file mode 100644
index 35633a254..000000000
--- a/apps/api/src/services/billing/auto_charge.ts
+++ /dev/null
@@ -1,667 +0,0 @@
-// Import necessary dependencies and types
-import { AuthCreditUsageChunk } from "../../controllers/v1/types";
-import { config } from "../../config";
-import { clearACUC, clearACUCTeam, getACUC } from "../../controllers/auth";
-import { redlock } from "../redlock";
-import { supabase_rr_service, supabase_service } from "../supabase";
-import {
- createPaymentIntent,
- createSubscription,
- customerToUserId,
-} from "./stripe";
-import { issueCredits } from "./issue_credits";
-import {
- sendNotification,
- sendNotificationWithCustomDays,
-} from "../notification/email_notification";
-import { NotificationType } from "../../types";
-import { deleteKey, getValue, redisEvictConnection, setValue } from "../redis";
-import { redisRateLimitClient } from "../rate-limiter";
-import { sendSlackWebhook } from "../alerts/slack";
-import { logger as _logger } from "../../lib/logger";
-
-// Define the number of credits to be added during auto-recharge
-const AUTO_RECHARGE_CREDITS = 1000;
-const AUTO_RECHARGE_COOLDOWN = 600; // 10 minutes in seconds
-const MAX_CHARGES_PER_HOUR = 5; // Maximum number of auto-charges per hour
-const HOURLY_COUNTER_EXPIRY = 3600; // 1 hour in seconds
-
-// Type definition for auto-charge response
-type AutoChargeResponse = {
- success: boolean;
- message: string;
- remainingCredits: number;
- chunk: AuthCreditUsageChunk;
-};
-
-/**
- * Attempt to automatically charge a user's account when their credit balance falls below a threshold
- * @param chunk The user's current usage data
- * @param autoRechargeThreshold The credit threshold that triggers auto-recharge
- */
-export async function autoCharge(
- chunk: AuthCreditUsageChunk,
- autoRechargeThreshold: number,
-): Promise<AutoChargeResponse> {
- if (chunk.price_associated_auto_recharge_price_id !== null) {
- return _autoChargeScale(
- chunk as AuthCreditUsageChunk & {
- price_associated_auto_recharge_price_id: string;
- },
- autoRechargeThreshold,
- );
- } else {
- return _autoChargeSelfServe(chunk, autoRechargeThreshold);
- }
-}
-
-async function _autoChargeScale(
- chunk: AuthCreditUsageChunk & {
- price_associated_auto_recharge_price_id: string;
- },
- autoRechargeThreshold: number,
-): Promise<AutoChargeResponse> {
- const logger = _logger.child({
- module: "auto_charge",
- method: "_autoChargeScale",
- team_id: chunk.team_id,
- teamId: chunk.team_id,
- });
-
- logger.info("Scale auto-recharge triggered", {});
-
- const resource = `auto-recharge:${chunk.team_id}`;
-
- try {
- return await redlock.using(
- [resource],
- 15000,
- async (signal: unknown): Promise<AutoChargeResponse> => {
- logger.info("Lock acquired");
-
- const cooldownCheck = await redisEvictConnection.set(
- "auto-recharge-cooldown:" + chunk.team_id,
- "cooling",
- "EX",
- AUTO_RECHARGE_COOLDOWN,
- "NX",
- );
-
- if (cooldownCheck === null) {
- logger.warn("Auto-recharge is on cooldown, aborting.");
- return {
- success: false,
- message: "Auto-recharge is on cooldown",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
-
- const updatedChunk = await getACUC(chunk.api_key, false, false);
-
- if (
- updatedChunk &&
- updatedChunk.remaining_credits < autoRechargeThreshold
- ) {
- const { data: price, error: priceError } = await supabase_service
- .from("prices")
- .select("*")
- .eq("id", chunk.price_associated_auto_recharge_price_id)
- .single();
- if (priceError || !price) {
- logger.error("Error fetching price", {
- error: priceError,
- priceId:
- chunk.price_associated_auto_recharge_price_id === undefined
- ? "undefined"
- : JSON.stringify(
- chunk.price_associated_auto_recharge_price_id,
- ),
- });
- return {
- success: false,
- message: "Error fetching price",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- }
-
- const invoiceOnly = Boolean(
- (price.metadata as { invoice?: boolean } | null | undefined)
- ?.invoice,
- );
-
- // Check for recharges this month
-
- const currentMonth = new Date();
- currentMonth.setUTCDate(1);
- currentMonth.setUTCHours(0, 0, 0, 0);
-
- const { data: rechargesThisMonth, error: rechargesThisMonthError } =
- await supabase_service
- .from("subscriptions")
- .select("*, prices!inner(*)")
- .eq("team_id", chunk.team_id)
- .eq("prices.metadata->>auto_recharge", "true")
- .gte("current_period_start", currentMonth.toISOString());
-
- if (rechargesThisMonthError || !rechargesThisMonth) {
- logger.error("Error fetching recharges this month", {
- error: rechargesThisMonthError,
- });
- return {
- success: false,
- message: "Error fetching recharges this month",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- } else if (
- rechargesThisMonth.length >= (price.exp_pack_max_per_month ?? 4)
- ) {
- logger.warn("Auto-recharge failed: too many recharges this month");
- return {
- success: false,
- message: "Auto-recharge failed: too many recharges this month",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- } else {
- // Actually re-charge
-
- if (!chunk.sub_user_id) {
- logger.error("No sub_user_id found in chunk");
- return {
- success: false,
- message: "No sub_user_id found in chunk",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- }
-
- const { data: customer, error: customersError } =
- await supabase_rr_service
- .from("customers")
- .select("id, stripe_customer_id")
- .eq("id", chunk.sub_user_id)
- .single();
-
- if (customersError || !customer) {
- logger.error("Error fetching customer data", {
- error: customersError,
- });
- return {
- success: false,
- message: "Error fetching customer data",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- }
-
- if (!customer.stripe_customer_id) {
- logger.error("No stripe_customer_id found in customer");
- return {
- success: false,
- message: "No stripe_customer_id found in customer",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- }
-
- if (!chunk.sub_id) {
- logger.error("No sub_id found in chunk");
- return {
- success: false,
- message: "No sub_id found in chunk",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- }
-
- const subscription = await createSubscription(
- chunk.team_id,
- customer.stripe_customer_id,
- chunk.price_associated_auto_recharge_price_id,
- chunk.sub_id,
- invoiceOnly,
- );
- if (!subscription) {
- logger.error("Failed to create subscription");
- return {
- success: false,
- message: "Failed to create subscription",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- }
-
- const userId = await customerToUserId(customer.stripe_customer_id);
- if (!userId) {
- logger.error("Failed to get user id from customer");
- return {
- success: false,
- message: "Failed to get user id from customer",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- }
-
- // Try to insert it into subscriptions ourselves in case webhook is slow
- const { error: subscriptionError } = await supabase_service
- .from("subscriptions")
- .insert({
- id: subscription.id,
- user_id: userId,
- metadata: subscription.metadata,
- status: subscription.status,
- price_id: chunk.price_associated_auto_recharge_price_id,
- quantity: 1,
- cancel_at_period_end: false,
- cancel_at: null,
- canceled_at: null,
- current_period_start: subscription.current_period_start
- ? new Date(
- subscription.current_period_start * 1000,
- ).toISOString()
- : null,
- current_period_end: subscription.current_period_end
- ? new Date(
- subscription.current_period_end * 1000,
- ).toISOString()
- : null,
- created: subscription.created
- ? new Date(subscription.created * 1000).toISOString()
- : null,
- ended_at: null,
- trial_start: null,
- trial_end: null,
- team_id: chunk.team_id,
- is_extract: false,
- });
-
- if (subscriptionError) {
- logger.warn(
- "Failed to add subscription to supabase -- maybe we got sniped by the webhook?",
- { error: subscriptionError },
- );
- }
-
- // Reset ACUC cache to reflect the new credit balance
- await clearACUC(chunk.api_key);
- await clearACUCTeam(chunk.team_id);
-
- try {
- // Check for frequent auto-recharges in the past week
- const weeklyAutoRechargeKey = `auto-recharge-weekly:${chunk.team_id}`;
- const weeklyRecharges = await redisRateLimitClient.incr(
- weeklyAutoRechargeKey,
- );
- // Set expiry for 7 days if not already set
- await redisRateLimitClient.expire(
- weeklyAutoRechargeKey,
- 7 * 24 * 60 * 60,
- );
-
- // If this is the second auto-recharge in a week, send notification
- if (weeklyRecharges >= 2) {
- await sendNotificationWithCustomDays(
- chunk.team_id,
- NotificationType.AUTO_RECHARGE_FREQUENT,
- 7, // Send at most once per week
- false,
- );
- }
- } catch (error) {
- logger.error(
- `Error sending frequent auto-recharge notification`,
- {
- error,
- },
- );
- }
-
- await sendNotification(
- chunk.team_id,
- NotificationType.AUTO_RECHARGE_SUCCESS,
- chunk.sub_current_period_start,
- chunk.sub_current_period_end,
- chunk,
- true,
- false,
- { autoRechargeCredits: price.credits },
- );
-
- logger.info("Scale auto-recharge successful");
-
- if (config.SLACK_ADMIN_WEBHOOK_URL) {
- sendSlackWebhook(
- `💰 Auto-recharge successful on team ${chunk.team_id} for ${price.credits} credits (total auto-recharges this month: ${rechargesThisMonth.length + 1}).`,
- false,
- config.SLACK_ADMIN_WEBHOOK_URL,
- ).catch(error => {
- logger.debug(`Error sending slack notification: ${error}`);
- });
- }
-
- return {
- success: true,
- message: "Auto-recharge successful",
- remainingCredits:
- (updatedChunk?.remaining_credits ?? chunk.remaining_credits) +
- price.credits,
- chunk: {
- ...(updatedChunk ?? chunk),
- remaining_credits:
- (updatedChunk?.remaining_credits ?? chunk.remaining_credits) +
- price.credits,
- },
- };
- }
- } else {
- return {
- success: false,
- message: "No need to auto-recharge",
- remainingCredits:
- updatedChunk?.remaining_credits ?? chunk.remaining_credits,
- chunk: updatedChunk ?? chunk,
- };
- }
- },
- );
- } catch (error) {
- logger.error("Auto-recharge failed", { error });
- return {
- success: false,
- message: "Failed to acquire lock for auto-recharge",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
-}
-
-async function _autoChargeSelfServe(
- chunk: AuthCreditUsageChunk,
- autoRechargeThreshold: number,
-): Promise<AutoChargeResponse> {
- const logger = _logger.child({
- module: "auto_charge",
- method: "_autoChargeSelfServe",
- team_id: chunk.team_id,
- teamId: chunk.team_id,
- });
-
- const resource = `auto-recharge:${chunk.team_id}`;
- const cooldownKey = `auto-recharge-cooldown:${chunk.team_id}`;
- const hourlyCounterKey = `auto-recharge-hourly:${chunk.team_id}`;
-
- if (
- chunk.team_id === "285bb597-6eaf-4b96-801c-51461fc3c543" ||
- chunk.team_id === "dec639a0-98ca-4995-95b5-48ac1ffab5b7"
- ) {
- return {
- success: false,
- message: "Auto-recharge failed: blocked team",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
-
- try {
- // Check hourly rate limit first without incrementing
- const currentCharges = await redisRateLimitClient.get(hourlyCounterKey);
- const hourlyCharges = currentCharges ? parseInt(currentCharges) : 0;
-
- if (hourlyCharges >= MAX_CHARGES_PER_HOUR) {
- logger.warn(
- `Auto-recharge exceeded hourly limit of ${MAX_CHARGES_PER_HOUR}`,
- );
- return {
- success: false,
- message: "Auto-recharge hourly limit exceeded",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
-
- // Check cooldown period
- const cooldownValue = await getValue(cooldownKey);
- if (cooldownValue) {
- logger.info(`Auto-recharge is in cooldown period`);
- return {
- success: false,
- message: "Auto-recharge is in cooldown period",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
-
- // Use a distributed lock to prevent concurrent auto-charge attempts
- return await redlock.using(
- [resource],
- 5000,
- async (signal: unknown): Promise<AutoChargeResponse> => {
- // Recheck all conditions inside the lock to prevent race conditions
- const updatedChunk = await getACUC(chunk.api_key, false, false);
-
- // Recheck cooldown
- const cooldownValue = await getValue(cooldownKey);
- if (cooldownValue) {
- logger.info(`Auto-recharge is in cooldown period`);
- return {
- success: false,
- message: "Auto-recharge is in cooldown period",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
-
- // Recheck hourly limit inside lock
- const currentCharges = await redisRateLimitClient.get(hourlyCounterKey);
- const hourlyCharges = currentCharges ? parseInt(currentCharges) : 0;
- if (hourlyCharges >= MAX_CHARGES_PER_HOUR) {
- return {
- success: false,
- message: "Auto-recharge hourly limit exceeded",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
-
- if (
- updatedChunk &&
- updatedChunk.remaining_credits < autoRechargeThreshold
- ) {
- if (chunk.sub_user_id) {
- // Fetch the customer's Stripe information
- const { data: customer, error: customersError } =
- await supabase_rr_service
- .from("customers")
- .select("id, stripe_customer_id")
- .eq("id", chunk.sub_user_id)
- .single();
-
- if (customersError) {
- logger.error(`Error fetching customer data`, {
- error: customersError,
- });
- return {
- success: false,
- message: "Error fetching customer data",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
-
- if (customer && customer.stripe_customer_id) {
- let issueCreditsSuccess = false;
-
- // Set cooldown BEFORE attempting payment
- await setValue(cooldownKey, "true", AUTO_RECHARGE_COOLDOWN);
-
- // Attempt to create a payment intent
- const paymentStatus = await createPaymentIntent(
- chunk.team_id,
- customer.stripe_customer_id,
- );
-
- // If payment is successful or requires further action, issue credits
- if (
- paymentStatus.return_status === "succeeded" ||
- paymentStatus.return_status === "requires_action"
- ) {
- issueCreditsSuccess = await issueCredits(
- chunk.team_id,
- AUTO_RECHARGE_CREDITS,
- );
- }
-
- // Record the auto-recharge transaction
- await supabase_service.from("auto_recharge_transactions").insert({
- team_id: chunk.team_id,
- initial_payment_status: paymentStatus.return_status,
- credits_issued: issueCreditsSuccess ? AUTO_RECHARGE_CREDITS : 0,
- stripe_charge_id: paymentStatus.charge_id,
- });
-
- // Send a notification if credits were successfully issued
- if (issueCreditsSuccess) {
- // Increment hourly counter and set expiry if it doesn't exist
- await redisRateLimitClient.incr(hourlyCounterKey);
- await redisRateLimitClient.expire(
- hourlyCounterKey,
- HOURLY_COUNTER_EXPIRY,
- "NX",
- );
-
- try {
- // Check for frequent auto-recharges in the past week
- const weeklyAutoRechargeKey = `auto-recharge-weekly:${chunk.team_id}`;
- const weeklyRecharges = await redisRateLimitClient.incr(
- weeklyAutoRechargeKey,
- );
- // Set expiry for 7 days if not already set
- await redisRateLimitClient.expire(
- weeklyAutoRechargeKey,
- 7 * 24 * 60 * 60,
- );
-
- // If this is the second auto-recharge in a week, send notification
- if (weeklyRecharges >= 2) {
- await sendNotificationWithCustomDays(
- chunk.team_id,
- NotificationType.AUTO_RECHARGE_FREQUENT,
- 7, // Send at most once per week
- false,
- );
- }
- } catch (error) {
- logger.error(
- `Error sending frequent auto-recharge notification`,
- { error },
- );
- }
-
- await sendNotification(
- chunk.team_id,
- NotificationType.AUTO_RECHARGE_SUCCESS,
- chunk.sub_current_period_start,
- chunk.sub_current_period_end,
- chunk,
- true,
- false,
- { autoRechargeCredits: AUTO_RECHARGE_CREDITS },
- );
-
- // Reset ACUC cache to reflect the new credit balance
- await clearACUC(chunk.api_key);
- await clearACUCTeam(chunk.team_id);
-
- logger.info(`Auto-recharge successful`, {
- credits: AUTO_RECHARGE_CREDITS,
- paymentStatus: paymentStatus.return_status,
- });
-
- if (config.SLACK_ADMIN_WEBHOOK_URL) {
- const webhookCooldownKey = `webhook_cooldown_${chunk.team_id}`;
- const isInCooldown = await getValue(webhookCooldownKey);
-
- if (!isInCooldown) {
- sendSlackWebhook(
- `Auto-recharge: Team ${chunk.team_id}. ${AUTO_RECHARGE_CREDITS} credits added. Payment status: ${paymentStatus.return_status}.`,
- false,
- config.SLACK_ADMIN_WEBHOOK_URL,
- ).catch(error => {
- logger.debug(
- `Error sending slack notification: ${error}`,
- );
- });
-
- // Set cooldown for 1 hour
- await setValue(webhookCooldownKey, "true", 60 * 60);
- }
- }
- return {
- success: true,
- message: "Auto-recharge successful",
- remainingCredits:
- chunk.remaining_credits + AUTO_RECHARGE_CREDITS,
- chunk: {
- ...chunk,
- remaining_credits:
- chunk.remaining_credits + AUTO_RECHARGE_CREDITS,
- },
- };
- } else {
- logger.error("No Stripe customer ID found for user");
- return {
- success: false,
- message: "No Stripe customer ID found for user",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
- } else {
- logger.error("No Stripe customer ID found for user");
- return {
- success: false,
- message: "No Stripe customer ID found for user",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
- } else {
- logger.error("No sub_user_id found in chunk");
- return {
- success: false,
- message: "No sub_user_id found in chunk",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
- }
- return {
- success: false,
- message: "No need to auto-recharge",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- },
- );
- } catch (error) {
- logger.error(`Failed to acquire lock for auto-recharge`, { error });
- return {
- success: false,
- message: "Failed to acquire lock for auto-recharge",
- remainingCredits: chunk.remaining_credits,
- chunk,
- };
- }
-}
diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.ts
index 00005c2bd..738ce56f6 100644
--- a/apps/api/src/services/billing/batch_billing.ts
+++ b/apps/api/src/services/billing/batch_billing.ts
@@ -1,7 +1,7 @@
import { logger } from "../../lib/logger";
import { config } from "../../config";
import { getRedisConnection } from "../queue-service";
-import { supabase_service } from "../supabase";
+import { billTeam6 } from "../../db/rpc";
import * as Sentry from "@sentry/node";
import { withAuth } from "../../lib/withAuth";
import { setCachedACUC, setCachedACUCTeam } from "../../controllers/auth";
@@ -223,7 +223,6 @@ export async function processBillingBatch() {
},
});
}
-
} catch (error) {
await refundRequestTrackedCredits(group);
logger.error(`❌ Failed to bill team ${group.team_id}`, {
@@ -402,16 +401,17 @@ async function supaBillTeam(
_logger.info(`Batch billing team ${team_id} for ${credits} credits`);
// Perform the actual database operation
- const { data, error } = await supabase_service.rpc("bill_team_6", {
- _team_id: team_id,
- sub_id: subscription_id ?? null,
- fetch_subscription: subscription_id === undefined,
- credits,
- i_api_key_id: api_key_id ?? null,
- is_extract_param: is_extract,
- });
-
- if (error) {
+ let data: { api_key: string }[];
+ try {
+ data = await billTeam6({
+ team_id,
+ subscription_id: subscription_id ?? null,
+ fetch_subscription: subscription_id === undefined,
+ credits,
+ api_key_id: api_key_id ?? null,
+ is_extract,
+ });
+ } catch (error) {
Sentry.captureException(error);
_logger.error("Failed to bill team.", { error });
return { success: false, error };
diff --git a/apps/api/src/services/billing/issue_credits.ts b/apps/api/src/services/billing/issue_credits.ts
deleted file mode 100644
index 2ca057dda..000000000
--- a/apps/api/src/services/billing/issue_credits.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import { logger } from "../../lib/logger";
-import { supabase_service } from "../supabase";
-
-export async function issueCredits(team_id: string, credits: number) {
- // Add an entry to supabase coupons
- const { error } = await supabase_service.from("coupons").insert({
- team_id: team_id,
- credits: credits,
- status: "active",
- // indicates that this coupon was issued from auto recharge
- from_auto_recharge: true,
- initial_credits: credits,
- });
-
- if (error) {
- logger.error(`Error adding coupon: ${error}`);
- return false;
- }
-
- return true;
-}
diff --git a/apps/api/src/services/billing/stripe.ts b/apps/api/src/services/billing/stripe.ts
deleted file mode 100644
index d74838f29..000000000
--- a/apps/api/src/services/billing/stripe.ts
+++ /dev/null
@@ -1,190 +0,0 @@
-import { logger } from "../../lib/logger";
-import { config } from "../../config";
-import Stripe from "stripe";
-
-const stripe = new Stripe(config.STRIPE_SECRET_KEY ?? "");
-
-async function getCustomerDefaultPaymentMethod(customerId: string) {
- const paymentMethods = await stripe.customers.listPaymentMethods(customerId, {
- limit: 3,
- });
- return paymentMethods.data[0] ?? null;
-}
-
-type ReturnStatus = "succeeded" | "requires_action" | "failed";
-export async function createPaymentIntent(
- team_id: string,
- customer_id: string,
-): Promise<{ return_status: ReturnStatus; charge_id: string }> {
- try {
- const defaultPaymentMethod =
- await getCustomerDefaultPaymentMethod(customer_id);
- if (!defaultPaymentMethod) {
- logger.error(
- `No default payment method found for customer: ${customer_id}`,
- { team_id },
- );
- return { return_status: "failed", charge_id: "" };
- }
- const paymentIntent = await stripe.paymentIntents.create({
- amount: 1100,
- currency: "usd",
- customer: customer_id,
- description: "Firecrawl: Auto re-charge of 1000 credits",
- payment_method_types: [defaultPaymentMethod?.type ?? "card"],
- payment_method: defaultPaymentMethod?.id,
- off_session: true,
- confirm: true,
- metadata: {
- team_id,
- auto_recharge: "true",
- },
- });
-
- if (paymentIntent.status === "succeeded") {
- logger.info(`Payment succeeded for team: ${team_id}`);
- return { return_status: "succeeded", charge_id: paymentIntent.id };
- } else if (
- paymentIntent.status === "requires_action" ||
- paymentIntent.status === "processing" ||
- paymentIntent.status === "requires_capture"
- ) {
- logger.warn(`Payment requires further action for team: ${team_id}`);
- return { return_status: "requires_action", charge_id: paymentIntent.id };
- } else {
- logger.error(`Payment failed for team: ${team_id}`);
- return { return_status: "failed", charge_id: paymentIntent.id };
- }
- } catch (error) {
- logger.error(
- `Failed to create or confirm PaymentIntent for team: ${team_id}`,
- );
- console.error(error);
- return { return_status: "failed", charge_id: "" };
- }
-}
-
-export async function createSubscription(
- team_id: string,
- customer_id: string,
- price_id: string,
- main_subscription_id: string,
- invoiceOnly = false,
-) {
- const defaultPaymentMethod = invoiceOnly
- ? null
- : await getCustomerDefaultPaymentMethod(customer_id);
- if (!invoiceOnly && !defaultPaymentMethod) {
- logger.error("No default payment method found for customer", {
- team_id,
- customer_id,
- });
- return null;
- }
-
- let useCoupons: string[] = [];
-
- const mainSub = await stripe.subscriptions.retrieve(main_subscription_id, {
- expand: ["discounts"],
- });
- if (mainSub) {
- // Clone coupons from main subscription to auto recharge expansion
- useCoupons = (
- await Promise.all(
- ((mainSub.discounts ?? []) as Stripe.Discount[]).map(async discount => {
- if (discount.coupon.duration === "once") {
- return null;
- }
-
- if (
- discount.coupon.duration === "repeating" &&
- new Date(discount.end! * 1000) < new Date()
- ) {
- return null;
- }
-
- if (!discount.coupon.valid) {
- const duration_in_months =
- discount.coupon.duration === "repeating"
- ? Math.round(
- (new Date(discount.end! * 1000).valueOf() -
- new Date(discount.start! * 1000).valueOf()) /
- (1000 * 60 * 60 * 24 * 30),
- )
- : 0;
- const coupon = await stripe.coupons.create({
- name: discount.coupon.name ?? undefined,
- duration: duration_in_months > 0 ? "repeating" : "once",
- ...(duration_in_months > 0 ? { duration_in_months } : {}),
- max_redemptions: 1,
- ...(discount.coupon.amount_off
- ? { amount_off: discount.coupon.amount_off }
- : {}),
- ...(discount.coupon.currency
- ? { currency: discount.coupon.currency }
- : {}),
- ...(discount.coupon.currency_options
- ? { currency_options: discount.coupon.currency_options }
- : {}),
- metadata: {
- ...(discount.coupon.metadata ?? {}),
- team_id,
- auto_recharge: "true",
- },
- ...(discount.coupon.percent_off
- ? { percent_off: discount.coupon.percent_off }
- : {}),
- ...(discount.coupon.applies_to
- ? { applies_to: discount.coupon.applies_to }
- : {}),
- });
- return coupon.id;
- } else {
- return discount.coupon.id;
- }
- }),
- )
- ).filter(x => x !== null);
- } else {
- logger.warn("No main subscription found for customer", {
- team_id,
- customer_id,
- });
- }
-
- const subscription = await stripe.subscriptions.create({
- customer: customer_id,
- items: [{ price: price_id }],
- ...(invoiceOnly
- ? {
- collection_method: "send_invoice",
- days_until_due: 30,
- }
- : {
- off_session: true,
- payment_settings: {
- payment_method_types: [
- (defaultPaymentMethod?.type as Stripe.SubscriptionCreateParams.PaymentSettings.PaymentMethodType) ??
- "card",
- ],
- },
- default_payment_method: defaultPaymentMethod?.id,
- collection_method: "charge_automatically",
- }),
- metadata: {
- team_id,
- auto_recharge: "true",
- },
- ...(useCoupons.length > 0
- ? { discounts: useCoupons.map(coupon => ({ coupon })) }
- : {}),
- });
-
- return subscription;
-}
-
-export async function customerToUserId(customerId: string) {
- const customer = await stripe.customers.retrieve(customerId);
- if (!customer || customer.deleted) return null;
- return customer.metadata.supabaseUUID ?? null;
-}
diff --git a/apps/api/src/services/idempotency/create.ts b/apps/api/src/services/idempotency/create.ts
index 8e1ede445..21b03c361 100644
--- a/apps/api/src/services/idempotency/create.ts
+++ b/apps/api/src/services/idempotency/create.ts
@@ -1,5 +1,6 @@
import { Request } from "express";
-import { supabase_service } from "../supabase";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
import { logger } from "../../../src/lib/logger";
export async function createIdempotencyKey(req: Request): Promise<string> {
@@ -8,11 +9,9 @@ export async function createIdempotencyKey(req: Request): Promise<string> {
throw new Error("No idempotency key provided in the request headers.");
}
- const { data, error } = await supabase_service
- .from("idempotency_keys")
- .insert({ key: idempotencyKey });
-
- if (error) {
+ try {
+ await db.insert(schema.idempotency_keys).values({ key: idempotencyKey });
+ } catch (error) {
logger.error(`Failed to create idempotency key: ${error}`);
throw error;
}
diff --git a/apps/api/src/services/idempotency/validate.ts b/apps/api/src/services/idempotency/validate.ts
index 54ec7bd04..94749d2f4 100644
--- a/apps/api/src/services/idempotency/validate.ts
+++ b/apps/api/src/services/idempotency/validate.ts
@@ -1,5 +1,7 @@
import { Request } from "express";
-import { supabase_rr_service, supabase_service } from "../supabase";
+import { eq } from "drizzle-orm";
+import { dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { validate as isUuid } from "uuid";
import { logger } from "../../../src/lib/logger";
@@ -18,12 +20,13 @@ export async function validateIdempotencyKey(req: Request): Promise<boolean> {
return false;
}
- const { data, error } = await supabase_rr_service
- .from("idempotency_keys")
- .select("key")
- .eq("key", idempotencyKey);
-
- if (error) {
+ let data: { key: string }[] = [];
+ try {
+ data = await dbRr
+ .select({ key: schema.idempotency_keys.key })
+ .from(schema.idempotency_keys)
+ .where(eq(schema.idempotency_keys.key, key));
+ } catch (error) {
logger.error(`Error validating idempotency key: ${error}`);
}
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 2711afbc4..dbdab04c3 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -1,5 +1,16 @@
-import { createClient, SupabaseClient } from "@supabase/supabase-js";
import { logger as _logger } from "../lib/logger";
+import { dbIndex } from "../db/connection";
+import * as schema from "../db/schema";
+import {
+ insertOmceJobIfNeeded,
+ queryIndexAtSplitLevel as rpcQueryIndexAtSplitLevel,
+ queryIndexAtDomainSplitLevel as rpcQueryIndexAtDomainSplitLevel,
+ queryOmceSignatures as rpcQueryOmceSignatures,
+ queryEngpickerVerdict as rpcQueryEngpickerVerdict,
+ queryIndexAtSplitLevelWithMeta as rpcQueryIndexAtSplitLevelWithMeta,
+ queryIndexAtDomainSplitLevelWithMeta as rpcQueryIndexAtDomainSplitLevelWithMeta,
+ queryDomainPriority as rpcQueryDomainPriority,
+} from "../db/rpc";
import { configDotenv } from "dotenv";
import { ApiError } from "@google-cloud/storage";
import crypto from "crypto";
@@ -13,51 +24,6 @@ import { withSpan, setSpanAttributes } from "../lib/otel-tracer";
import { config } from "../config";
configDotenv();
-// SupabaseService class initializes the Supabase client conditionally based on environment variables.
-class IndexSupabaseService {
- private client: SupabaseClient | null = null;
-
- constructor() {
- const supabaseUrl = config.INDEX_SUPABASE_URL;
- const supabaseServiceToken = config.INDEX_SUPABASE_SERVICE_TOKEN;
- // Only initialize the Supabase client if both URL and Service Token are provided.
- if (!supabaseUrl || !supabaseServiceToken) {
- // Warn the user that Authentication is disabled by setting the client to null
- _logger.warn("Index supabase client will not be initialized.");
- this.client = null;
- } else {
- this.client = createClient(supabaseUrl, supabaseServiceToken);
- }
- }
-
- // Provides access to the initialized Supabase client, if available.
- getClient(): SupabaseClient | null {
- return this.client;
- }
-}
-
-const serv = new IndexSupabaseService();
-
-// Using a Proxy to handle dynamic access to the Supabase client or service methods.
-// This approach ensures that if Supabase is not configured, any attempt to use it will result in a clear error.
-export const index_supabase_service: SupabaseClient = new Proxy(serv, {
- get: function (target, prop, receiver) {
- const client = target.getClient();
- // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
- if (client === null) {
- return () => {
- throw new Error("Index supabase client is not configured.");
- };
- }
- // Direct access to SupabaseService properties takes precedence.
- if (prop in target) {
- return Reflect.get(target, prop, receiver);
- }
- // Otherwise, delegate access to the Supabase client.
- return Reflect.get(client, prop, receiver);
- },
-}) as unknown as SupabaseClient;
-
export async function getIndexFromGCS(
url: string,
logger?: Logger,
@@ -208,11 +174,10 @@ export async function saveIndexToGCS(
}
export const useIndex =
- config.INDEX_SUPABASE_URL !== "" && config.INDEX_SUPABASE_URL !== undefined;
+ config.INDEX_DATABASE_URL !== "" && config.INDEX_DATABASE_URL !== undefined;
export const useSearchIndex =
- config.SEARCH_INDEX_SUPABASE_URL !== "" &&
- config.SEARCH_INDEX_SUPABASE_URL !== undefined;
+ config.SEARCH_SERVICE_URL !== "" && config.SEARCH_SERVICE_URL !== undefined;
export function normalizeURLForIndex(url: string): string {
const urlObj = new URL(url);
@@ -254,8 +219,8 @@ export function normalizeURLForIndex(url: string): string {
return urlObj.toString();
}
-export function hashURL(url: string): string {
- return "\\x" + crypto.createHash("sha256").update(url).digest("hex");
+export function hashURL(url: string): Buffer {
+ return crypto.createHash("sha256").update(url).digest();
}
export function generateURLSplits(url: string): string[] {
@@ -331,13 +296,34 @@ export async function addIndexInsertJob(data: any) {
);
}
+function reviveBuffers(_key: string, value: any) {
+ return value?.type === "Buffer" && Array.isArray(value.data)
+ ? Buffer.from(value.data)
+ : value;
+}
+
+function safeParseJob<T>(raw: string, queueKey: string): T | undefined {
+ try {
+ return JSON.parse(raw, reviveBuffers) as T;
+ } catch (error) {
+ _logger.error(`Failed to parse queued job, skipping`, {
+ error,
+ queueKey,
+ raw,
+ });
+ return undefined;
+ }
+}
+
async function getIndexInsertJobs(): Promise<any[]> {
const jobs =
(await redisEvictConnection.lpop(
INDEX_INSERT_QUEUE_KEY,
INDEX_INSERT_BATCH_SIZE,
)) ?? [];
- return jobs.map(x => JSON.parse(x));
+ return jobs
+ .map(x => safeParseJob<any>(x, INDEX_INSERT_QUEUE_KEY))
+ .filter(x => x !== undefined);
}
export async function processIndexInsertJobs() {
@@ -349,13 +335,7 @@ export async function processIndexInsertJobs() {
jobCount: jobs.length,
});
try {
- const { error } = await index_supabase_service.from("index").insert(jobs);
- if (error) {
- _logger.error(`Index inserter failed to insert jobs`, {
- error,
- jobCount: jobs.length,
- });
- }
+ await dbIndex.insert(schema.index).values(jobs);
_logger.info(`Index inserter inserted jobs`, { jobCount: jobs.length });
} catch (error) {
_logger.error(`Index inserter failed to insert jobs`, {
@@ -372,17 +352,19 @@ export async function getIndexInsertQueueLength(): Promise<number> {
const OMCE_JOB_QUEUE_KEY = "omce-job-queue";
const OMCE_JOB_QUEUE_BATCH_SIZE = 100;
-export async function addOMCEJob(data: [number, string]) {
+export async function addOMCEJob(data: [number, Buffer]) {
await redisEvictConnection.sadd(OMCE_JOB_QUEUE_KEY, JSON.stringify(data));
}
-async function getOMCEJobs(): Promise<[number, string][]> {
+async function getOMCEJobs(): Promise<[number, Buffer][]> {
const jobs =
(await redisEvictConnection.spop(
OMCE_JOB_QUEUE_KEY,
OMCE_JOB_QUEUE_BATCH_SIZE,
)) ?? [];
- return jobs.map(x => JSON.parse(x) as [number, string]);
+ return jobs
+ .map(x => safeParseJob<[number, Buffer]>(x, OMCE_JOB_QUEUE_KEY))
+ .filter((x): x is [number, Buffer] => x !== undefined);
}
export async function processOMCEJobs() {
@@ -396,15 +378,9 @@ export async function processOMCEJobs() {
try {
for (const job of jobs) {
const [level, hash] = job;
- const { error } = await index_supabase_service.rpc(
- "insert_omce_job_if_needed",
- {
- i_domain_level: level,
- i_domain_hash: hash,
- },
- );
-
- if (error) {
+ try {
+ await insertOmceJobIfNeeded(level, hash);
+ } catch (error) {
_logger.error(`OMCE job inserter failed to insert job`, {
error,
job,
@@ -441,40 +417,20 @@ export async function queryIndexAtSplitLevel(
const level = urlSplitsHash.length - 1;
- let links: Set<string> = new Set();
- let iteration = 0;
-
- while (true) {
- // Query the index for the next set of links
- const { data: _data, error } = await index_supabase_service
- .rpc("query_index_at_split_level", {
- i_level: level,
- i_url_hash: urlSplitsHash[level],
- i_newer_than: new Date(Date.now() - maxAge).toISOString(),
- })
- .range(iteration * 1000, (iteration + 1) * 1000);
-
- // If there's an error, return the links we have
- if (error) {
- _logger.warn("Error querying index", { error, url, limit });
- return [...links].slice(0, limit);
- }
-
- // Add the links to the set
- const data = _data ?? [];
+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the
+ // previous .range() pagination loop is no longer needed.
+ try {
+ const data = await rpcQueryIndexAtSplitLevel(
+ level,
+ urlSplitsHash[level],
+ new Date(Date.now() - maxAge).toISOString(),
+ );
+ const links = new Set<string>();
data.forEach(x => links.add(x.resolved_url));
-
- // If we have enough links, return them
- if (links.size >= limit) {
- return [...links].slice(0, limit);
- }
-
- // If we get less than 1000 links from the query, we're done
- if (data.length < 1000) {
- return [...links].slice(0, limit);
- }
-
- iteration++;
+ return [...links].slice(0, limit);
+ } catch (error) {
+ _logger.warn("Error querying index", { error, url, limit });
+ return [];
}
}
@@ -494,40 +450,20 @@ export async function queryIndexAtDomainSplitLevel(
return [];
}
- let links: Set<string> = new Set();
- let iteration = 0;
-
- while (true) {
- // Query the index for the next set of links
- const { data: _data, error } = await index_supabase_service
- .rpc("query_index_at_domain_split_level", {
- i_level: level,
- i_domain_hash: domainSplitsHash[level],
- i_newer_than: new Date(Date.now() - maxAge).toISOString(),
- })
- .range(iteration * 1000, (iteration + 1) * 1000);
-
- // If there's an error, return the links we have
- if (error) {
- _logger.warn("Error querying index", { error, hostname, limit });
- return [...links].slice(0, limit);
- }
-
- // Add the links to the set
- const data = _data ?? [];
+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the
+ // previous .range() pagination loop is no longer needed.
+ try {
+ const data = await rpcQueryIndexAtDomainSplitLevel(
+ level,
+ domainSplitsHash[level],
+ new Date(Date.now() - maxAge).toISOString(),
+ );
+ const links = new Set<string>();
data.forEach(x => links.add(x.resolved_url));
-
- // If we have enough links, return them
- if (links.size >= limit) {
- return [...links].slice(0, limit);
- }
-
- // If we get less than 1000 links from the query, we're done
- if (data.length < 1000) {
- return [...links].slice(0, limit);
- }
-
- iteration++;
+ return [...links].slice(0, limit);
+ } catch (error) {
+ _logger.warn("Error querying index", { error, hostname, limit });
+ return [];
}
}
@@ -546,20 +482,16 @@ export async function queryOMCESignatures(
return [];
}
- const { data, error } = await index_supabase_service.rpc(
- "query_omce_signatures",
- {
- i_domain_hash: domainSplitsHash[level],
- i_newer_than: new Date(Date.now() - maxAge).toISOString(),
- },
- );
-
- if (error) {
+ try {
+ const data = await rpcQueryOmceSignatures(
+ domainSplitsHash[level],
+ new Date(Date.now() - maxAge).toISOString(),
+ );
+ return data?.[0]?.signatures ?? [];
+ } catch (error) {
_logger.warn("Error querying index (omce)", { error, hostname });
return [];
}
-
- return data?.[0]?.signatures ?? [];
}
export async function queryEngpickerVerdict(
@@ -577,37 +509,26 @@ export async function queryEngpickerVerdict(
}
// 250ms max time taken
-
- const res: { data: any; error: any } = await Promise.any([
- index_supabase_service.rpc("query_engpicker_verdict", {
- i_domain_hash: domainSplitsHash[level],
- }),
- new Promise<{
- data: {
- verdict: "TlsClientOk" | "ChromeCdpRequired" | "Uncertain" | "Unknown";
- }[];
- error: any;
- }>(resolve =>
- setTimeout(
- () =>
- resolve({
- data: [{ verdict: "Unknown" }],
- error: "Took longer than 250ms",
- }),
- 250,
+ try {
+ const data = await Promise.race([
+ rpcQueryEngpickerVerdict(domainSplitsHash[level]),
+ new Promise<{ verdict: string }[]>(resolve =>
+ setTimeout(() => resolve([{ verdict: "Unknown" }]), 250),
),
- ),
- ]);
+ ]);
- if (res.error) {
+ return (data?.[0]?.verdict ?? "Unknown") as
+ | "TlsClientOk"
+ | "ChromeCdpRequired"
+ | "Uncertain"
+ | "Unknown";
+ } catch (error) {
_logger.warn("Error querying index (engpicker)", {
- error: res.error,
+ error,
hostname,
});
return "Unknown";
}
-
- return res.data?.[0]?.verdict ?? "Unknown";
}
export async function queryIndexAtSplitLevelWithMeta(
@@ -625,48 +546,23 @@ export async function queryIndexAtSplitLevelWithMeta(
const level = urlSplitsHash.length - 1;
- let links: MapDocument[] = [];
- let iteration = 0;
-
- while (true) {
- // Query the index for the next set of links
- const { data: _data, error } = await index_supabase_service
- .rpc("query_index_at_split_level_with_meta", {
- i_level: level,
- i_url_hash: urlSplitsHash[level],
- i_newer_than: new Date(
- Date.now() - 2 * 24 * 60 * 60 * 1000,
- ).toISOString(),
- })
- .range(iteration * 1000, (iteration + 1) * 1000);
-
- // If there's an error, return the links we have
- if (error) {
- _logger.warn("Error querying index", { error, url, limit });
- return links.slice(0, limit);
- }
-
- // Add the links to the set
- const data = _data ?? [];
- data.forEach(x =>
- links.push({
- url: x.resolved_url,
- title: x.title ?? undefined,
- description: x.description ?? undefined,
- }),
+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the
+ // previous .range() pagination loop is no longer needed.
+ try {
+ const data = await rpcQueryIndexAtSplitLevelWithMeta(
+ level,
+ urlSplitsHash[level],
+ new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
);
-
- // If we have enough links, return them
- if (links.length >= limit) {
- return links.slice(0, limit);
- }
-
- // If we get less than 1000 links from the query, we're done
- if (data.length < 1000) {
- return links.slice(0, limit);
- }
-
- iteration++;
+ const links: MapDocument[] = data.map(x => ({
+ url: x.resolved_url,
+ title: x.title ?? undefined,
+ description: x.description ?? undefined,
+ }));
+ return links.slice(0, limit);
+ } catch (error) {
+ _logger.warn("Error querying index", { error, url, limit });
+ return [];
}
}
@@ -685,53 +581,28 @@ export async function queryIndexAtDomainSplitLevelWithMeta(
return [];
}
- let links: MapDocument[] = [];
- let iteration = 0;
-
- while (true) {
- // Query the index for the next set of links
- const { data: _data, error } = await index_supabase_service
- .rpc("query_index_at_domain_split_level_with_meta", {
- i_level: level,
- i_domain_hash: domainSplitsHash[level],
- i_newer_than: new Date(
- Date.now() - 2 * 24 * 60 * 60 * 1000,
- ).toISOString(),
- })
- .range(iteration * 1000, (iteration + 1) * 1000);
-
- // If there's an error, return the links we have
- if (error) {
- _logger.warn("Error querying index", { error, hostname, limit });
- return links.slice(0, limit);
- }
-
- // Add the links to the set
- const data = _data ?? [];
- data.forEach(x =>
- links.push({
- url: x.resolved_url,
- title: x.title ?? undefined,
- description: x.description ?? undefined,
- }),
+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the
+ // previous .range() pagination loop is no longer needed.
+ try {
+ const data = await rpcQueryIndexAtDomainSplitLevelWithMeta(
+ level,
+ domainSplitsHash[level],
+ new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
);
-
- // If we have enough links, return them
- if (links.length >= limit) {
- return links.slice(0, limit);
- }
-
- // If we get less than 1000 links from the query, we're done
- if (data.length < 1000) {
- return links.slice(0, limit);
- }
-
- iteration++;
+ const links: MapDocument[] = data.map(x => ({
+ url: x.resolved_url,
+ title: x.title ?? undefined,
+ description: x.description ?? undefined,
+ }));
+ return links.slice(0, limit);
+ } catch (error) {
+ _logger.warn("Error querying index", { error, hostname, limit });
+ return [];
}
}
type DomainPriority = {
- domain_hash: string;
+ domain_hash: Buffer;
priority: number;
};
@@ -746,40 +617,18 @@ export async function queryDomainsForPrecrawl(
return [];
}
- let results: DomainPriority[] = [];
- let iteration = 0;
-
- while (true) {
- const { data, error } = await index_supabase_service
- .rpc("query_domain_priority", {
- p_min_total: minEvents,
- p_min_priority: minPriority,
- p_lim: maxDomains,
- p_time: date.toISOString(),
- })
- .range(
- iteration * 1000,
- Math.min((iteration + 1) * 1000, maxDomains) - 1,
- );
-
- if (error) {
- logger.error("Error getting domain priorities", {
- error,
- });
- return results.slice(0, maxDomains);
- }
-
- const batchData = data ?? [];
- results = results.concat(batchData);
-
- if (results.length >= maxDomains) {
- return results.slice(0, maxDomains);
- }
-
- if (batchData.length < 1000) {
- return results.slice(0, maxDomains);
- }
-
- iteration++;
+ // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the
+ // previous .range() pagination loop is no longer needed.
+ try {
+ const data = await rpcQueryDomainPriority(
+ minEvents,
+ minPriority,
+ maxDomains,
+ date.toISOString(),
+ );
+ return data.slice(0, maxDomains);
+ } catch (error) {
+ logger.error("Error getting domain priorities", { error });
+ return [];
}
}
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 52dcd4e2c..3072069d7 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -20,11 +20,11 @@ import { resolveBillingMetadata } from "../billing/types";
import systemMonitor from "../system-monitor";
import { v7 as uuidv7 } from "uuid";
import {
- index_supabase_service,
processIndexInsertJobs,
processOMCEJobs,
queryDomainsForPrecrawl,
} from "..";
+import { queryTopUrlsForDomain, updateTallyTeam } from "../../db/rpc";
import { getSearchIndexClient } from "../../lib/search-index-client";
// Search indexing is now handled by the separate search service
// import { processSearchIndexJobs } from "../../lib/search-index/queue";
@@ -40,7 +40,6 @@ import { _addScrapeJobToBullMQ } from "../queue-jobs";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
import { crawlGroup } from "../worker/nuq";
import { getACUCTeam } from "../../controllers/auth";
-import { supabase_service } from "../supabase";
import { processEngpickerJob } from "../../lib/engpicker";
import { logRequest } from "../logging/log_job";
@@ -90,8 +89,7 @@ const processBillingJobInternal = async (token: string, job: Job) => {
is_extract,
api_key_id,
autumnTrackInRequest,
- } =
- job.data;
+ } = job.data;
logger.info(`Adding team ${team_id} billing operation to batch queue`, {
credits,
@@ -270,7 +268,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {
type DomainUrlResult = {
url: string;
- domain_hash: string;
+ domain_hash: Buffer;
event_count: number;
rank: number;
};
@@ -284,13 +282,16 @@ const processPrecrawlJob = async (token: string, job: Job) => {
const batch = batches[i];
const batchFutures = batch.map(({ hash, urlsToFetch }) => {
- return index_supabase_service
- .rpc("query_top_urls_for_domain", {
- p_domain_hash: hash,
- p_time_window: "8 days", // increasing window can significantly slow down the query, modify with caution
- p_top_n: urlsToFetch,
- })
- .overrideTypes<DomainUrlResult[]>();
+ return queryTopUrlsForDomain<DomainUrlResult>(
+ hash,
+ "8 days", // increasing window can significantly slow down the query, modify with caution
+ urlsToFetch,
+ )
+ .then(data => ({ data, error: null }))
+ .catch(error => ({
+ data: null as DomainUrlResult[] | null,
+ error,
+ }));
});
const startTimeNs = process.hrtime.bigint();
@@ -352,10 +353,11 @@ const processPrecrawlJob = async (token: string, job: Job) => {
const bucketedByDomain: Map<string, DomainUrlResult[]> = new Map();
for (const item of urls) {
- if (!bucketedByDomain.has(item.domain_hash)) {
- bucketedByDomain.set(item.domain_hash, []);
+ const key = item.domain_hash.toString("hex");
+ if (!bucketedByDomain.has(key)) {
+ bucketedByDomain.set(key, []);
}
- bucketedByDomain.get(item.domain_hash)!.push(item);
+ bucketedByDomain.get(key)!.push(item);
}
const crawlTargets: Map<
@@ -372,7 +374,9 @@ const processPrecrawlJob = async (token: string, job: Job) => {
for (const domain of domains) {
try {
- const pages = bucketedByDomain.get(domain.domain_hash);
+ const pages = bucketedByDomain.get(
+ domain.domain_hash.toString("hex"),
+ );
// if this doesn't have any pages, do we want to locate the domain itself and add root only?
if (!pages || pages.length === 0) {
@@ -682,14 +686,11 @@ async function tallyBilling() {
for (const teamId of billedTeams) {
logger.info("Updating tally for team", { teamId });
- const { error } = await supabase_service.rpc("update_tally_10_team", {
- i_team_id: teamId,
- });
-
- if (error) {
- logger.warn("Failed to update tally for team", { teamId, error });
- } else {
+ try {
+ await updateTallyTeam(teamId);
logger.info("Updated tally for team", { teamId });
+ } catch (error) {
+ logger.warn("Failed to update tally for team", { teamId, error });
}
}
diff --git a/apps/api/src/services/ledger/supabase-ledger.ts b/apps/api/src/services/ledger/supabase-ledger.ts
deleted file mode 100644
index 7767aded2..000000000
--- a/apps/api/src/services/ledger/supabase-ledger.ts
+++ /dev/null
@@ -1,82 +0,0 @@
-import { createClient, SupabaseClient } from "@supabase/supabase-js";
-import { logger } from "../../lib/logger";
-import { configDotenv } from "dotenv";
-import { config } from "../../config";
-configDotenv();
-
-// SupabaseLedgerService class initializes the Supabase client for the ledger schema
-class SupabaseLedgerService {
- private client: SupabaseClient<any, "ledger", any> | null = null;
- private rrClient: SupabaseClient<any, "ledger", any> | null = null;
-
- constructor() {
- const supabaseUrl = config.SUPABASE_URL;
- const supabaseReplicaUrl = config.SUPABASE_REPLICA_URL;
- const supabaseServiceToken = config.SUPABASE_SERVICE_TOKEN;
- const useDbAuthentication = config.USE_DB_AUTHENTICATION;
-
- // Only initialize the Supabase client if both URL and Service Token are provided.
- if (!useDbAuthentication) {
- // Warn the user that Authentication is disabled by setting the client to null
- logger.warn(
- "Authentication is disabled. Supabase ledger client will not be initialized.",
- );
- this.client = null;
- } else if (!supabaseUrl || !supabaseServiceToken || !supabaseReplicaUrl) {
- logger.error(
- "Supabase environment variables aren't configured correctly. Supabase ledger client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable",
- );
- } else {
- this.client = createClient(supabaseUrl, supabaseServiceToken, {
- global: {
- headers: {
- "sb-lb-routing-mode": "alpha-all-services",
- },
- },
- db: {
- //@ts-ignore
- schema: "ledger",
- },
- });
-
- this.rrClient = createClient(supabaseReplicaUrl, supabaseServiceToken, {
- db: {
- //@ts-ignore
- schema: "ledger",
- },
- });
- }
- }
-
- // Provides access to the initialized Supabase client, if available.
- getClient(): SupabaseClient<any, "ledger", any> | null {
- return this.client;
- }
-
- getRRClient(): SupabaseClient<any, "ledger", any> | null {
- return this.rrClient;
- }
-}
-
-const ledgerServ = new SupabaseLedgerService();
-
-// Using a Proxy to handle dynamic access to the Supabase ledger client or service methods.
-// This approach ensures that if Supabase is not configured, any attempt to use it will result in a clear error.
-export const supabase_ledger_service: SupabaseClient<any, "ledger", any> =
- new Proxy(ledgerServ, {
- get: function (target, prop, receiver) {
- const client = target.getClient();
- // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
- if (client === null) {
- return () => {
- throw new Error("Supabase ledger client is not configured.");
- };
- }
- // Direct access to SupabaseLedgerService properties takes precedence.
- if (prop in target) {
- return Reflect.get(target, prop, receiver);
- }
- // Otherwise, delegate access to the Supabase client.
- return Reflect.get(client, prop, receiver);
- },
- }) as unknown as SupabaseClient<any, "ledger", any>;
diff --git a/apps/api/src/services/ledger/tracking.ts b/apps/api/src/services/ledger/tracking.ts
index 2c24801e4..8aba5fd7f 100644
--- a/apps/api/src/services/ledger/tracking.ts
+++ b/apps/api/src/services/ledger/tracking.ts
@@ -1,6 +1,7 @@
import { EventDataMap, EventDefinitionSlug } from "./data-schemas";
-// @ts-nocheck - Schema not in types_db yet
-import { supabase_ledger_service } from "./supabase-ledger";
+import { eq } from "drizzle-orm";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
import { getValue, setValue } from "../redis";
import { logger } from "../../lib/logger";
@@ -26,14 +27,15 @@ export async function trackEvent<T extends EventDefinitionSlug>(
providerDefinition = JSON.parse(cachedData);
} else {
// If not in cache, fetch from database
- const result = await supabase_ledger_service
- .from("provider_definitions")
- .select("id")
- .eq("slug", definitionSlug)
- .single();
-
- definitionError = result.error;
- providerDefinition = result.data;
+ try {
+ [providerDefinition] = await db
+ .select({ id: schema.provider_definitions.id })
+ .from(schema.provider_definitions)
+ .where(eq(schema.provider_definitions.slug, definitionSlug))
+ .limit(1);
+ } catch (error) {
+ definitionError = error;
+ }
// Cache the result for 24 hours (1440 minutes)
if (!definitionError && providerDefinition) {
@@ -51,23 +53,24 @@ export async function trackEvent<T extends EventDefinitionSlug>(
}
// Create the track
- const { data: track, error: trackError } = await supabase_ledger_service
- .from("tracks")
- //@ts-ignore
- .insert({
- created_at: new Date().toISOString(),
- //@ts-ignore
- provider_definition_id: providerDefinition.id,
- data: data,
- })
- .select("uuid")
- .single();
-
- if (trackError || !track) {
+ let track: { uuid: string } | undefined;
+ try {
+ [track] = await db
+ .insert(schema.tracks)
+ .values({
+ created_at: new Date().toISOString(),
+ provider_definition_id: providerDefinition.id,
+ data: data,
+ })
+ .returning({ uuid: schema.tracks.uuid });
+ } catch (trackError) {
logger.error("Error creating track:", trackError);
return null;
}
- //@ts-ignore
+
+ if (!track) {
+ return null;
+ }
return track.uuid;
} catch (error) {
logger.error("Error tracking event:", error);
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index 617588321..0df813e01 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -23,14 +23,10 @@ jest.mock("../../lib/logger", () => ({
logger,
}));
-const insert = jest.fn<(data: any) => Promise<{ error: any }>>();
-const from = jest.fn(() => ({
- insert,
-}));
-jest.mock("../supabase", () => ({
- supabase_service: {
- from,
- },
+const values = jest.fn<(data: any) => Promise<void>>();
+const insert = jest.fn(() => ({ values }));
+jest.mock("../../db/connection", () => ({
+ db: { insert },
}));
jest.mock("../../lib/gcs-jobs", () => ({
@@ -47,6 +43,7 @@ jest.mock("../../lib/extract/extract-redis", () => ({
}));
import { logSearch, type LoggedSearch } from "./log_job";
+import * as schema from "../../db/schema";
function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {
return {
@@ -71,7 +68,7 @@ function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {
describe("logSearch", () => {
beforeEach(() => {
jest.clearAllMocks();
- insert.mockResolvedValue({ error: null });
+ values.mockResolvedValue(undefined);
});
it("removes null bytes from search query log fields", async () => {
@@ -85,8 +82,8 @@ describe("logSearch", () => {
await logSearch(search);
- expect(from).toHaveBeenCalledWith("searches");
- const inserted = insert.mock.calls[0][0];
+ expect(insert).toHaveBeenCalledWith(schema.searches);
+ const inserted = values.mock.calls[0][0];
expect(inserted.query).toBe("helloworld");
expect(inserted.options.query).toBe("nestedquery");
expect(inserted.options.sources[0].location).toBe("New\u0000York");
@@ -94,12 +91,11 @@ describe("logSearch", () => {
});
it("uses sanitized data in Sentry insert failure context", async () => {
- insert.mockResolvedValueOnce({
- error: {
+ values.mockRejectedValueOnce(
+ Object.assign(new Error("unsupported Unicode escape sequence"), {
code: "22P05",
- message: "unsupported Unicode escape sequence",
- },
- });
+ }),
+ );
await logSearch(
makeSearch({
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 508ce7423..ada3d82c9 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -1,9 +1,12 @@
-import { supabase_service } from "../supabase";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
+import { changeTrackingInsertScrape } from "../../db/rpc";
import { config } from "../../config";
import "dotenv/config";
import { logger as _logger } from "../../lib/logger";
import { configDotenv } from "dotenv";
import * as Sentry from "@sentry/node";
+import type { PgTable } from "drizzle-orm/pg-core";
import {
saveDeepResearchToGCS,
saveExtractToGCS,
@@ -33,58 +36,90 @@ function sanitizeString(value: string | null | undefined): string | null {
return value.replace(nullByteRegex, "");
}
+const tableMap: Record<string, PgTable> = {
+ requests: schema.requests,
+ scrapes: schema.scrapes,
+ parses: schema.parses,
+ crawls: schema.crawls,
+ batch_scrapes: schema.batch_scrapes,
+ searches: schema.searches,
+ extracts: schema.extracts,
+ maps: schema.maps,
+ llmstxts: schema.llmstxts,
+ deep_researches: schema.deep_researches,
+};
+
async function robustInsert(
table: string,
data: any,
force: boolean,
- logger: Logger,
+ _logger: Logger,
) {
+ const logger = _logger.child({
+ module: "log_job",
+ method: "robustInsert",
+ table,
+ canonicalLog: "log_job/robustInsert",
+ });
+
if (config.USE_DB_AUTHENTICATION !== true) {
logger.info(
"Skipping database insertion due to USE_DB_AUTHENTICATION being off",
- { table },
);
return;
}
+ const target = tableMap[table];
+
+ const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];
+
if (force) {
- let i = 0,
- done = false;
- let lastError: any = null;
- while (i++ <= 10) {
+ for (let i = 0; i < 10; i++) {
+ const start = Date.now();
try {
- const { error } = await supabase_service.from(table).insert(data);
- if (error) {
- lastError = error;
- logger.error(
- "Error inserting into database due to Supabase error, trying again",
- { error, table, attempt: i },
- );
- await new Promise(resolve => setTimeout(resolve, 75));
- } else {
- done = true;
- break;
- }
+ await db.insert(target).values(data);
+ attempts.push({
+ error: null,
+ timeMs: Date.now() - start,
+ backoffMs: i === 0 ? 0 : 75,
+ });
+ break;
} catch (error) {
- lastError = error;
- logger.error(
- "Error inserting into database due to unknown error, trying again",
- { error, table, attempt: i },
- );
+ attempts.push({
+ error,
+ timeMs: Date.now() - start,
+ backoffMs: i === 0 ? 0 : 75,
+ });
await new Promise(resolve => setTimeout(resolve, 75));
}
}
- if (done) {
- logger.info("Inserted into database successfully", { table });
+ if (attempts.length === 1 && attempts[0].error === null) {
+ logger.debug(
+ "Inserted into database successfully (" + table + ", " + data.id + ")",
+ { attempts },
+ );
+ } else if (
+ attempts.length > 1 &&
+ attempts[attempts.length - 1].error === null
+ ) {
+ logger.warn(
+ "Inserted into database successfully with retries (" +
+ table +
+ ", " +
+ data.id +
+ ")",
+ { attempts },
+ );
} else {
- logger.error("Failed to insert into database after 10 attempts", {
- table,
- lastError,
- });
+ logger.error(
+ "Failed to insert into database (" + table + ", " + data.id + ")",
+ { attempts },
+ );
// Report to Sentry with context
Sentry.captureException(
- lastError || new Error("Database insert failed after 10 attempts"),
+ attempts[attempts.length - 1]?.error ||
+ new Error("Database insert failed after 10 attempts"),
{
tags: {
table,
@@ -94,40 +129,28 @@ async function robustInsert(
table,
data: JSON.stringify(data).substring(0, 500), // Limit size
attempts: 10,
- lastError: lastError ? JSON.stringify(lastError) : null,
+ lastError: attempts[attempts.length - 1]?.error
+ ? JSON.stringify(attempts[attempts.length - 1].error)
+ : null,
},
},
);
}
} else {
+ const start = Date.now();
try {
- const { error } = await supabase_service.from(table).insert(data);
- if (error) {
- logger.error("Error inserting into database due to Supabase error", {
- error,
- table,
- });
- // Report to Sentry
- Sentry.captureException(error, {
- tags: {
- table,
- operation: "robustInsert",
- force: "false",
- },
- extra: {
- table,
- error: JSON.stringify(error),
- data: JSON.stringify(data).substring(0, 500), // Limit size
- },
- });
- } else {
- logger.info("Inserted into database successfully", { table });
- }
+ await db.insert(target).values(data);
+ attempts.push({ error: null, timeMs: Date.now() - start, backoffMs: 0 });
+ logger.debug(
+ "Inserted into database successfully (" + table + ", " + data.id + ")",
+ { attempts },
+ );
} catch (error) {
- logger.error("Error inserting into database due to unknown error", {
- error,
- table,
- });
+ attempts.push({ error, timeMs: Date.now() - start, backoffMs: 0 });
+ logger.error(
+ "Failed to insert into database (" + table + ", " + data.id + ")",
+ { attempts },
+ );
// Report to Sentry
Sentry.captureException(error, {
tags: {
@@ -295,27 +318,21 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
);
if (hasMarkdown || hasChangeTracking) {
- const { error } = await supabase_service.rpc(
- "change_tracking_insert_scrape",
- {
- p_team_id: scrape.team_id,
- p_url: scrape.url,
- p_job_id: scrape.id,
- p_change_tracking_tag: hasChangeTracking
- ? hasChangeTracking.tag
- : null,
- p_date_added: new Date().toISOString(),
- },
- );
-
- if (error) {
+ try {
+ await changeTrackingInsertScrape({
+ team_id: scrape.team_id,
+ url: scrape.url,
+ job_id: scrape.id,
+ change_tracking_tag: hasChangeTracking ? hasChangeTracking.tag : null,
+ date_added: new Date().toISOString(),
+ });
+ _logger.debug("Change tracking record inserted successfully");
+ } catch (error) {
_logger.warn("Error inserting into change_tracking_scrapes", {
error,
scrapeId: scrape.id,
teamId: scrape.team_id,
});
- } else {
- _logger.debug("Change tracking record inserted successfully");
}
}
}
diff --git a/apps/api/src/services/monitoring/email_recipients.test.ts b/apps/api/src/services/monitoring/email_recipients.test.ts
index 8e7b60e2d..a737b6cab 100644
--- a/apps/api/src/services/monitoring/email_recipients.test.ts
+++ b/apps/api/src/services/monitoring/email_recipients.test.ts
@@ -13,10 +13,7 @@ const calls: CallRecord[] = [];
// eq/neq calls and resolves at .maybeSingle() / .single() with the next
// queued result. Tests stay explicit about query order without coupling to
// the precise method chain shape.
-type QueuedResponse = (
- client: ClientKind,
- op: ChainOp,
-) => Promise<QueryResult>;
+type QueuedResponse = (client: ClientKind, op: ChainOp) => Promise<QueryResult>;
let queue: QueuedResponse[] = [];
function queueResponses(responses: QueuedResponse[]): void {
@@ -28,16 +25,21 @@ function makeChain(client: ClientKind): any {
const setOp = (next: ChainOp) => {
if (op === null) op = next;
};
- const resolve = (): Promise<QueryResult> => {
+ // Drizzle returns row arrays and throws on error; convert the queued
+ // {data, error} responses accordingly.
+ const resolve = (): Promise<unknown[]> => {
const finalOp = op ?? "unknown";
calls.push({ client, op: finalOp });
const next = queue.shift();
if (!next) {
throw new Error(
- `No queued Supabase response for client=${client} op=${finalOp} (queue exhausted)`,
+ `No queued response for client=${client} op=${finalOp} (queue exhausted)`,
);
}
- return next(client, finalOp);
+ return next(client, finalOp).then(({ data, error }) => {
+ if (error) return Promise.reject(error);
+ return data === null || data === undefined ? [] : [data];
+ });
};
const builder: any = {
select: () => {
@@ -56,21 +58,27 @@ function makeChain(client: ClientKind): any {
setOp("delete");
return builder;
},
+ from: () => builder,
+ values: () => builder,
+ set: () => builder,
+ where: () => builder,
eq: () => builder,
neq: () => builder,
in: () => builder,
- maybeSingle: resolve,
- single: resolve,
+ orderBy: () => builder,
+ limit: () => resolve(),
+ returning: () => resolve(),
+ then: (res: any, rej: any) => resolve().then(res, rej),
};
return builder;
}
-jest.mock("../supabase", () => ({
- supabase_service: {
- from: (...args: unknown[]) => fromPrimary(...args),
+jest.mock("../../db/connection", () => ({
+ get db() {
+ return makeChain("primary");
},
- supabase_rr_service: {
- from: (...args: unknown[]) => fromRR(...args),
+ get dbRr() {
+ return makeChain("rr");
},
}));
@@ -118,10 +126,14 @@ const baseEnsureInput = {
},
};
-const ok = (data: unknown): QueuedResponse => () =>
- Promise.resolve({ data, error: null });
-const fail = (error: unknown): QueuedResponse => () =>
- Promise.resolve({ data: null, error });
+const ok =
+ (data: unknown): QueuedResponse =>
+ () =>
+ Promise.resolve({ data, error: null });
+const fail =
+ (error: unknown): QueuedResponse =>
+ () =>
+ Promise.resolve({ data: null, error });
describe("ensureMonitorEmailRecipient", () => {
it("returns existing row when the recipient already exists (no insert)", async () => {
diff --git a/apps/api/src/services/monitoring/email_recipients.ts b/apps/api/src/services/monitoring/email_recipients.ts
index c00a36e46..4644028c5 100644
--- a/apps/api/src/services/monitoring/email_recipients.ts
+++ b/apps/api/src/services/monitoring/email_recipients.ts
@@ -1,6 +1,8 @@
import { randomBytes } from "crypto";
+import { and, eq, inArray, ne } from "drizzle-orm";
+import { db, dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { logger as _logger } from "../../lib/logger";
-import { supabase_rr_service, supabase_service } from "../supabase";
const logger = _logger.child({ module: "monitor-email-recipients" });
@@ -35,22 +37,28 @@ function generateRecipientToken(): string {
return randomBytes(32).toString("base64url");
}
-function throwIfError(error: any, message: string): void {
- if (error) {
- throw new Error(`${message}: ${error.message ?? JSON.stringify(error)}`);
+async function run<T>(fn: () => Promise<T>, message: string): Promise<T> {
+ try {
+ return await fn();
+ } catch (error) {
+ throw new Error(
+ `${message}: ${error instanceof Error ? error.message : JSON.stringify(error)}`,
+ );
}
}
export async function listMonitorEmailRecipients(
monitorId: string,
): Promise<MonitorEmailRecipientRow[]> {
- const { data, error } = await supabase_rr_service
- .from("monitor_email_recipients")
- .select("*")
- .eq("monitor_id", monitorId);
-
- throwIfError(error, "Failed to list monitor email recipients");
- return (data ?? []) as MonitorEmailRecipientRow[];
+ const data = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitor_email_recipients)
+ .where(eq(schema.monitor_email_recipients.monitor_id, monitorId)),
+ "Failed to list monitor email recipients",
+ );
+ return data as MonitorEmailRecipientRow[];
}
async function getRecipientByToken(
@@ -59,33 +67,35 @@ async function getRecipientByToken(
const trimmed = token.trim();
if (!trimmed) return null;
- const { data, error } = await supabase_rr_service
- .from("monitor_email_recipients")
- .select("*")
- .eq("token", trimmed)
- .maybeSingle();
-
- throwIfError(error, "Failed to look up monitor email recipient by token");
+ const [data] = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitor_email_recipients)
+ .where(eq(schema.monitor_email_recipients.token, trimmed))
+ .limit(1),
+ "Failed to look up monitor email recipient by token",
+ );
return (data ?? null) as MonitorEmailRecipientRow | null;
}
export async function getMonitorNameById(
monitorId: string,
): Promise<string | null> {
- const { data, error } = await supabase_rr_service
- .from("monitors")
- .select("name")
- .eq("id", monitorId)
- .maybeSingle();
-
- if (error) {
+ try {
+ const [data] = await dbRr
+ .select({ name: schema.monitors.name })
+ .from(schema.monitors)
+ .where(eq(schema.monitors.id, monitorId))
+ .limit(1);
+ return data?.name ?? null;
+ } catch (error) {
logger.warn("Failed to load monitor name for opt-in response", {
error,
monitorId,
});
return null;
}
- return (data?.name as string | undefined) ?? null;
}
// Team members are auto-confirmed; they already have dashboard access.
@@ -95,12 +105,14 @@ export async function getTeamMemberEmails(
): Promise<Set<string>> {
if (emails.length === 0) return new Set();
- const { data, error } = await supabase_rr_service
- .from("user_teams")
- .select("users(email)")
- .eq("team_id", teamId);
-
- if (error) {
+ let rows: { email: string | null }[];
+ try {
+ rows = await dbRr
+ .select({ email: schema.users.email })
+ .from(schema.user_teams)
+ .innerJoin(schema.users, eq(schema.user_teams.user_id, schema.users.id))
+ .where(eq(schema.user_teams.team_id, teamId));
+ } catch (error) {
logger.warn("Failed to load team member emails for recipient sync", {
error,
teamId,
@@ -110,10 +122,9 @@ export async function getTeamMemberEmails(
const wanted = new Set(emails.map(normalizeRecipientEmail));
const matches = new Set<string>();
- for (const row of data ?? []) {
- const email = (row as any).users?.email;
- if (typeof email === "string") {
- const normalized = normalizeRecipientEmail(email);
+ for (const row of rows) {
+ if (typeof row.email === "string") {
+ const normalized = normalizeRecipientEmail(row.email);
if (wanted.has(normalized)) matches.add(normalized);
}
}
@@ -135,13 +146,20 @@ async function fetchRecipientByMonitorEmail(
monitorId: string,
email: string,
): Promise<MonitorEmailRecipientRow | null> {
- const { data, error } = await supabase_rr_service
- .from("monitor_email_recipients")
- .select("*")
- .eq("monitor_id", monitorId)
- .eq("email", email)
- .maybeSingle();
- throwIfError(error, "Failed to look up monitor email recipient");
+ const [data] = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitor_email_recipients)
+ .where(
+ and(
+ eq(schema.monitor_email_recipients.monitor_id, monitorId),
+ eq(schema.monitor_email_recipients.email, email),
+ ),
+ )
+ .limit(1),
+ "Failed to look up monitor email recipient",
+ );
return (data ?? null) as MonitorEmailRecipientRow | null;
}
@@ -152,13 +170,20 @@ async function fetchRecipientByMonitorEmailPrimary(
monitorId: string,
email: string,
): Promise<MonitorEmailRecipientRow | null> {
- const { data, error } = await supabase_service
- .from("monitor_email_recipients")
- .select("*")
- .eq("monitor_id", monitorId)
- .eq("email", email)
- .maybeSingle();
- throwIfError(error, "Failed to look up monitor email recipient");
+ const [data] = await run(
+ () =>
+ db
+ .select()
+ .from(schema.monitor_email_recipients)
+ .where(
+ and(
+ eq(schema.monitor_email_recipients.monitor_id, monitorId),
+ eq(schema.monitor_email_recipients.email, email),
+ ),
+ )
+ .limit(1),
+ "Failed to look up monitor email recipient",
+ );
return (data ?? null) as MonitorEmailRecipientRow | null;
}
@@ -167,12 +192,15 @@ async function fetchRecipientByMonitorEmailPrimary(
async function fetchRecipientByIdPrimary(
id: string,
): Promise<MonitorEmailRecipientRow | null> {
- const { data, error } = await supabase_service
- .from("monitor_email_recipients")
- .select("*")
- .eq("id", id)
- .maybeSingle();
- throwIfError(error, "Failed to look up monitor email recipient by id");
+ const [data] = await run(
+ () =>
+ db
+ .select()
+ .from(schema.monitor_email_recipients)
+ .where(eq(schema.monitor_email_recipients.id, id))
+ .limit(1),
+ "Failed to look up monitor email recipient by id",
+ );
return (data ?? null) as MonitorEmailRecipientRow | null;
}
@@ -207,13 +235,13 @@ export async function ensureMonitorEmailRecipient(params: {
updated_at: now,
};
- const { data, error } = await supabase_service
- .from("monitor_email_recipients")
- .insert(insert)
- .select("*")
- .maybeSingle();
-
- if (error) {
+ let data: MonitorEmailRecipientRow | undefined;
+ try {
+ [data] = (await db
+ .insert(schema.monitor_email_recipients)
+ .values(insert)
+ .returning()) as MonitorEmailRecipientRow[];
+ } catch (error) {
if ((error as { code?: string }).code === POSTGRES_UNIQUE_VIOLATION) {
const winner = await fetchRecipientByMonitorEmailPrimary(
params.monitorId,
@@ -223,7 +251,9 @@ export async function ensureMonitorEmailRecipient(params: {
return { row: winner, created: false };
}
}
- throwIfError(error, "Failed to insert monitor email recipient");
+ throw new Error(
+ `Failed to insert monitor email recipient: ${error instanceof Error ? error.message : JSON.stringify(error)}`,
+ );
}
return { row: data as MonitorEmailRecipientRow, created: true };
@@ -231,12 +261,14 @@ export async function ensureMonitorEmailRecipient(params: {
export async function markRecipientConfirmationSent(id: string): Promise<void> {
const now = new Date().toISOString();
- const { error } = await supabase_service
- .from("monitor_email_recipients")
- .update({ confirmation_sent_at: now, updated_at: now })
- .eq("id", id);
-
- throwIfError(error, "Failed to mark recipient confirmation sent");
+ await run(
+ () =>
+ db
+ .update(schema.monitor_email_recipients)
+ .set({ confirmation_sent_at: now, updated_at: now })
+ .where(eq(schema.monitor_email_recipients.id, id)),
+ "Failed to mark recipient confirmation sent",
+ );
}
export async function confirmRecipientByToken(
@@ -254,15 +286,20 @@ export async function confirmRecipientByToken(
// affect 0 rows; re-fetch and return the actual current state so an
// in-flight confirm can't silently overwrite a terminal unsubscribe.
const now = new Date().toISOString();
- const { data, error } = await supabase_service
- .from("monitor_email_recipients")
- .update({ status: "confirmed", confirmed_at: now, updated_at: now })
- .eq("id", row.id)
- .eq("status", "pending")
- .select("*")
- .maybeSingle();
-
- throwIfError(error, "Failed to confirm monitor email recipient");
+ const [data] = await run(
+ () =>
+ db
+ .update(schema.monitor_email_recipients)
+ .set({ status: "confirmed", confirmed_at: now, updated_at: now })
+ .where(
+ and(
+ eq(schema.monitor_email_recipients.id, row.id),
+ eq(schema.monitor_email_recipients.status, "pending"),
+ ),
+ )
+ .returning(),
+ "Failed to confirm monitor email recipient",
+ );
if (data) return data as MonitorEmailRecipientRow;
return await fetchRecipientByIdPrimary(row.id);
@@ -279,15 +316,20 @@ export async function unsubscribeRecipientByToken(
// so two racing unsubscribes don't double-write timestamps. 0 rows here
// means a concurrent call beat us to the terminal state.
const now = new Date().toISOString();
- const { data, error } = await supabase_service
- .from("monitor_email_recipients")
- .update({ status: "unsubscribed", unsubscribed_at: now, updated_at: now })
- .eq("id", row.id)
- .neq("status", "unsubscribed")
- .select("*")
- .maybeSingle();
-
- throwIfError(error, "Failed to unsubscribe monitor email recipient");
+ const [data] = await run(
+ () =>
+ db
+ .update(schema.monitor_email_recipients)
+ .set({ status: "unsubscribed", unsubscribed_at: now, updated_at: now })
+ .where(
+ and(
+ eq(schema.monitor_email_recipients.id, row.id),
+ ne(schema.monitor_email_recipients.status, "unsubscribed"),
+ ),
+ )
+ .returning(),
+ "Failed to unsubscribe monitor email recipient",
+ );
if (data) return data as MonitorEmailRecipientRow;
return await fetchRecipientByIdPrimary(row.id);
@@ -296,12 +338,12 @@ export async function unsubscribeRecipientByToken(
export async function touchRecipientsNotified(ids: string[]): Promise<void> {
if (ids.length === 0) return;
const now = new Date().toISOString();
- const { error } = await supabase_service
- .from("monitor_email_recipients")
- .update({ last_notified_at: now, updated_at: now })
- .in("id", ids);
-
- if (error) {
+ try {
+ await db
+ .update(schema.monitor_email_recipients)
+ .set({ last_notified_at: now, updated_at: now })
+ .where(inArray(schema.monitor_email_recipients.id, ids));
+ } catch (error) {
logger.warn("Failed to update last_notified_at on recipients", {
error,
ids,
diff --git a/apps/api/src/services/monitoring/email_recipients_sync.test.ts b/apps/api/src/services/monitoring/email_recipients_sync.test.ts
index 4f38d06a2..84e504839 100644
--- a/apps/api/src/services/monitoring/email_recipients_sync.test.ts
+++ b/apps/api/src/services/monitoring/email_recipients_sync.test.ts
@@ -1,8 +1,3 @@
-jest.mock("../supabase", () => ({
- supabase_service: {},
- supabase_rr_service: {},
-}));
-
const mockEnsureRecipient = jest.fn();
const mockListRecipients = jest.fn();
const mockGetTeamMemberEmails = jest.fn();
@@ -52,7 +47,8 @@ function recipientRow(
status,
token: `tok-${email}`,
source,
- confirmation_sent_at: status === "pending" ? new Date().toISOString() : null,
+ confirmation_sent_at:
+ status === "pending" ? new Date().toISOString() : null,
confirmed_at: status === "confirmed" ? new Date().toISOString() : null,
unsubscribed_at:
status === "unsubscribed" ? new Date().toISOString() : null,
@@ -201,11 +197,7 @@ describe("syncMonitorEmailRecipients", () => {
mockListRecipients.mockResolvedValue([]);
mockGetTeamMemberEmails.mockResolvedValue(new Set(["[email protected]"]));
mockEnsureRecipient.mockImplementation(async ({ input }) => ({
- row: recipientRow(
- input.email,
- input.status,
- input.source,
- ),
+ row: recipientRow(input.email, input.status, input.source),
created: true,
}));
@@ -214,7 +206,8 @@ describe("syncMonitorEmailRecipients", () => {
});
expect(mockSendConfirmationEmail).toHaveBeenCalledTimes(1);
- const emailedTo = mockSendConfirmationEmail.mock.calls[0][0].recipient.email;
+ const emailedTo =
+ mockSendConfirmationEmail.mock.calls[0][0].recipient.email;
expect(emailedTo).toBe("[email protected]");
const byEmail = new Map(result.recipients.map(r => [r.email, r]));
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 25c1d94d1..e4cade567 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -533,7 +533,7 @@ async function runCrawlTarget(params: {
const doc = job.returnvalue ?? (await getJobFromGCS(job.id))?.[0];
if (!doc) continue;
const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);
- seen.add(hashMonitorUrl(url));
+ seen.add(hashMonitorUrl(url).toString("hex"));
credits += estimateActualCredits(doc, body.scrapeOptions);
pages.push(
await diffAndPersistPage({
@@ -554,7 +554,7 @@ async function runCrawlTarget(params: {
targetId: params.target.id,
});
for (const previous of previousPages) {
- if (seen.has(previous.url_hash)) continue;
+ if (seen.has(previous.url_hash.toString("hex"))) continue;
await upsertMonitorPage({
monitorId: params.monitor.id,
teamId: params.monitor.team_id,
@@ -1054,7 +1054,7 @@ async function processRemovedPagesForCompletedCrawls(params: {
const seen = new Set(
checkPages
.filter(page => page.target_id === target.targetId)
- .map(page => page.url_hash),
+ .map(page => page.url_hash.toString("hex")),
);
const activePages = await listActiveMonitorPages({
monitorId: params.monitor.id,
@@ -1063,7 +1063,7 @@ async function processRemovedPagesForCompletedCrawls(params: {
const removed: MonitorCheckPageInsert[] = [];
for (const previous of activePages) {
- if (seen.has(previous.url_hash)) continue;
+ if (seen.has(previous.url_hash.toString("hex"))) continue;
await upsertMonitorPage({
monitorId: params.monitor.id,
teamId: params.monitor.team_id,
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 2cec0f79b..51f0f7cb9 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -1,6 +1,9 @@
import { createHash } from "crypto";
import { v7 as uuidv7 } from "uuid";
-import { supabase_rr_service, supabase_service } from "../supabase";
+import { and, asc, count, desc, eq, isNull, ne } from "drizzle-orm";
+import { db, dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
+import { monitoringClaimDueMonitors } from "../../db/rpc";
import {
getNextMonitorRunAt,
estimateRunsPerMonth,
@@ -17,8 +20,8 @@ import type {
UpdateMonitorRequest,
} from "./types";
-export function hashMonitorUrl(url: string): string {
- return `\\x${createHash("sha256").update(url).digest("hex")}`;
+export function hashMonitorUrl(url: string): Buffer {
+ return createHash("sha256").update(url).digest();
}
function ensureTargetIds(targets: Array<Record<string, any>>): MonitorTarget[] {
@@ -62,9 +65,13 @@ function toMonitorSummary(check: MonitorCheckRow): MonitorSummary {
};
}
-function throwIfError(error: any, message: string): void {
- if (error) {
- throw new Error(`${message}: ${error.message ?? JSON.stringify(error)}`);
+async function run<T>(fn: () => Promise<T>, message: string): Promise<T> {
+ try {
+ return await fn();
+ } catch (error) {
+ throw new Error(
+ `${message}: ${error instanceof Error ? error.message : JSON.stringify(error)}`,
+ );
}
}
@@ -93,7 +100,7 @@ export async function createMonitor(params: {
// Omit goal/judge_enabled keys when undefined so a pre-migration DB
// doesn't reject the insert. Migration lives in a separate repo.
- const insert: Record<string, unknown> = {
+ const insert: typeof schema.monitors.$inferInsert = {
id: uuidv7(),
team_id: params.teamId,
name: params.input.name,
@@ -112,13 +119,11 @@ export async function createMonitor(params: {
if (params.input.judgeEnabled !== undefined) {
insert.judge_enabled = params.input.judgeEnabled;
}
- const { data, error } = await supabase_service
- .from("monitors")
- .insert(insert)
- .select("*")
- .single();
+ const [data] = await run(
+ () => db.insert(schema.monitors).values(insert).returning(),
+ "Failed to create monitor",
+ );
- throwIfError(error, "Failed to create monitor");
return data as MonitorRow;
}
@@ -127,48 +132,67 @@ export async function listMonitors(params: {
limit: number;
offset: number;
}): Promise<MonitorRow[]> {
- const { data, error } = await supabase_rr_service
- .from("monitors")
- .select("*")
- .eq("team_id", params.teamId)
- .neq("status", "deleted")
- .order("created_at", { ascending: false })
- .range(params.offset, params.offset + params.limit - 1);
-
- throwIfError(error, "Failed to list monitors");
- return (data ?? []) as MonitorRow[];
+ const data = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitors)
+ .where(
+ and(
+ eq(schema.monitors.team_id, params.teamId),
+ ne(schema.monitors.status, "deleted"),
+ ),
+ )
+ .orderBy(desc(schema.monitors.created_at))
+ .limit(params.limit)
+ .offset(params.offset),
+ "Failed to list monitors",
+ );
+ return data as MonitorRow[];
}
export async function getMonitor(
teamId: string,
monitorId: string,
): Promise<MonitorRow | null> {
- const { data, error } = await supabase_rr_service
- .from("monitors")
- .select("*")
- .eq("id", monitorId)
- .eq("team_id", teamId)
- .neq("status", "deleted")
- .maybeSingle();
-
- throwIfError(error, "Failed to get monitor");
- return data as MonitorRow | null;
+ const [data] = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitors)
+ .where(
+ and(
+ eq(schema.monitors.id, monitorId),
+ eq(schema.monitors.team_id, teamId),
+ ne(schema.monitors.status, "deleted"),
+ ),
+ )
+ .limit(1),
+ "Failed to get monitor",
+ );
+ return (data ?? null) as MonitorRow | null;
}
export async function getMonitorForUpdate(
teamId: string,
monitorId: string,
): Promise<MonitorRow | null> {
- const { data, error } = await supabase_service
- .from("monitors")
- .select("*")
- .eq("id", monitorId)
- .eq("team_id", teamId)
- .neq("status", "deleted")
- .maybeSingle();
-
- throwIfError(error, "Failed to get monitor");
- return data as MonitorRow | null;
+ const [data] = await run(
+ () =>
+ db
+ .select()
+ .from(schema.monitors)
+ .where(
+ and(
+ eq(schema.monitors.id, monitorId),
+ eq(schema.monitors.team_id, teamId),
+ ne(schema.monitors.status, "deleted"),
+ ),
+ )
+ .limit(1),
+ "Failed to get monitor",
+ );
+ return (data ?? null) as MonitorRow | null;
}
export async function updateMonitor(params: {
@@ -178,7 +202,7 @@ export async function updateMonitor(params: {
nextRunAt?: Date;
intervalMs?: number;
}): Promise<MonitorRow | null> {
- const patch: Record<string, unknown> = {
+ const patch: Partial<typeof schema.monitors.$inferInsert> = {
updated_at: new Date().toISOString(),
};
@@ -243,39 +267,49 @@ export async function updateMonitor(params: {
}
}
- const { data, error } = await supabase_service
- .from("monitors")
- .update(patch)
- .eq("id", params.monitorId)
- .eq("team_id", params.teamId)
- .neq("status", "deleted")
- .select("*")
- .maybeSingle();
-
- throwIfError(error, "Failed to update monitor");
- return data as MonitorRow | null;
+ const [data] = await run(
+ () =>
+ db
+ .update(schema.monitors)
+ .set(patch)
+ .where(
+ and(
+ eq(schema.monitors.id, params.monitorId),
+ eq(schema.monitors.team_id, params.teamId),
+ ne(schema.monitors.status, "deleted"),
+ ),
+ )
+ .returning(),
+ "Failed to update monitor",
+ );
+ return (data ?? null) as MonitorRow | null;
}
export async function deleteMonitor(params: {
teamId: string;
monitorId: string;
}): Promise<boolean> {
- const { data, error } = await supabase_service
- .from("monitors")
- .update({
- status: "deleted",
- deleted_at: new Date().toISOString(),
- next_run_at: null,
- updated_at: new Date().toISOString(),
- })
- .eq("id", params.monitorId)
- .eq("team_id", params.teamId)
- .neq("status", "deleted")
- .select("id")
- .maybeSingle();
-
- throwIfError(error, "Failed to delete monitor");
- return !!data;
+ const data = await run(
+ () =>
+ db
+ .update(schema.monitors)
+ .set({
+ status: "deleted",
+ deleted_at: new Date().toISOString(),
+ next_run_at: null,
+ updated_at: new Date().toISOString(),
+ })
+ .where(
+ and(
+ eq(schema.monitors.id, params.monitorId),
+ eq(schema.monitors.team_id, params.teamId),
+ ne(schema.monitors.status, "deleted"),
+ ),
+ )
+ .returning({ id: schema.monitors.id }),
+ "Failed to delete monitor",
+ );
+ return data.length > 0;
}
export async function createMonitorCheck(params: {
@@ -288,21 +322,22 @@ export async function createMonitorCheck(params: {
params.monitor.targets,
Boolean(params.monitor.judge_enabled) && Boolean(params.monitor.goal),
);
- const { data, error } = await supabase_service
- .from("monitor_checks")
- .insert({
- id: uuidv7(),
- monitor_id: params.monitor.id,
- team_id: params.monitor.team_id,
- trigger: params.trigger,
- status: params.status ?? "queued",
- scheduled_for: params.scheduledFor ?? null,
- estimated_credits: estimated,
- })
- .select("*")
- .single();
-
- throwIfError(error, "Failed to create monitor check");
+ const [data] = await run(
+ () =>
+ db
+ .insert(schema.monitor_checks)
+ .values({
+ id: uuidv7(),
+ monitor_id: params.monitor.id,
+ team_id: params.monitor.team_id,
+ trigger: params.trigger,
+ status: params.status ?? "queued",
+ scheduled_for: params.scheduledFor ?? null,
+ estimated_credits: estimated,
+ })
+ .returning(),
+ "Failed to create monitor check",
+ );
return data as MonitorCheckRow;
}
@@ -310,16 +345,22 @@ export async function markMonitorRunning(params: {
monitorId: string;
checkId: string;
}): Promise<void> {
- const { error } = await supabase_service
- .from("monitors")
- .update({
- current_check_id: params.checkId,
- updated_at: new Date().toISOString(),
- })
- .eq("id", params.monitorId)
- .is("current_check_id", null);
-
- throwIfError(error, "Failed to mark monitor running");
+ await run(
+ () =>
+ db
+ .update(schema.monitors)
+ .set({
+ current_check_id: params.checkId,
+ updated_at: new Date().toISOString(),
+ })
+ .where(
+ and(
+ eq(schema.monitors.id, params.monitorId),
+ isNull(schema.monitors.current_check_id),
+ ),
+ ),
+ "Failed to mark monitor running",
+ );
}
export async function dispatchScheduledMonitorCheck(params: {
@@ -335,22 +376,27 @@ export async function dispatchScheduledMonitorCheck(params: {
).toISOString()
: null;
- const { data, error } = await supabase_service
- .from("monitors")
- .update({
- current_check_id: params.checkId,
- locked_at: null,
- locked_until: null,
- next_run_at: nextRunAt,
- updated_at: new Date().toISOString(),
- })
- .eq("id", params.monitor.id)
- .is("current_check_id", null)
- .select("id")
- .maybeSingle();
-
- throwIfError(error, "Failed to dispatch scheduled monitor check");
- return !!data;
+ const data = await run(
+ () =>
+ db
+ .update(schema.monitors)
+ .set({
+ current_check_id: params.checkId,
+ locked_at: null,
+ locked_until: null,
+ next_run_at: nextRunAt,
+ updated_at: new Date().toISOString(),
+ })
+ .where(
+ and(
+ eq(schema.monitors.id, params.monitor.id),
+ isNull(schema.monitors.current_check_id),
+ ),
+ )
+ .returning({ id: schema.monitors.id }),
+ "Failed to dispatch scheduled monitor check",
+ );
+ return data.length > 0;
}
export async function updateMonitorScheduleAfterRun(params: {
@@ -366,21 +412,23 @@ export async function updateMonitorScheduleAfterRun(params: {
params.monitor.schedule_timezone,
).toISOString()
: null;
- const { error } = await supabase_service
- .from("monitors")
- .update({
- current_check_id: null,
- locked_at: null,
- locked_until: null,
- last_run_at: params.check.finished_at ?? new Date().toISOString(),
- last_check_id: params.check.id,
- next_run_at: nextRunAt,
- last_check_summary: params.summary ?? toMonitorSummary(params.check),
- updated_at: new Date().toISOString(),
- })
- .eq("id", params.monitor.id);
-
- throwIfError(error, "Failed to update monitor after run");
+ await run(
+ () =>
+ db
+ .update(schema.monitors)
+ .set({
+ current_check_id: null,
+ locked_at: null,
+ locked_until: null,
+ last_run_at: params.check.finished_at ?? new Date().toISOString(),
+ last_check_id: params.check.id,
+ next_run_at: nextRunAt,
+ last_check_summary: params.summary ?? toMonitorSummary(params.check),
+ updated_at: new Date().toISOString(),
+ })
+ .where(eq(schema.monitors.id, params.monitor.id)),
+ "Failed to update monitor after run",
+ );
}
export async function advanceMonitorAfterSkippedCheck(params: {
@@ -395,20 +443,22 @@ export async function advanceMonitorAfterSkippedCheck(params: {
params.monitor.schedule_timezone,
).toISOString()
: null;
- const { error } = await supabase_service
- .from("monitors")
- .update({
- locked_at: null,
- locked_until: null,
- last_run_at: params.check.finished_at ?? new Date().toISOString(),
- last_check_id: params.check.id,
- next_run_at: nextRunAt,
- last_check_summary: toMonitorSummary(params.check),
- updated_at: new Date().toISOString(),
- })
- .eq("id", params.monitor.id);
-
- throwIfError(error, "Failed to advance monitor after skipped check");
+ await run(
+ () =>
+ db
+ .update(schema.monitors)
+ .set({
+ locked_at: null,
+ locked_until: null,
+ last_run_at: params.check.finished_at ?? new Date().toISOString(),
+ last_check_id: params.check.id,
+ next_run_at: nextRunAt,
+ last_check_summary: toMonitorSummary(params.check),
+ updated_at: new Date().toISOString(),
+ })
+ .where(eq(schema.monitors.id, params.monitor.id)),
+ "Failed to advance monitor after skipped check",
+ );
}
export async function getMonitorCheck(
@@ -416,30 +466,38 @@ export async function getMonitorCheck(
monitorId: string,
checkId: string,
): Promise<MonitorCheckRow | null> {
- const { data, error } = await supabase_rr_service
- .from("monitor_checks")
- .select("*")
- .eq("id", checkId)
- .eq("monitor_id", monitorId)
- .eq("team_id", teamId)
- .maybeSingle();
-
- throwIfError(error, "Failed to get monitor check");
- return data as MonitorCheckRow | null;
+ const [data] = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitor_checks)
+ .where(
+ and(
+ eq(schema.monitor_checks.id, checkId),
+ eq(schema.monitor_checks.monitor_id, monitorId),
+ eq(schema.monitor_checks.team_id, teamId),
+ ),
+ )
+ .limit(1),
+ "Failed to get monitor check",
+ );
+ return (data ?? null) as MonitorCheckRow | null;
}
export async function listRunningMonitorChecks(
limit: number = 100,
): Promise<MonitorCheckRow[]> {
- const { data, error } = await supabase_service
- .from("monitor_checks")
- .select("*")
- .eq("status", "running")
- .order("created_at", { ascending: true })
- .limit(limit);
-
- throwIfError(error, "Failed to list running monitor checks");
- return (data ?? []) as MonitorCheckRow[];
+ const data = await run(
+ () =>
+ db
+ .select()
+ .from(schema.monitor_checks)
+ .where(eq(schema.monitor_checks.status, "running"))
+ .orderBy(asc(schema.monitor_checks.created_at))
+ .limit(limit),
+ "Failed to list running monitor checks",
+ );
+ return data as MonitorCheckRow[];
}
export async function listMonitorChecks(params: {
@@ -449,41 +507,44 @@ export async function listMonitorChecks(params: {
offset: number;
status?: MonitorCheckRow["status"];
}): Promise<MonitorCheckRow[]> {
- let query = supabase_rr_service
- .from("monitor_checks")
- .select("*")
- .eq("monitor_id", params.monitorId)
- .eq("team_id", params.teamId)
- .order("created_at", { ascending: false });
-
+ const conditions = [
+ eq(schema.monitor_checks.monitor_id, params.monitorId),
+ eq(schema.monitor_checks.team_id, params.teamId),
+ ];
if (params.status) {
- query = query.eq("status", params.status);
+ conditions.push(eq(schema.monitor_checks.status, params.status));
}
- const { data, error } = await query.range(
- params.offset,
- params.offset + params.limit - 1,
+ const data = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitor_checks)
+ .where(and(...conditions))
+ .orderBy(desc(schema.monitor_checks.created_at))
+ .limit(params.limit)
+ .offset(params.offset),
+ "Failed to list monitor checks",
);
-
- throwIfError(error, "Failed to list monitor checks");
- return (data ?? []) as MonitorCheckRow[];
+ return data as MonitorCheckRow[];
}
export async function updateMonitorCheck(
checkId: string,
patch: Partial<MonitorCheckRow>,
): Promise<MonitorCheckRow> {
- const { data, error } = await supabase_service
- .from("monitor_checks")
- .update({
- ...patch,
- updated_at: new Date().toISOString(),
- })
- .eq("id", checkId)
- .select("*")
- .single();
-
- throwIfError(error, "Failed to update monitor check");
+ const [data] = await run(
+ () =>
+ db
+ .update(schema.monitor_checks)
+ .set({
+ ...patch,
+ updated_at: new Date().toISOString(),
+ })
+ .where(eq(schema.monitor_checks.id, checkId))
+ .returning(),
+ "Failed to update monitor check",
+ );
return data as MonitorCheckRow;
}
@@ -492,15 +553,17 @@ export async function insertMonitorCheckPages(
): Promise<void> {
if (pages.length === 0) return;
- const { error } = await supabase_service.from("monitor_check_pages").insert(
- pages.map(page => ({
- id: uuidv7(),
- ...page,
- url_hash: page.url_hash ?? hashMonitorUrl(page.url),
- })),
+ await run(
+ () =>
+ db.insert(schema.monitor_check_pages).values(
+ pages.map(page => ({
+ id: uuidv7(),
+ ...page,
+ url_hash: page.url_hash ?? hashMonitorUrl(page.url),
+ })),
+ ),
+ "Failed to insert monitor check pages",
);
-
- throwIfError(error, "Failed to insert monitor check pages");
}
export async function listMonitorCheckPages(params: {
@@ -511,25 +574,27 @@ export async function listMonitorCheckPages(params: {
skip: number;
status?: string;
}): Promise<any[]> {
- let query = supabase_rr_service
- .from("monitor_check_pages")
- .select("*")
- .eq("check_id", params.checkId)
- .eq("monitor_id", params.monitorId)
- .eq("team_id", params.teamId)
- .order("created_at", { ascending: true });
-
+ const conditions = [
+ eq(schema.monitor_check_pages.check_id, params.checkId),
+ eq(schema.monitor_check_pages.monitor_id, params.monitorId),
+ eq(schema.monitor_check_pages.team_id, params.teamId),
+ ];
if (params.status) {
- query = query.eq("status", params.status);
+ conditions.push(eq(schema.monitor_check_pages.status, params.status));
}
- const { data, error } = await query.range(
- params.skip,
- params.skip + params.limit - 1,
+ const data = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitor_check_pages)
+ .where(and(...conditions))
+ .orderBy(asc(schema.monitor_check_pages.created_at))
+ .limit(params.limit)
+ .offset(params.skip),
+ "Failed to list monitor check pages",
);
-
- throwIfError(error, "Failed to list monitor check pages");
- return data ?? [];
+ return data;
}
export async function countMonitorCheckPages(params: {
@@ -537,33 +602,24 @@ export async function countMonitorCheckPages(params: {
targetId?: string;
status?: string;
}): Promise<number> {
- const pageSize = 1000;
- let total = 0;
- let offset = 0;
-
- while (true) {
- let query = supabase_rr_service
- .from("monitor_check_pages")
- .select("id")
- .eq("check_id", params.checkId);
-
- if (params.targetId) {
- query = query.eq("target_id", params.targetId);
- }
- if (params.status) {
- query = query.eq("status", params.status);
- }
-
- const { data, error } = await query.range(offset, offset + pageSize - 1);
- throwIfError(error, "Failed to count monitor check pages");
-
- const batch = data ?? [];
- total += batch.length;
- if (batch.length < pageSize) break;
- offset += pageSize;
+ const conditions = [eq(schema.monitor_check_pages.check_id, params.checkId)];
+ if (params.targetId) {
+ conditions.push(eq(schema.monitor_check_pages.target_id, params.targetId));
}
+ if (params.status) {
+ conditions.push(eq(schema.monitor_check_pages.status, params.status));
+ }
+
+ const [row] = await run(
+ () =>
+ dbRr
+ .select({ value: count() })
+ .from(schema.monitor_check_pages)
+ .where(and(...conditions)),
+ "Failed to count monitor check pages",
+ );
- return total;
+ return row?.value ?? 0;
}
export async function getMonitorPage(params: {
@@ -571,16 +627,22 @@ export async function getMonitorPage(params: {
targetId: string;
url: string;
}): Promise<MonitorPageRow | null> {
- const { data, error } = await supabase_rr_service
- .from("monitor_pages")
- .select("*")
- .eq("monitor_id", params.monitorId)
- .eq("target_id", params.targetId)
- .eq("url_hash", hashMonitorUrl(params.url))
- .maybeSingle();
-
- throwIfError(error, "Failed to get monitor page");
- return data as MonitorPageRow | null;
+ const [data] = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitor_pages)
+ .where(
+ and(
+ eq(schema.monitor_pages.monitor_id, params.monitorId),
+ eq(schema.monitor_pages.target_id, params.targetId),
+ eq(schema.monitor_pages.url_hash, hashMonitorUrl(params.url)),
+ ),
+ )
+ .limit(1),
+ "Failed to get monitor page",
+ );
+ return (data ?? null) as MonitorPageRow | null;
}
export async function upsertMonitorPage(params: {
@@ -603,34 +665,36 @@ export async function upsertMonitorPage(params: {
});
if (!existing) {
- const { error } = await supabase_service.from("monitor_pages").insert({
- monitor_id: params.monitorId,
- team_id: params.teamId,
- target_id: params.targetId,
- url: params.url,
- url_hash: hashMonitorUrl(params.url),
- source: params.source,
- first_seen_check_id: params.checkId,
- last_seen_check_id:
- params.status === "removed" ? undefined : params.checkId,
- last_changed_check_id:
- params.status === "changed" || params.status === "new"
- ? params.checkId
- : undefined,
- last_scrape_id: params.scrapeId,
- last_status: params.status,
- is_removed: params.status === "removed",
- removed_at: params.status === "removed" ? now : null,
- metadata: params.metadata ?? null,
- created_at: now,
- updated_at: now,
- });
-
- throwIfError(error, "Failed to insert monitor page");
+ await run(
+ () =>
+ db.insert(schema.monitor_pages).values({
+ monitor_id: params.monitorId,
+ team_id: params.teamId,
+ target_id: params.targetId,
+ url: params.url,
+ url_hash: hashMonitorUrl(params.url),
+ source: params.source,
+ first_seen_check_id: params.checkId,
+ last_seen_check_id:
+ params.status === "removed" ? undefined : params.checkId,
+ last_changed_check_id:
+ params.status === "changed" || params.status === "new"
+ ? params.checkId
+ : undefined,
+ last_scrape_id: params.scrapeId,
+ last_status: params.status,
+ is_removed: params.status === "removed",
+ removed_at: params.status === "removed" ? now : null,
+ metadata: params.metadata ?? null,
+ created_at: now,
+ updated_at: now,
+ }),
+ "Failed to insert monitor page",
+ );
return;
}
- const patch: Record<string, unknown> = {
+ const patch: Partial<typeof schema.monitor_pages.$inferInsert> = {
last_status: params.status,
is_removed: params.status === "removed",
removed_at: params.status === "removed" ? now : null,
@@ -645,40 +709,36 @@ export async function upsertMonitorPage(params: {
patch.last_changed_check_id = params.checkId;
}
- const { error } = await supabase_service
- .from("monitor_pages")
- .update(patch)
- .eq("id", existing.id);
-
- throwIfError(error, "Failed to update monitor page");
+ await run(
+ () =>
+ db
+ .update(schema.monitor_pages)
+ .set(patch)
+ .where(eq(schema.monitor_pages.id, existing.id)),
+ "Failed to update monitor page",
+ );
}
export async function listActiveMonitorPages(params: {
monitorId: string;
targetId: string;
}): Promise<MonitorPageRow[]> {
- const pageSize = 1000;
- const pages: MonitorPageRow[] = [];
- let offset = 0;
-
- while (true) {
- const { data, error } = await supabase_rr_service
- .from("monitor_pages")
- .select("*")
- .eq("monitor_id", params.monitorId)
- .eq("target_id", params.targetId)
- .eq("is_removed", false)
- .order("created_at", { ascending: true })
- .range(offset, offset + pageSize - 1);
-
- throwIfError(error, "Failed to list active monitor pages");
- const batch = (data ?? []) as MonitorPageRow[];
- pages.push(...batch);
- if (batch.length < pageSize) break;
- offset += pageSize;
- }
-
- return pages;
+ const data = await run(
+ () =>
+ dbRr
+ .select()
+ .from(schema.monitor_pages)
+ .where(
+ and(
+ eq(schema.monitor_pages.monitor_id, params.monitorId),
+ eq(schema.monitor_pages.target_id, params.targetId),
+ eq(schema.monitor_pages.is_removed, false),
+ ),
+ )
+ .orderBy(asc(schema.monitor_pages.created_at)),
+ "Failed to list active monitor pages",
+ );
+ return data as MonitorPageRow[];
}
export async function claimDueMonitors(params: {
@@ -686,31 +746,27 @@ export async function claimDueMonitors(params: {
limit: number;
leaseSeconds: number;
}): Promise<MonitorRow[]> {
- const { data, error } = await supabase_service.rpc(
- "monitoring_claim_due_monitors",
- {
- p_worker_id: params.workerId,
- p_limit: params.limit,
- p_lease_seconds: params.leaseSeconds,
- },
+ const data = await run(
+ () => monitoringClaimDueMonitors<MonitorRow>(params),
+ "Failed to claim due monitors",
);
-
- throwIfError(error, "Failed to claim due monitors");
- return (data ?? []) as MonitorRow[];
+ return data;
}
export async function deferMonitorClaim(
monitorId: string,
until: Date,
): Promise<void> {
- const { error } = await supabase_service
- .from("monitors")
- .update({
- locked_until: until.toISOString(),
- locked_at: null,
- updated_at: new Date().toISOString(),
- })
- .eq("id", monitorId);
-
- throwIfError(error, "Failed to defer monitor claim");
+ await run(
+ () =>
+ db
+ .update(schema.monitors)
+ .set({
+ locked_until: until.toISOString(),
+ locked_at: null,
+ updated_at: new Date().toISOString(),
+ })
+ .where(eq(schema.monitors.id, monitorId)),
+ "Failed to defer monitor claim",
+ );
}
diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.ts
index e043a6ef8..4f3d33c35 100644
--- a/apps/api/src/services/monitoring/types.ts
+++ b/apps/api/src/services/monitoring/types.ts
@@ -238,7 +238,7 @@ export type MonitorPageRow = {
team_id: string;
target_id: string;
url: string;
- url_hash: string;
+ url_hash: Buffer;
source: MonitorPageSource;
first_seen_check_id: string | null;
last_seen_check_id: string | null;
@@ -267,7 +267,7 @@ export type MonitorCheckPageInsert = {
team_id: string;
target_id: string;
url: string;
- url_hash?: string;
+ url_hash?: Buffer;
status: MonitorPageStatus;
previous_scrape_id?: string | null;
current_scrape_id?: string | null;
diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.ts
index c91b2a846..dbda0a2ec 100644
--- a/apps/api/src/services/notification/email_notification.ts
+++ b/apps/api/src/services/notification/email_notification.ts
@@ -1,4 +1,6 @@
-import { supabase_service } from "../supabase";
+import { and, eq, gte, lte } from "drizzle-orm";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
import { config } from "../../config";
import { withAuth } from "../../lib/withAuth";
import { Resend } from "resend";
@@ -104,25 +106,35 @@ async function sendEmailNotification(
try {
// Get user's email preferences
- const { data: user, error: userError } = await supabase_service
- .from("users")
- .select("id")
- .eq("email", email)
- .single();
-
- if (userError) {
+ let user: { id: string } | undefined;
+ try {
+ [user] = await db
+ .select({ id: schema.users.id })
+ .from(schema.users)
+ .where(eq(schema.users.email, email))
+ .limit(1);
+ } catch (userError) {
logger.debug(`Error fetching user: ${userError}`);
return { success: false };
}
+ if (!user) {
+ return { success: false };
+ }
// Check user's email preferences
- const { data: preferences, error: prefError } = await supabase_service
- .from("notification_preferences")
- .select("unsubscribed_all, email_preferences")
- .eq("user_id", user.id)
- .single();
-
- if (prefError) {
+ let preferences:
+ | { unsubscribed_all: boolean | null; email_preferences: string[] | null }
+ | undefined;
+ try {
+ [preferences] = await db
+ .select({
+ unsubscribed_all: schema.notification_preferences.unsubscribed_all,
+ email_preferences: schema.notification_preferences.email_preferences,
+ })
+ .from(schema.notification_preferences)
+ .where(eq(schema.notification_preferences.user_id, user.id))
+ .limit(1);
+ } catch (prefError) {
logger.debug(`Error fetching preferences: ${prefError}`);
return { success: false };
}
@@ -214,14 +226,25 @@ async function sendNotificationInternal(
const fifteenDaysAgo = new Date();
fifteenDaysAgo.setDate(fifteenDaysAgo.getDate() - 15);
- const { data, error } = await supabase_service
- .from("user_notifications")
- .select("*")
- .eq("team_id", team_id)
- .eq("notification_type", notificationType)
- .gte("sent_date", fifteenDaysAgo.toISOString());
-
- if (error) {
+ let data: { id: string }[];
+ try {
+ data = await db
+ .select({ id: schema.user_notifications.id })
+ .from(schema.user_notifications)
+ .where(
+ and(
+ eq(schema.user_notifications.team_id, team_id),
+ eq(
+ schema.user_notifications.notification_type,
+ notificationType,
+ ),
+ gte(
+ schema.user_notifications.sent_date,
+ fifteenDaysAgo.toISOString(),
+ ),
+ ),
+ );
+ } catch (error) {
logger.debug(`Error fetching notifications: ${error}`);
return { success: false };
}
@@ -232,18 +255,30 @@ async function sendNotificationInternal(
// TODO: observation: Free credits people are not receiving notifications
- const { data: recentData, error: recentError } = await supabase_service
- .from("user_notifications")
- .select("*")
- .eq("team_id", team_id)
- .eq("notification_type", notificationType)
- .gte("sent_date", startDateString)
- .lte("sent_date", endDateString);
-
- if (recentError) {
- logger.debug(
- `Error fetching recent notifications: ${recentError.message}`,
- );
+ let recentData: { id: string }[];
+ try {
+ recentData = await db
+ .select({ id: schema.user_notifications.id })
+ .from(schema.user_notifications)
+ .where(
+ and(
+ eq(schema.user_notifications.team_id, team_id),
+ eq(
+ schema.user_notifications.notification_type,
+ notificationType,
+ ),
+ gte(
+ schema.user_notifications.sent_date,
+ startDateString as string,
+ ),
+ lte(
+ schema.user_notifications.sent_date,
+ endDateString as string,
+ ),
+ ),
+ );
+ } catch (recentError) {
+ logger.debug(`Error fetching recent notifications: ${recentError}`);
return { success: false };
}
@@ -266,32 +301,36 @@ async function sendNotificationInternal(
});
}
// get the emails from the user with the team_id
- const { data: emails, error: emailsError } = await supabase_service
- .from("users")
- .select("email")
- .eq("team_id", team_id);
-
- if (emailsError) {
+ let emails: { email: string | null }[];
+ try {
+ emails = await db
+ .select({ email: schema.users.email })
+ .from(schema.users)
+ .where(eq(schema.users.team_id, team_id));
+ } catch (emailsError) {
logger.debug(`Error fetching emails: ${emailsError}`);
return { success: false };
}
if (!is_ledger_enabled) {
for (const email of emails) {
- await sendEmailNotification(email.email, notificationType, context);
+ if (email.email) {
+ await sendEmailNotification(email.email, notificationType, context);
+ }
}
}
- const { error: insertError } = await supabase_service
- .from("user_notifications")
- .insert([
- {
- team_id: team_id,
- notification_type: notificationType,
- sent_date: new Date().toISOString(),
- timestamp: new Date().toISOString(),
- },
- ]);
+ let insertError: unknown = null;
+ try {
+ await db.insert(schema.user_notifications).values({
+ team_id: team_id,
+ notification_type: notificationType,
+ sent_date: new Date().toISOString(),
+ timestamp: new Date().toISOString(),
+ });
+ } catch (error) {
+ insertError = error;
+ }
if (config.SLACK_ADMIN_WEBHOOK_URL && emails.length > 0) {
sendSlackWebhook(
@@ -352,15 +391,19 @@ export async function sendNotificationWithCustomDays(
now.getTime() - daysBetweenEmails * 24 * 60 * 60 * 1000,
);
- const { data: recentNotifications, error: recentNotificationsError } =
- await supabase_service
- .from("user_notifications")
- .select("*")
- .eq("team_id", team_id)
- .eq("notification_type", notificationType)
- .gte("sent_date", pastDate.toISOString());
-
- if (recentNotificationsError) {
+ let recentNotifications: { id: string }[];
+ try {
+ recentNotifications = await db
+ .select({ id: schema.user_notifications.id })
+ .from(schema.user_notifications)
+ .where(
+ and(
+ eq(schema.user_notifications.team_id, team_id),
+ eq(schema.user_notifications.notification_type, notificationType),
+ gte(schema.user_notifications.sent_date, pastDate.toISOString()),
+ ),
+ );
+ } catch (recentNotificationsError) {
logger.debug(
`Error fetching recent notifications: ${recentNotificationsError}`,
);
@@ -385,12 +428,13 @@ export async function sendNotificationWithCustomDays(
`Sending notification for team_id: ${team_id} and notificationType: ${notificationType}`,
);
// get the emails from the user with the team_id
- const { data: emails, error: emailsError } = await supabase_service
- .from("users")
- .select("email")
- .eq("team_id", team_id);
-
- if (emailsError) {
+ let emails: { email: string | null }[];
+ try {
+ emails = await db
+ .select({ email: schema.users.email })
+ .from(schema.users)
+ .where(eq(schema.users.team_id, team_id));
+ } catch (emailsError) {
logger.debug(`Error fetching emails: ${emailsError}`);
await redisEvictConnection.del(redisKey); // free up redis, let it try again
return { success: false };
@@ -408,20 +452,23 @@ export async function sendNotificationWithCustomDays(
if (!is_ledger_enabled) {
for (const email of emails) {
- await sendEmailNotification(email.email, notificationType);
+ if (email.email) {
+ await sendEmailNotification(email.email, notificationType);
+ }
}
}
- const { error: insertError } = await supabase_service
- .from("user_notifications")
- .insert([
- {
- team_id: team_id,
- notification_type: notificationType,
- sent_date: new Date().toISOString(),
- timestamp: new Date().toISOString(),
- },
- ]);
+ let insertError: unknown = null;
+ try {
+ await db.insert(schema.user_notifications).values({
+ team_id: team_id,
+ notification_type: notificationType,
+ sent_date: new Date().toISOString(),
+ timestamp: new Date().toISOString(),
+ });
+ } catch (error) {
+ insertError = error;
+ }
if (
config.SLACK_ADMIN_WEBHOOK_URL &&
diff --git a/apps/api/src/services/notification/monitoring_email.test.ts b/apps/api/src/services/notification/monitoring_email.test.ts
index a568abf0d..1757f4e60 100644
--- a/apps/api/src/services/notification/monitoring_email.test.ts
+++ b/apps/api/src/services/notification/monitoring_email.test.ts
@@ -1,8 +1,3 @@
-jest.mock("../supabase", () => ({
- supabase_service: {},
- supabase_rr_service: {},
-}));
-
const mockEnsureRecipient = jest.fn();
const mockListRecipients = jest.fn();
const mockTouchNotified = jest.fn();
@@ -70,7 +65,8 @@ function fakeRecipient(
status,
token: `tok-${sanitized}`,
source,
- confirmation_sent_at: status === "pending" ? new Date().toISOString() : null,
+ confirmation_sent_at:
+ status === "pending" ? new Date().toISOString() : null,
confirmed_at: status === "confirmed" ? new Date().toISOString() : null,
unsubscribed_at:
status === "unsubscribed" ? new Date().toISOString() : null,
@@ -200,7 +196,7 @@ describe("sendMonitoringConfirmationEmail", () => {
expect(mockResendSend).toHaveBeenCalledTimes(1);
expect(mockResendSend.mock.calls[0][0].to).toBe("[email protected]");
expect(mockResendSend.mock.calls[0][0].subject).toContain(
- 'Confirm subscription',
+ "Confirm subscription",
);
expect(mockMarkConfirmationSent).toHaveBeenCalledWith("rec-newexamplecom");
});
diff --git a/apps/api/src/services/notification/monitoring_email.ts b/apps/api/src/services/notification/monitoring_email.ts
index 7b8a8a446..2b85e4bc9 100644
--- a/apps/api/src/services/notification/monitoring_email.ts
+++ b/apps/api/src/services/notification/monitoring_email.ts
@@ -1,8 +1,10 @@
import { Resend } from "resend";
import escapeHtml from "escape-html";
+import { eq } from "drizzle-orm";
import { config } from "../../config";
import { logger as _logger } from "../../lib/logger";
-import { supabase_service } from "../supabase";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
import type { MonitorCheckRow, MonitorRow } from "../monitoring/types";
import {
ensureMonitorEmailRecipient,
@@ -88,31 +90,39 @@ export type MonitoringEmailPayload = {
};
async function getTeamEmails(teamId: string): Promise<string[]> {
- const { data, error } = await supabase_service
- .from("user_teams")
- .select(
- "users(email, id, notification_preferences(unsubscribed_all, email_preferences))",
- )
- .eq("team_id", teamId);
-
- if (error) {
+ let rows: {
+ email: string | null;
+ unsubscribed_all: boolean | null;
+ email_preferences: string[] | null;
+ }[];
+ try {
+ rows = await db
+ .select({
+ email: schema.users.email,
+ unsubscribed_all: schema.notification_preferences.unsubscribed_all,
+ email_preferences: schema.notification_preferences.email_preferences,
+ })
+ .from(schema.user_teams)
+ .innerJoin(schema.users, eq(schema.user_teams.user_id, schema.users.id))
+ .leftJoin(
+ schema.notification_preferences,
+ eq(schema.notification_preferences.user_id, schema.users.id),
+ )
+ .where(eq(schema.user_teams.team_id, teamId));
+ } catch (error) {
logger.warn("Failed to load team emails", { error, teamId });
return [];
}
const emails = new Set<string>();
- for (const row of data ?? []) {
- const user = (row as any).users;
- const email = user?.email;
+ for (const row of rows) {
+ const email = row.email;
if (!email) continue;
- const prefs = Array.isArray(user.notification_preferences)
- ? user.notification_preferences[0]
- : user.notification_preferences;
- if (prefs?.unsubscribed_all) continue;
+ if (row.unsubscribed_all) continue;
if (
- Array.isArray(prefs?.email_preferences) &&
- !prefs.email_preferences.includes("system_alerts")
+ Array.isArray(row.email_preferences) &&
+ !row.email_preferences.includes("system_alerts")
) {
continue;
}
@@ -266,13 +276,10 @@ export async function sendMonitoringConfirmationEmail(params: {
}): Promise<{ attempted: boolean; success: boolean; error?: string }> {
const resend = getResendClient();
if (!resend) {
- logger.warn(
- "Skipping monitor opt-in email; RESEND_API_KEY is not set",
- {
- monitorId: params.recipient.monitor_id,
- recipientId: params.recipient.id,
- },
- );
+ logger.warn("Skipping monitor opt-in email; RESEND_API_KEY is not set", {
+ monitorId: params.recipient.monitor_id,
+ recipientId: params.recipient.id,
+ });
return { attempted: false, success: true };
}
@@ -366,10 +373,13 @@ async function resolveSendableRecipients(
}),
);
rows = legacyRows;
- logger.info("Bootstrapped legacy monitor recipients without DB backfill", {
- monitorId: monitor.id,
- recipients: explicitConfigured,
- });
+ logger.info(
+ "Bootstrapped legacy monitor recipients without DB backfill",
+ {
+ monitorId: monitor.id,
+ recipients: explicitConfigured,
+ },
+ );
}
const rowsByEmail = new Map(rows.map(r => [r.email, r]));
@@ -514,16 +524,13 @@ export async function sendMonitoringEmailSummary(params: {
const resolved = await resolveSendableRecipients(params.monitor);
if (resolved.confirmedRecipients.length === 0) {
- logger.info(
- "Skipping monitoring email summary; no confirmed recipients",
- {
- monitorId: params.monitor.id,
- checkId: params.check.id,
- configured: resolved.total,
- pending: resolved.pending,
- unsubscribed: resolved.unsubscribed,
- },
- );
+ logger.info("Skipping monitoring email summary; no confirmed recipients", {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ configured: resolved.total,
+ pending: resolved.pending,
+ unsubscribed: resolved.unsubscribed,
+ });
return {
attempted: false,
success: true,
@@ -589,8 +596,7 @@ export async function sendMonitoringEmailSummary(params: {
return {
recipient,
success: false,
- error:
- typeof error === "string" ? error : JSON.stringify(error),
+ error: typeof error === "string" ? error : JSON.stringify(error),
};
}
return { recipient, success: true };
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 289a544e9..a96a70961 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -459,7 +459,7 @@ app.listen(workerPort, () => {
initializeEngineForcing();
- if (config.USE_DB_AUTHENTICATION) {
+ if (config.USE_DB_AUTHENTICATION && !config.DISABLE_MONITORING) {
monitorSchedulerInterval = setInterval(() => {
enqueueDueMonitorChecks().catch(error => {
_logger.error("Failed to enqueue due monitor checks", { error });
diff --git a/apps/api/src/services/subscription/enterprise-check.ts b/apps/api/src/services/subscription/enterprise-check.ts
index 577eaab85..cfd9e6c46 100644
--- a/apps/api/src/services/subscription/enterprise-check.ts
+++ b/apps/api/src/services/subscription/enterprise-check.ts
@@ -1,39 +1,38 @@
-import { supabase_service } from "../supabase";
-
-interface SubscriptionResponse {
- prices: {
- products: {
- is_enterprise: boolean;
- };
- };
-}
+import { and, eq, gt } from "drizzle-orm";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
const RATE_LIMIT_CHANGE_NOTIFICATION_START_DATE = new Date("2025-03-12");
export async function isEnterpriseTeamCreatedAfterRateLimitChange(
team_id: string,
): Promise<boolean> {
- const { data, error } = (await supabase_service
- .from("subscriptions")
- .select("prices(products(is_enterprise))")
- .eq("status", "active")
- .eq("team_id", team_id)
- .gt(
- "created",
- RATE_LIMIT_CHANGE_NOTIFICATION_START_DATE.toISOString(),
- )) as {
- data: SubscriptionResponse[] | null;
- error: any;
- };
+ try {
+ const data = await db
+ .select({ is_enterprise: schema.products.is_enterprise })
+ .from(schema.subscriptions)
+ .innerJoin(
+ schema.prices,
+ eq(schema.subscriptions.price_id, schema.prices.id),
+ )
+ .innerJoin(
+ schema.products,
+ eq(schema.prices.product_id, schema.products.id),
+ )
+ .where(
+ and(
+ eq(schema.subscriptions.status, "active"),
+ eq(schema.subscriptions.team_id, team_id),
+ gt(
+ schema.subscriptions.created,
+ RATE_LIMIT_CHANGE_NOTIFICATION_START_DATE.toISOString(),
+ ),
+ ),
+ );
- if (error || !data) {
+ return data.some(sub => sub.is_enterprise === true);
+ } catch (error) {
// If there's an error or no subscription found, assume non-enterprise
return false;
}
-
- const isEnterprise = data.find(
- sub => sub.prices?.products?.is_enterprise === true,
- );
-
- return !!isEnterprise;
}
diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.ts
deleted file mode 100644
index 224c58c25..000000000
--- a/apps/api/src/services/supabase.ts
+++ /dev/null
@@ -1,99 +0,0 @@
-import { createClient, SupabaseClient } from "@supabase/supabase-js";
-import { configDotenv } from "dotenv";
-import { config } from "../config";
-import { logger } from "../lib/logger";
-configDotenv();
-
-/** PostgREST code when .single() returns 0 or >1 rows. Use to distinguish "no row" from real DB errors. */
-const POSTGREST_NO_ROWS_CODE = "PGRST116";
-
-export function isPostgrestNoRowsError(
- error: { code?: string } | null | undefined,
-): boolean {
- return error?.code === POSTGREST_NO_ROWS_CODE;
-}
-
-// SupabaseService class initializes the Supabase client conditionally based on environment variables.
-class SupabaseService {
- private client: SupabaseClient | null = null;
- private rrClient: SupabaseClient | null = null;
-
- constructor() {
- const supabaseUrl = config.SUPABASE_URL;
- const supabaseReplicaUrl = config.SUPABASE_REPLICA_URL;
- const supabaseServiceToken = config.SUPABASE_SERVICE_TOKEN;
- const useDbAuthentication = config.USE_DB_AUTHENTICATION;
- // Only initialize the Supabase client if both URL and Service Token are provided.
- if (!useDbAuthentication) {
- // Warn the user that Authentication is disabled by setting the client to null
- logger.warn(
- "Authentication is disabled. Supabase client will not be initialized.",
- );
- this.client = null;
- } else if (!supabaseUrl || !supabaseServiceToken || !supabaseReplicaUrl) {
- logger.error(
- "Supabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable",
- );
- } else {
- this.client = createClient(supabaseUrl, supabaseServiceToken, {
- global: {
- headers: {
- "sb-lb-routing-mode": "alpha-all-services",
- },
- },
- });
-
- this.rrClient = createClient(supabaseReplicaUrl, supabaseServiceToken);
- }
- }
-
- // Provides access to the initialized Supabase client, if available.
- getClient(): SupabaseClient | null {
- return this.client;
- }
-
- getRRClient(): SupabaseClient | null {
- return this.rrClient;
- }
-}
-
-const serv = new SupabaseService();
-
-// Using a Proxy to handle dynamic access to the Supabase client or service methods.
-// This approach ensures that if Supabase is not configured, any attempt to use it will result in a clear error.
-export const supabase_service: SupabaseClient = new Proxy(serv, {
- get: function (target, prop, receiver) {
- const client = target.getClient();
- // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
- if (client === null) {
- return () => {
- throw new Error("Supabase client is not configured.");
- };
- }
- // Direct access to SupabaseService properties takes precedence.
- if (prop in target) {
- return Reflect.get(target, prop, receiver);
- }
- // Otherwise, delegate access to the Supabase client.
- return Reflect.get(client, prop, receiver);
- },
-}) as unknown as SupabaseClient;
-
-export const supabase_rr_service: SupabaseClient = new Proxy(serv, {
- get: function (target, prop, receiver) {
- const client = target.getRRClient();
- // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
- if (client === null) {
- return () => {
- throw new Error("Supabase RR client is not configured.");
- };
- }
- // Direct access to SupabaseService properties takes precedence.
- if (prop in target) {
- return Reflect.get(target, prop, receiver);
- }
- // Otherwise, delegate access to the Supabase client.
- return Reflect.get(client, prop, receiver);
- },
-}) as unknown as SupabaseClient;
-
diff --git a/apps/api/src/services/webhook/config.ts b/apps/api/src/services/webhook/config.ts
index f59ad3eca..3f6cda422 100644
--- a/apps/api/src/services/webhook/config.ts
+++ b/apps/api/src/services/webhook/config.ts
@@ -1,4 +1,6 @@
-import { supabase_rr_service } from "../supabase";
+import { eq } from "drizzle-orm";
+import { dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { WebhookConfig } from "./types";
import { config } from "../../config";
@@ -39,14 +41,13 @@ async function getHmacSecret(teamId: string): Promise<string | undefined> {
}
try {
- const { data, error } = await supabase_rr_service
- .from("teams")
- .select("hmac_secret")
- .eq("id", teamId)
- .limit(1)
- .single();
+ const [data] = await dbRr
+ .select({ hmac_secret: schema.teams.hmac_secret })
+ .from(schema.teams)
+ .where(eq(schema.teams.id, teamId))
+ .limit(1);
- return error ? undefined : data?.hmac_secret;
+ return data?.hmac_secret ?? undefined;
} catch {
return undefined;
}
diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.ts
index 93c06e3ca..60afbbecb 100644
--- a/apps/api/src/services/webhook/delivery.ts
+++ b/apps/api/src/services/webhook/delivery.ts
@@ -13,7 +13,8 @@ import type {
WebhookQueueMessage,
} from "./types";
import { redisEvictConnection } from "../redis";
-import { supabase_service } from "../supabase";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
import { webhookQueue } from "./queue";
import { randomUUID } from "crypto";
@@ -254,7 +255,7 @@ export async function processWebhookInsertJobs() {
});
try {
- await supabase_service.from("webhook_logs").insert(parsedJobs);
+ await db.insert(schema.webhook_logs).values(parsedJobs);
_logger.info("Webhook inserter inserted jobs", {
jobCount: parsedJobs.length,
});
diff --git a/apps/api/src/services/webhook/logs.ts b/apps/api/src/services/webhook/logs.ts
index af69ac1d1..77117afaa 100644
--- a/apps/api/src/services/webhook/logs.ts
+++ b/apps/api/src/services/webhook/logs.ts
@@ -1,4 +1,6 @@
-import { supabase_rr_service } from "../supabase";
+import { and, desc, eq, inArray } from "drizzle-orm";
+import { dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
import { logger as _logger } from "../../lib/logger";
export type WebhookLogRow = {
@@ -16,15 +18,29 @@ export async function getLatestWebhookLog(params: {
jobId: string;
event: string;
}): Promise<WebhookLogRow | null> {
- const { data, error } = await supabase_rr_service
- .from("webhook_logs")
- .select("id,success,error,status_code,latency_ms,url,event,created_at")
- .eq("crawl_id", params.jobId)
- .eq("event", params.event)
- .order("created_at", { ascending: false })
- .limit(1)
- .maybeSingle();
- if (error) {
+ try {
+ const [data] = await dbRr
+ .select({
+ id: schema.webhook_logs.id,
+ success: schema.webhook_logs.success,
+ error: schema.webhook_logs.error,
+ status_code: schema.webhook_logs.status_code,
+ latency_ms: schema.webhook_logs.latency_ms,
+ url: schema.webhook_logs.url,
+ event: schema.webhook_logs.event,
+ created_at: schema.webhook_logs.created_at,
+ })
+ .from(schema.webhook_logs)
+ .where(
+ and(
+ eq(schema.webhook_logs.crawl_id, params.jobId),
+ eq(schema.webhook_logs.event, params.event),
+ ),
+ )
+ .orderBy(desc(schema.webhook_logs.created_at))
+ .limit(1);
+ return (data ?? null) as WebhookLogRow | null;
+ } catch (error) {
_logger.warn("Failed to fetch latest webhook log", {
module: "webhook-logs",
jobId: params.jobId,
@@ -33,7 +49,6 @@ export async function getLatestWebhookLog(params: {
});
return null;
}
- return data as WebhookLogRow | null;
}
export async function getLatestWebhookLogsByJob(params: {
@@ -43,16 +58,29 @@ export async function getLatestWebhookLogsByJob(params: {
const result = new Map<string, WebhookLogRow>();
if (params.jobIds.length === 0) return result;
- const { data, error } = await supabase_rr_service
- .from("webhook_logs")
- .select(
- "id,success,error,status_code,latency_ms,url,event,created_at,crawl_id",
- )
- .in("crawl_id", params.jobIds)
- .eq("event", params.event)
- .order("created_at", { ascending: false });
-
- if (error) {
+ let data: (WebhookLogRow & { crawl_id: string })[];
+ try {
+ data = await dbRr
+ .select({
+ id: schema.webhook_logs.id,
+ success: schema.webhook_logs.success,
+ error: schema.webhook_logs.error,
+ status_code: schema.webhook_logs.status_code,
+ latency_ms: schema.webhook_logs.latency_ms,
+ url: schema.webhook_logs.url,
+ event: schema.webhook_logs.event,
+ created_at: schema.webhook_logs.created_at,
+ crawl_id: schema.webhook_logs.crawl_id,
+ })
+ .from(schema.webhook_logs)
+ .where(
+ and(
+ inArray(schema.webhook_logs.crawl_id, params.jobIds),
+ eq(schema.webhook_logs.event, params.event),
+ ),
+ )
+ .orderBy(desc(schema.webhook_logs.created_at));
+ } catch (error) {
_logger.warn("Failed to fetch webhook logs batch", {
module: "webhook-logs",
event: params.event,
@@ -61,7 +89,7 @@ export async function getLatestWebhookLogsByJob(params: {
return result;
}
- for (const row of (data ?? []) as (WebhookLogRow & { crawl_id: string })[]) {
+ for (const row of data) {
if (!result.has(row.crawl_id)) result.set(row.crawl_id, row);
}
return result;
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index d35926142..d3ea7aa6f 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -6,7 +6,7 @@ import {
getDoneJobsOrderedLength,
} from "../../lib/crawl-redis";
import { getCrawl } from "../../lib/crawl-redis";
-import { supabase_service } from "../supabase";
+import { creditsBilledByCrawlId } from "../../db/rpc";
import { getJobs } from "../../controllers/v1/crawl-status";
import { logCrawl, logBatchScrape } from "../logging/log_job";
import { createWebhookSender, WebhookEvent } from "../webhook/index";
@@ -129,22 +129,18 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
} else {
const num_docs = await getDoneJobsOrderedLength(crawlId);
- let credits_billed = null;
+ let credits_billed: number | null = null;
if (config.USE_DB_AUTHENTICATION) {
- const creditsRpc = await supabase_service.rpc(
- "credits_billed_by_crawl_id_2",
- {
- i_crawl_id: crawlId,
- },
- );
-
- credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;
+ try {
+ const creditsRows = await creditsBilledByCrawlId(crawlId);
+ credits_billed = creditsRows?.[0]?.credits_billed ?? null;
+ } catch (error) {
+ logger.warn("Credits billed is null", { error });
+ }
if (credits_billed === null) {
- logger.warn("Credits billed is null", {
- error: creditsRpc.error,
- });
+ logger.warn("Credits billed is null", {});
}
}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index cfe39abae..849e19aa4 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -314,6 +314,21 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
document: doc,
};
+ // Ensure the parent `requests` row is committed before any child
+ // `scrapes`/`parses` insert, to avoid a request_id FK violation. Runs on
+ // both the crawl and single-scrape paths; only single scrapes actually
+ // carry a logRequestPromise.
+ if (job.data.logRequestPromise) {
+ const start = Date.now();
+ await job.data.logRequestPromise;
+ const waited = Date.now() - start;
+ if (waited > 0) {
+ logger.warn("Had to wait for log request promise to complete", {
+ timeMs: waited,
+ });
+ }
+ }
+
if (job.data.crawl_id) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 7d4775380..172171645 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -62,6 +62,8 @@ type ScrapeJobSingleUrlsUnique = {
sentry?: any;
is_extract?: boolean;
apiKeyId: number | null;
+
+ logRequestPromise?: Promise<any>;
};
export type ScrapeJobSingleUrls = ScrapeJobCommon & ScrapeJobSingleUrlsUnique;
diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json
index b28b89b70..4b17b82b6 100644
--- a/apps/api/tsconfig.json
+++ b/apps/api/tsconfig.json
@@ -12,6 +12,7 @@
"outDir": "./dist/src",
"moduleResolution": "NodeNext",
"strictNullChecks": true,
+ "skipLibCheck": true,
"inlineSources": true,
diff --git a/requests.http b/requests.http
deleted file mode 100644
index e69de29bb..000000000