Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
perf(api): shift lag-tolerant reads from primary to read replica
mogery49aa749941f1
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex e98cda064..95de50010 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -196,11 +196,11 @@ async function getACUC(let retries = 0;const maxRetries = 5;while (retries < maxRetries) {- const database = requiresPrimaryRead- ? db- : Math.random() > 2 / 3- ? dbRr- : db;+ // General-purpose reads prefer the replica: the result is Redis-cached+ // for 10 minutes, so sub-second replication lag is irrelevant. Fall back+ // to the primary on replica error. hosted_mcp_oauth must stay on the+ // primary — revocation has to observe fresh state.+ const database = requiresPrimaryRead ? db : retries === 0 ? dbRr : db;try {data = await authCreditUsageChunk(database, api_key, credentialPurpose);break;@@ -320,7 +320,9 @@ export async function getACUCTeam(const maxRetries = 5;while (retries < maxRetries) {- const database = Math.random() > 2 / 3 ? dbRr : db;+ // Prefer the replica (10-minute Redis cache makes lag irrelevant); fall+ // back to the primary on replica error.+ const database = retries === 0 ? dbRr : db;try {data = await authCreditUsageChunkFromTeam(database, team_id);break;diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex b33a81ba6..897e2bf22 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -22,6 +22,7 @@ import {import { configDotenv } from "dotenv";import { logger } from "../../lib/logger";import { creditsBilledByCrawlId } from "../../db/rpc";+import { dbRr } from "../../db/connection";import { getJobFromGCS } from "../../lib/gcs-jobs";import {scrapeQueue,@@ -197,7 +198,7 @@ export async function crawlStatusController();const creditsBilled = config.USE_DB_AUTHENTICATION- ? await creditsBilledByCrawlId(req.params.jobId).catch(() => null)+ ? await creditsBilledByCrawlId(dbRr, req.params.jobId).catch(() => null): null;// check if the crawl failed during kickoff (e.g. queue full)diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 54d053364..3afa1633b 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -23,6 +23,7 @@ import {import { configDotenv } from "dotenv";import { logger } from "../../lib/logger";import { creditsBilledByCrawlId } from "../../db/rpc";+import { dbRr } from "../../db/connection";import { getJobFromGCS } from "../../lib/gcs-jobs";import {scrapeQueue,@@ -218,7 +219,7 @@ export async function crawlStatusController();const creditsBilled = config.USE_DB_AUTHENTICATION- ? await creditsBilledByCrawlId(req.params.jobId).catch(() => null)+ ? await creditsBilledByCrawlId(dbRr, req.params.jobId).catch(() => null): null;// check if the crawl failed during kickoff (e.g. queue full)diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.tsindex 69932be04..817fa88bc 100644--- a/apps/api/src/db/rpc.ts+++ b/apps/api/src/db/rpc.ts@@ -100,11 +100,16 @@ export async function changeTrackingInsertScrape(params: {);}+// `database` is a parameter because callers split between the primary and the+// read replica: status controllers (informational `creditsUsed`) read from the+// replica, while crawl finalization (crawl-logic) reads its own recent billing+// writes and must stay on the primary.export function creditsBilledByCrawlId(+ database: DB,i_crawl_id: string,): Promise<{ credits_billed: number }[]> {return execRows(- db,+ database,sql`select * from credits_billed_by_crawl_id_2(i_crawl_id => ${i_crawl_id})`,);}diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.tsindex 5943fc929..a1e1c70fa 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts@@ -1,5 +1,5 @@import { and, desc, eq, gte } from "drizzle-orm";-import { db } from "../../db/connection";+import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { config } from "../../config";import { logger } from "../logger";@@ -23,7 +23,7 @@ export async function getLlmsTextFromCache(const originUrl = normalizeUrlOnlyHostname(url);try {- const [data] = await db+ const [data] = await dbRr.select().from(schema.llm_texts).where(diff --git a/apps/api/src/lib/siem-logging/store.ts b/apps/api/src/lib/siem-logging/store.tsindex 35a59ca09..5b61734e5 100644--- a/apps/api/src/lib/siem-logging/store.ts+++ b/apps/api/src/lib/siem-logging/store.ts@@ -125,7 +125,7 @@ export async function isOrgSiemLoggingEnabled(orgId: string): Promise<boolean> {let rows: { enabled: boolean }[] = [];try {- rows = await db+ rows = await dbRr.select({ enabled: schema.siem_logging_config.enabled }).from(schema.siem_logging_config).where(eq(schema.siem_logging_config.org_id, orgId))diff --git a/apps/api/src/services/integrations/slack/store.ts b/apps/api/src/services/integrations/slack/store.tsindex a6305dca5..ffbffda7f 100644--- a/apps/api/src/services/integrations/slack/store.ts+++ b/apps/api/src/services/integrations/slack/store.ts@@ -1,9 +1,11 @@import { and, desc, eq, isNull } from "drizzle-orm";-import { db } from "../../../db/connection";+import { db, dbRr } from "../../../db/connection";import * as schema from "../../../db/schema";import type { SlackIncomingWebhook, SlackInstallationRow } from "./types";-function toRow(row: typeof schema.slack_installations.$inferSelect): SlackInstallationRow {+function toRow(+ row: typeof schema.slack_installations.$inferSelect,+): SlackInstallationRow {return {id: row.id,team_id: row.team_id,@@ -15,7 +17,8 @@ function toRow(row: typeof schema.slack_installations.$inferSelect): SlackInstalscope: row.scope,authed_user_id: row.authed_user_id,app_id: row.app_id,- incoming_webhook: (row.incoming_webhook as SlackIncomingWebhook | null) ?? null,+ incoming_webhook:+ (row.incoming_webhook as SlackIncomingWebhook | null) ?? null,revoked_at: row.revoked_at,created_at: row.created_at,updated_at: row.updated_at,@@ -66,7 +69,7 @@ export async function upsertSlackInstallation(params: {export async function getSlackInstallationByTeam(teamId: string,): Promise<SlackInstallationRow | null> {- const rows = await db+ const rows = await dbRr.select().from(schema.slack_installations).where(@@ -85,7 +88,7 @@ export async function getSlackInstallationByTeam(export async function getSlackInstallationBySlackTeam(slackTeamId: string,): Promise<SlackInstallationRow | null> {- const rows = await db+ const rows = await dbRr.select().from(schema.slack_installations).where(diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 1837c7862..b68f02eb1 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -762,7 +762,7 @@ export async function listRunningMonitorChecks(): Promise<MonitorCheckRow[]> {const data = await run(() =>- db+ dbRr.select().from(schema.monitor_checks).where(eq(schema.monitor_checks.status, "running"))diff --git a/apps/api/src/services/notification/monitoring_email.ts b/apps/api/src/services/notification/monitoring_email.tsindex dc461ba73..1caefadf6 100644--- a/apps/api/src/services/notification/monitoring_email.ts+++ b/apps/api/src/services/notification/monitoring_email.ts@@ -3,7 +3,7 @@ import escapeHtml from "escape-html";import { eq } from "drizzle-orm";import { config } from "../../config";import { logger as _logger } from "../../lib/logger";-import { db } from "../../db/connection";+import { dbRr } from "../../db/connection";import * as schema from "../../db/schema";import type { MonitorCheckRow, MonitorRow } from "../monitoring/types";import {@@ -96,7 +96,7 @@ async function getTeamEmails(teamId: string): Promise<string[]> {email_preferences: string[] | null;}[];try {- rows = await db+ rows = await dbRr.select({email: schema.users.email,unsubscribed_all: schema.notification_preferences.unsubscribed_all,diff --git a/apps/api/src/services/subscription/enterprise-check.ts b/apps/api/src/services/subscription/enterprise-check.tsindex cfd9e6c46..49c8e96ef 100644--- a/apps/api/src/services/subscription/enterprise-check.ts+++ b/apps/api/src/services/subscription/enterprise-check.ts@@ -1,5 +1,5 @@import { and, eq, gt } from "drizzle-orm";-import { db } from "../../db/connection";+import { dbRr } from "../../db/connection";import * as schema from "../../db/schema";const RATE_LIMIT_CHANGE_NOTIFICATION_START_DATE = new Date("2025-03-12");@@ -8,7 +8,7 @@ export async function isEnterpriseTeamCreatedAfterRateLimitChange(team_id: string,): Promise<boolean> {try {- const data = await db+ const data = await dbRr.select({ is_enterprise: schema.products.is_enterprise }).from(schema.subscriptions).innerJoin(diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex ae044f172..0e5120eb9 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -7,6 +7,7 @@ import {} from "../../lib/crawl-redis";import { getCrawl } from "../../lib/crawl-redis";import { creditsBilledByCrawlId } from "../../db/rpc";+import { db } from "../../db/connection";import { getJobs } from "../../controllers/v1/crawl-status";import { logCrawl, logBatchScrape } from "../logging/log_job";import { createWebhookSender, WebhookEvent } from "../webhook/index";@@ -144,7 +145,7 @@ export async function finishCrawlSuper(job: NuQJob<any>) {if (config.USE_DB_AUTHENTICATION) {try {- const creditsRows = await creditsBilledByCrawlId(crawlId);+ const creditsRows = await creditsBilledByCrawlId(db, crawlId);credits_billed = creditsRows?.[0]?.credits_billed ?? null;} catch (error) {logger.warn("Credits billed is null", { error });