Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): avoid replica race in scrape interact (#4649)
Gergő Móriczfea716ebf02a
diff --git a/apps/api/src/__tests__/snips/v2/scrape-browser.test.ts b/apps/api/src/__tests__/snips/v2/scrape-browser.test.tsindex e7dd2e8a6..a51c3c3ae 100644--- a/apps/api/src/__tests__/snips/v2/scrape-browser.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-browser.test.ts@@ -17,30 +17,6 @@ import {scrapeTimeout,} from "./lib";-const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));--async function interactWithReplicaRetry(- jobId: string,- body: {- code: string;- language?: "python" | "node" | "bash";- timeout?: number;- },- identity: Identity,- attempts: number = 5,-) {- let lastResponse: Awaited<ReturnType<typeof scrapeInteractRaw>> | null = null;-- for (let i = 0; i < attempts; i += 1) {- const response = await scrapeInteractRaw(jobId, body, identity);- lastResponse = response;- if (response.statusCode !== 404) return response;- await sleep(500);- }-- return lastResponse!;-}-describe("Scrape browser interact replay", () => {let identity: Identity;let otherIdentity: Identity;@@ -91,7 +67,7 @@ describe("Scrape browser interact replay", () => {expect(typeof scrapeResponse.body.scrape_id).toBe("string");scrapeId = scrapeResponse.body.scrape_id as string;- const executeResponse = await interactWithReplicaRetry(+ const executeResponse = await scrapeInteractRaw(scrapeId,{language: "node",@@ -144,7 +120,7 @@ describe("Scrape browser interact replay", () => {expect(typeof scrapeResponse.body.scrape_id).toBe("string");scrapeId = scrapeResponse.body.scrape_id as string;- const executeResponse = await interactWithReplicaRetry(+ const executeResponse = await scrapeInteractRaw(scrapeId,{language: "node",@@ -213,7 +189,7 @@ describe("Scrape browser interact replay", () => {// Session creation primes agent-browser and consolidates tabs, so// user code must see exactly one tab: the content page.- const executeResponse = await interactWithReplicaRetry(+ const executeResponse = await scrapeInteractRaw(scrapeId,{language: "node",@@ -306,7 +282,7 @@ describe("Scrape browser interact replay", () => {expect(typeof scrapeResponse.body.scrape_id).toBe("string");const scrapeId = scrapeResponse.body.scrape_id as string;- const executeResponse = await interactWithReplicaRetry(+ const executeResponse = await scrapeInteractRaw(scrapeId,{code: "console.log('should fail')",@@ -339,7 +315,7 @@ describe("Scrape browser interact replay", () => {expect(typeof scrapeResponse.body.scrape_id).toBe("string");const scrapeId = scrapeResponse.body.scrape_id as string;- const executeResponse = await interactWithReplicaRetry(+ const executeResponse = await scrapeInteractRaw(scrapeId,{code: "console.log('should not run')",diff --git a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.tsindex 4d88c1da5..77c99f0fd 100644--- a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts+++ b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts@@ -1,7 +1,7 @@import type { Response } from "express";import { vi } from "vitest";import { config } from "../../../config";-import { supabaseGetScrapeById } from "../../../lib/supabase-jobs";+import { supabaseGetScrapeByIdDirect } from "../../../lib/supabase-jobs";import { scrapeInteractController } from "../scrape-browser";import type { RequestWithAuth } from "../types";@@ -10,7 +10,7 @@ vi.mock("uuid", () => ({}));vi.mock("../../../lib/supabase-jobs", () => ({- supabaseGetScrapeById: vi.fn(),+ supabaseGetScrapeByIdDirect: vi.fn(),}));vi.mock("../../../lib/browser-sessions", () => ({@@ -93,7 +93,7 @@ describe("scrapeInteractController", () => {await scrapeInteractController(req, res);- expect(supabaseGetScrapeById).not.toHaveBeenCalled();+ expect(supabaseGetScrapeByIdDirect).not.toHaveBeenCalled();expect(res.status).toHaveBeenCalledWith(501);expect(res.json).toHaveBeenCalledWith({success: false,diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex e0e5273f4..a50600ba0 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -61,7 +61,7 @@ import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activitimport { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";import { integrationSchema } from "../../utils/integration";-import { supabaseGetScrapeById } from "../../lib/supabase-jobs";+import { supabaseGetScrapeByIdDirect } from "../../lib/supabase-jobs";import {BROWSER_CREDITS_PER_HOUR,INTERACT_CREDITS_PER_HOUR,@@ -167,7 +167,7 @@ export async function scrapeInteractController(});}- const scrape = (await supabaseGetScrapeById(+ const scrape = (await supabaseGetScrapeByIdDirect(scrapeId,)) as ScrapeContextRow | null;if (!scrape) {diff --git a/apps/api/src/lib/supabase-jobs.test.ts b/apps/api/src/lib/supabase-jobs.test.tsnew file mode 100644index 000000000..a0842f12d--- /dev/null+++ b/apps/api/src/lib/supabase-jobs.test.ts@@ -0,0 +1,63 @@+const { primarySelect, replicaSelect } = vi.hoisted(() => ({+ primarySelect: vi.fn(),+ replicaSelect: vi.fn(),+}));++vi.mock("../db/connection", () => ({+ db: { select: primarySelect },+ dbRr: { select: replicaSelect },+}));++import { supabaseGetScrapeByIdDirect } from "./supabase-jobs";+import { logger } from "./logger";++describe("supabaseGetScrapeByIdDirect", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ afterEach(() => {+ vi.restoreAllMocks();+ });++ it("reads newly-created scrapes from the primary database", async () => {+ const scrape = { id: "scrape-123" };+ const query = {+ from: vi.fn(),+ where: vi.fn(),+ limit: vi.fn().mockResolvedValue([scrape]),+ };+ query.from.mockReturnValue(query);+ query.where.mockReturnValue(query);+ primarySelect.mockReturnValue(query);++ await expect(supabaseGetScrapeByIdDirect(scrape.id)).resolves.toBe(scrape);++ expect(primarySelect).toHaveBeenCalledOnce();+ expect(replicaSelect).not.toHaveBeenCalled();+ });++ it("surfaces primary database errors instead of returning a false miss", async () => {+ const error = new Error("primary unavailable");+ const query = {+ from: vi.fn(),+ where: vi.fn(),+ limit: vi.fn().mockRejectedValue(error),+ };+ query.from.mockReturnValue(query);+ query.where.mockReturnValue(query);+ primarySelect.mockReturnValue(query);+ const logError = vi.spyOn(logger, "error").mockImplementation(() => logger);++ await expect(supabaseGetScrapeByIdDirect("scrape-123")).rejects.toBe(error);++ expect(logError).toHaveBeenCalledWith(+ "Error in supabaseGetScrapeByIdDirect",+ {+ error,+ scrapeId: "scrape-123",+ },+ );+ expect(replicaSelect).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.tsindex 46e6672eb..2a03788bf 100644--- a/apps/api/src/lib/supabase-jobs.ts+++ b/apps/api/src/lib/supabase-jobs.ts@@ -22,6 +22,29 @@ export const supabaseGetScrapeById = async (scrapeId: string): Promise<any> => {}};+/**+ * Get a single scrape by ID from the primary database.+ * Use this when the scrape may have been created immediately before the read.+ */+export const supabaseGetScrapeByIdDirect = async (+ scrapeId: string,+): Promise<any> => {+ try {+ const [data] = await db+ .select()+ .from(schema.scrapes)+ .where(eq(schema.scrapes.id, scrapeId))+ .limit(1);+ return data ?? null;+ } catch (error) {+ logger.error("Error in supabaseGetScrapeByIdDirect", {+ error,+ scrapeId,+ });+ throw error;+ }+};+/*** Get multiple scrapes by ID from the scrapes table* @param scrapeIds IDs of Scrapes