Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): avoid replica race in scrape interact (#4649)

Gergő Móriczfea716ebf02a
diff --git a/apps/api/src/__tests__/snips/v2/scrape-browser.test.ts b/apps/api/src/__tests__/snips/v2/scrape-browser.test.ts
index e7dd2e8a6..a51c3c3ae 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-browser.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-browser.test.ts
@@ -17,30 +17,6 @@ import {
scrapeTimeout,
} from "./lib";
-const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-
-async function interactWithReplicaRetry(
- jobId: string,
- body: {
- code: string;
- language?: "python" | "node" | "bash";
- timeout?: number;
- },
- identity: Identity,
- attempts: number = 5,
-) {
- let lastResponse: Awaited<ReturnType<typeof scrapeInteractRaw>> | null = null;
-
- for (let i = 0; i < attempts; i += 1) {
- const response = await scrapeInteractRaw(jobId, body, identity);
- lastResponse = response;
- if (response.statusCode !== 404) return response;
- await sleep(500);
- }
-
- return lastResponse!;
-}
-
describe("Scrape browser interact replay", () => {
let identity: Identity;
let otherIdentity: Identity;
@@ -91,7 +67,7 @@ describe("Scrape browser interact replay", () => {
expect(typeof scrapeResponse.body.scrape_id).toBe("string");
scrapeId = scrapeResponse.body.scrape_id as string;
- const executeResponse = await interactWithReplicaRetry(
+ const executeResponse = await scrapeInteractRaw(
scrapeId,
{
language: "node",
@@ -144,7 +120,7 @@ describe("Scrape browser interact replay", () => {
expect(typeof scrapeResponse.body.scrape_id).toBe("string");
scrapeId = scrapeResponse.body.scrape_id as string;
- const executeResponse = await interactWithReplicaRetry(
+ const executeResponse = await scrapeInteractRaw(
scrapeId,
{
language: "node",
@@ -213,7 +189,7 @@ describe("Scrape browser interact replay", () => {
// Session creation primes agent-browser and consolidates tabs, so
// user code must see exactly one tab: the content page.
- const executeResponse = await interactWithReplicaRetry(
+ const executeResponse = await scrapeInteractRaw(
scrapeId,
{
language: "node",
@@ -306,7 +282,7 @@ describe("Scrape browser interact replay", () => {
expect(typeof scrapeResponse.body.scrape_id).toBe("string");
const scrapeId = scrapeResponse.body.scrape_id as string;
- const executeResponse = await interactWithReplicaRetry(
+ const executeResponse = await scrapeInteractRaw(
scrapeId,
{
code: "console.log('should fail')",
@@ -339,7 +315,7 @@ describe("Scrape browser interact replay", () => {
expect(typeof scrapeResponse.body.scrape_id).toBe("string");
const scrapeId = scrapeResponse.body.scrape_id as string;
- const executeResponse = await interactWithReplicaRetry(
+ const executeResponse = await scrapeInteractRaw(
scrapeId,
{
code: "console.log('should not run')",
diff --git a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts
index 4d88c1da5..77c99f0fd 100644
--- a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts
@@ -1,7 +1,7 @@
import type { Response } from "express";
import { vi } from "vitest";
import { config } from "../../../config";
-import { supabaseGetScrapeById } from "../../../lib/supabase-jobs";
+import { supabaseGetScrapeByIdDirect } from "../../../lib/supabase-jobs";
import { scrapeInteractController } from "../scrape-browser";
import type { RequestWithAuth } from "../types";
@@ -10,7 +10,7 @@ vi.mock("uuid", () => ({
}));
vi.mock("../../../lib/supabase-jobs", () => ({
- supabaseGetScrapeById: vi.fn(),
+ supabaseGetScrapeByIdDirect: vi.fn(),
}));
vi.mock("../../../lib/browser-sessions", () => ({
@@ -93,7 +93,7 @@ describe("scrapeInteractController", () => {
await scrapeInteractController(req, res);
- expect(supabaseGetScrapeById).not.toHaveBeenCalled();
+ expect(supabaseGetScrapeByIdDirect).not.toHaveBeenCalled();
expect(res.status).toHaveBeenCalledWith(501);
expect(res.json).toHaveBeenCalledWith({
success: false,
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index e0e5273f4..a50600ba0 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -61,7 +61,7 @@ import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activit
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { integrationSchema } from "../../utils/integration";
-import { supabaseGetScrapeById } from "../../lib/supabase-jobs";
+import { supabaseGetScrapeByIdDirect } from "../../lib/supabase-jobs";
import {
BROWSER_CREDITS_PER_HOUR,
INTERACT_CREDITS_PER_HOUR,
@@ -167,7 +167,7 @@ export async function scrapeInteractController(
});
}
- const scrape = (await supabaseGetScrapeById(
+ const scrape = (await supabaseGetScrapeByIdDirect(
scrapeId,
)) as ScrapeContextRow | null;
if (!scrape) {
diff --git a/apps/api/src/lib/supabase-jobs.test.ts b/apps/api/src/lib/supabase-jobs.test.ts
new file mode 100644
index 000000000..a0842f12d
--- /dev/null
+++ b/apps/api/src/lib/supabase-jobs.test.ts
@@ -0,0 +1,63 @@
+const { primarySelect, replicaSelect } = vi.hoisted(() => ({
+ primarySelect: vi.fn(),
+ replicaSelect: vi.fn(),
+}));
+
+vi.mock("../db/connection", () => ({
+ db: { select: primarySelect },
+ dbRr: { select: replicaSelect },
+}));
+
+import { supabaseGetScrapeByIdDirect } from "./supabase-jobs";
+import { logger } from "./logger";
+
+describe("supabaseGetScrapeByIdDirect", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it("reads newly-created scrapes from the primary database", async () => {
+ const scrape = { id: "scrape-123" };
+ const query = {
+ from: vi.fn(),
+ where: vi.fn(),
+ limit: vi.fn().mockResolvedValue([scrape]),
+ };
+ query.from.mockReturnValue(query);
+ query.where.mockReturnValue(query);
+ primarySelect.mockReturnValue(query);
+
+ await expect(supabaseGetScrapeByIdDirect(scrape.id)).resolves.toBe(scrape);
+
+ expect(primarySelect).toHaveBeenCalledOnce();
+ expect(replicaSelect).not.toHaveBeenCalled();
+ });
+
+ it("surfaces primary database errors instead of returning a false miss", async () => {
+ const error = new Error("primary unavailable");
+ const query = {
+ from: vi.fn(),
+ where: vi.fn(),
+ limit: vi.fn().mockRejectedValue(error),
+ };
+ query.from.mockReturnValue(query);
+ query.where.mockReturnValue(query);
+ primarySelect.mockReturnValue(query);
+ const logError = vi.spyOn(logger, "error").mockImplementation(() => logger);
+
+ await expect(supabaseGetScrapeByIdDirect("scrape-123")).rejects.toBe(error);
+
+ expect(logError).toHaveBeenCalledWith(
+ "Error in supabaseGetScrapeByIdDirect",
+ {
+ error,
+ scrapeId: "scrape-123",
+ },
+ );
+ expect(replicaSelect).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.ts
index 46e6672eb..2a03788bf 100644
--- a/apps/api/src/lib/supabase-jobs.ts
+++ b/apps/api/src/lib/supabase-jobs.ts
@@ -22,6 +22,29 @@ export const supabaseGetScrapeById = async (scrapeId: string): Promise<any> => {
}
};
+/**
+ * Get a single scrape by ID from the primary database.
+ * Use this when the scrape may have been created immediately before the read.
+ */
+export const supabaseGetScrapeByIdDirect = async (
+ scrapeId: string,
+): Promise<any> => {
+ try {
+ const [data] = await db
+ .select()
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.id, scrapeId))
+ .limit(1);
+ return data ?? null;
+ } catch (error) {
+ logger.error("Error in supabaseGetScrapeByIdDirect", {
+ error,
+ scrapeId,
+ });
+ throw error;
+ }
+};
+
/**
* Get multiple scrapes by ID from the scrapes table
* @param scrapeIds IDs of Scrapes