Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): scope site restriction errors to Safe Mode (#4648)

Gergő Móriczef6f09726a7d
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/siteProtection.test.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/siteProtection.test.ts
new file mode 100644
index 000000000..97c7ef957
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/siteProtection.test.ts
@@ -0,0 +1,103 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const fetchFake = vi.hoisted(() => ({
+ status: {} as Record<string, unknown>,
+}));
+
+vi.mock("../../../lib/fetch", () => ({
+ robustFetch: vi.fn(async () => fetchFake.status),
+}));
+
+vi.mock("../../../../../lib/gcs-jobs", () => ({
+ getDocFromGCS: vi.fn(async () => null),
+}));
+
+import { AddFeatureError, SiteRestrictionError } from "../../../error";
+import { fireEngineCheckStatus } from "../checkStatus";
+import { fireEngineScrape } from "../scrape";
+
+const fakeLogger = {
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ child() {
+ return this;
+ },
+} as any;
+
+const request = {
+ url: "https://example.com",
+ engine: "chrome-cdp",
+ timeout: 30000,
+} as any;
+
+const siteProtectionFailure = {
+ error: "Site protection detected",
+ failureReason: "site_protection",
+ retryWithStealth: true,
+};
+
+function fakeMeta(disableSiteHandling?: boolean) {
+ return {
+ options: { proxy: "auto" },
+ featureFlags: new Set<string>(),
+ internalOptions: {
+ ...(disableSiteHandling === undefined
+ ? {}
+ : { safeMode: { disableSiteHandling } }),
+ },
+ } as any;
+}
+
+describe.each([
+ [
+ "POST response",
+ (meta: any) =>
+ fireEngineScrape(
+ meta,
+ fakeLogger,
+ request,
+ null,
+ undefined,
+ "http://fire-engine.test",
+ ),
+ siteProtectionFailure,
+ ],
+ [
+ "polled response",
+ (meta: any) =>
+ fireEngineCheckStatus(
+ meta,
+ fakeLogger,
+ "job-id",
+ null,
+ undefined,
+ "http://fire-engine.test",
+ ),
+ {
+ ...siteProtectionFailure,
+ jobId: "job-id",
+ state: "failed",
+ processing: false,
+ },
+ ],
+])("site protection in a %s", (_, run, status) => {
+ beforeEach(() => {
+ fetchFake.status = status;
+ });
+
+ it("uses the normal retry path when Safe Mode is absent", async () => {
+ await expect(run(fakeMeta())).rejects.toBeInstanceOf(AddFeatureError);
+ });
+
+ it("uses the normal retry path when Safe Mode allows site handling", async () => {
+ await expect(run(fakeMeta(false))).rejects.toBeInstanceOf(AddFeatureError);
+ });
+
+ it("returns the terminal restriction error when Safe Mode disables site handling", async () => {
+ await expect(run(fakeMeta(true))).rejects.toBeInstanceOf(
+ SiteRestrictionError,
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index c262c0504..e96279522 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -192,7 +192,10 @@ export async function fireEngineCheckStatus(
throw new StillProcessingError(jobId);
} else if (failedParse.success) {
logger.debug("Scrape job failed", { status, jobId });
- if (failedParse.data.failureReason === "site_protection") {
+ if (
+ failedParse.data.failureReason === "site_protection" &&
+ meta.internalOptions.safeMode?.disableSiteHandling
+ ) {
throw new SiteRestrictionError();
}
if (
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 339ed97fa..f465de8a3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -269,7 +269,10 @@ export async function fireEngineScrape<
logger.debug("Scrape job failed", {
status,
});
- if (failedParse.data.failureReason === "site_protection") {
+ if (
+ failedParse.data.failureReason === "site_protection" &&
+ meta.internalOptions.safeMode?.disableSiteHandling
+ ) {
throw new SiteRestrictionError();
}
if (