Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): scope site restriction errors to Safe Mode (#4648)
Gergő Móriczef6f09726a7d
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/siteProtection.test.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/siteProtection.test.tsnew file mode 100644index 000000000..97c7ef957--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/__tests__/siteProtection.test.ts@@ -0,0 +1,103 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const fetchFake = vi.hoisted(() => ({+ status: {} as Record<string, unknown>,+}));++vi.mock("../../../lib/fetch", () => ({+ robustFetch: vi.fn(async () => fetchFake.status),+}));++vi.mock("../../../../../lib/gcs-jobs", () => ({+ getDocFromGCS: vi.fn(async () => null),+}));++import { AddFeatureError, SiteRestrictionError } from "../../../error";+import { fireEngineCheckStatus } from "../checkStatus";+import { fireEngineScrape } from "../scrape";++const fakeLogger = {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ child() {+ return this;+ },+} as any;++const request = {+ url: "https://example.com",+ engine: "chrome-cdp",+ timeout: 30000,+} as any;++const siteProtectionFailure = {+ error: "Site protection detected",+ failureReason: "site_protection",+ retryWithStealth: true,+};++function fakeMeta(disableSiteHandling?: boolean) {+ return {+ options: { proxy: "auto" },+ featureFlags: new Set<string>(),+ internalOptions: {+ ...(disableSiteHandling === undefined+ ? {}+ : { safeMode: { disableSiteHandling } }),+ },+ } as any;+}++describe.each([+ [+ "POST response",+ (meta: any) =>+ fireEngineScrape(+ meta,+ fakeLogger,+ request,+ null,+ undefined,+ "http://fire-engine.test",+ ),+ siteProtectionFailure,+ ],+ [+ "polled response",+ (meta: any) =>+ fireEngineCheckStatus(+ meta,+ fakeLogger,+ "job-id",+ null,+ undefined,+ "http://fire-engine.test",+ ),+ {+ ...siteProtectionFailure,+ jobId: "job-id",+ state: "failed",+ processing: false,+ },+ ],+])("site protection in a %s", (_, run, status) => {+ beforeEach(() => {+ fetchFake.status = status;+ });++ it("uses the normal retry path when Safe Mode is absent", async () => {+ await expect(run(fakeMeta())).rejects.toBeInstanceOf(AddFeatureError);+ });++ it("uses the normal retry path when Safe Mode allows site handling", async () => {+ await expect(run(fakeMeta(false))).rejects.toBeInstanceOf(AddFeatureError);+ });++ it("returns the terminal restriction error when Safe Mode disables site handling", async () => {+ await expect(run(fakeMeta(true))).rejects.toBeInstanceOf(+ SiteRestrictionError,+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex c262c0504..e96279522 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -192,7 +192,10 @@ export async function fireEngineCheckStatus(throw new StillProcessingError(jobId);} else if (failedParse.success) {logger.debug("Scrape job failed", { status, jobId });- if (failedParse.data.failureReason === "site_protection") {+ if (+ failedParse.data.failureReason === "site_protection" &&+ meta.internalOptions.safeMode?.disableSiteHandling+ ) {throw new SiteRestrictionError();}if (diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 339ed97fa..f465de8a3 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -269,7 +269,10 @@ export async function fireEngineScrape<logger.debug("Scrape job failed", {status,});- if (failedParse.data.failureReason === "site_protection") {+ if (+ failedParse.data.failureReason === "site_protection" &&+ meta.internalOptions.safeMode?.disableSiteHandling+ ) {throw new SiteRestrictionError();}if (