Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(lockdown): gate remaining outbound paths on lockdown mode (#3414)
Abimael Martell142b3ebced70
diff --git a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.tsindex 37acc7c56..f9b8672cf 100644--- a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts@@ -69,4 +69,36 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {},scrapeTimeout,);++ test(+ "should serve cache and skip audio fetch even when audio format is requested",+ async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testAudioGate=" + id;++ const seed = await scrape({ url }, identity);+ expect(seed).toBeDefined();+ expect(seed.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, 20000));++ // Without the lockdown audio gate this would either throw+ // AudioUnsupportedUrlError (firecrawl.dev is not an audio source) or+ // POST to AVGRAB_SERVICE_URL with the target URL. Success here implies+ // the gate short-circuited before any outbound call.+ const data = await scrape(+ {+ url,+ lockdown: true,+ formats: ["markdown", "audio"],+ },+ identity,+ );++ expect(data).toBeDefined();+ expect(data.metadata.cacheState).toBe("hit");+ expect(data.audio).toBeUndefined();+ },+ scrapeTimeout * 2 + 20000,+ );});diff --git a/apps/api/src/scraper/scrapeURL/__tests__/shouldCheckRobots.test.ts b/apps/api/src/scraper/scrapeURL/__tests__/shouldCheckRobots.test.tsnew file mode 100644index 000000000..710e21112--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/__tests__/shouldCheckRobots.test.ts@@ -0,0 +1,37 @@+import { shouldCheckRobots } from "../shouldCheckRobots";++describe("shouldCheckRobots", () => {+ it("returns false when lockdown is true, even if the team flag is set", () => {+ expect(+ shouldCheckRobots(+ { lockdown: true },+ { teamFlags: { checkRobotsOnScrape: true } as any },+ ),+ ).toBe(false);+ });++ it("returns false when the team flag is off, regardless of lockdown", () => {+ expect(+ shouldCheckRobots(+ { lockdown: false },+ { teamFlags: { checkRobotsOnScrape: false } as any },+ ),+ ).toBe(false);+ expect(+ shouldCheckRobots({ lockdown: true }, { teamFlags: {} as any }),+ ).toBe(false);+ });++ it("returns false when teamFlags is missing", () => {+ expect(shouldCheckRobots({ lockdown: false }, {})).toBe(false);+ });++ it("returns true only when the team flag is on and lockdown is off", () => {+ expect(+ shouldCheckRobots(+ { lockdown: false },+ { teamFlags: { checkRobotsOnScrape: true } as any },+ ),+ ).toBe(true);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex b44b49c4c..d79c2dc63 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -57,6 +57,7 @@ import { ScrapeRetryTracker } from "./retryTracker";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";import { urlSpecificParams } from "./lib/urlSpecificParams";+import { shouldCheckRobots } from "./shouldCheckRobots";import { loadMock, MockState } from "./lib/mock";import { CostTracking } from "../../lib/cost-tracking";import { getEngineForUrl } from "../WebScraper/utils/engine-forcing";@@ -1052,7 +1053,7 @@ export async function scrapeURL(});}- if (internalOptions.teamFlags?.checkRobotsOnScrape) {+ if (shouldCheckRobots(options, internalOptions)) {await withSpan("scrape.robots_check", async robotsSpan => {const urlToCheck = meta.rewrittenUrl || meta.url;meta.logger.info("Checking robots.txt", { url: urlToCheck });diff --git a/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.tsnew file mode 100644index 000000000..d2bc039f3--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts@@ -0,0 +1,15 @@+import type { TeamFlags } from "../../controllers/v1/types";++// The invariant "lockdown never fetches robots.txt" is load-bearing for the+// lockdown guarantee (robots.txt is a request to the target domain). Keep this+// in its own file so it can be unit-tested without dragging in scrapeURL's+// ESM-heavy module graph.+export function shouldCheckRobots(+ options: { lockdown?: boolean },+ internalOptions: { teamFlags?: TeamFlags },+): boolean {+ if (options.lockdown) {+ return false;+ }+ return !!internalOptions.teamFlags?.checkRobotsOnScrape;+}diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.tsnew file mode 100644index 000000000..8ad8fac45--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts@@ -0,0 +1,50 @@+import { fetchAudio } from "../audio";++describe("fetchAudio lockdown guard", () => {+ const originalFetch = global.fetch;++ afterEach(() => {+ global.fetch = originalFetch;+ jest.clearAllMocks();+ });++ it("does not issue any fetch when lockdown is true, even if audio format is requested", async () => {+ const fetchSpy = jest.fn();+ global.fetch = fetchSpy as any;++ const meta: any = {+ url: "https://example.com/audio",+ options: {+ lockdown: true,+ formats: [{ type: "audio" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = { markdown: "cached" };++ const result = await fetchAudio(meta, document);++ expect(fetchSpy).not.toHaveBeenCalled();+ expect(result).toBe(document);+ expect(result.audio).toBeUndefined();+ });++ it("returns early when audio format is not requested regardless of lockdown", async () => {+ const fetchSpy = jest.fn();+ global.fetch = fetchSpy as any;++ const meta: any = {+ url: "https://example.com/audio",+ options: {+ lockdown: false,+ formats: [{ type: "markdown" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = { markdown: "cached" };++ await fetchAudio(meta, document);++ expect(fetchSpy).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/sendToSearchIndex.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/sendToSearchIndex.test.tsnew file mode 100644index 000000000..5456e20a6--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/sendToSearchIndex.test.ts@@ -0,0 +1,86 @@+import { sendDocumentToSearchIndex } from "../sendToSearchIndex";+import { indexDocumentIfEnabled } from "../../../../lib/search-index-client";+import { config } from "../../../../config";++jest.mock("../../../../lib/search-index-client", () => ({+ indexDocumentIfEnabled: jest.fn(),+}));++const mockedIndex = indexDocumentIfEnabled as jest.MockedFunction<+ typeof indexDocumentIfEnabled+>;++describe("sendDocumentToSearchIndex lockdown guard", () => {+ const originalEnabled = config.ENABLE_SEARCH_INDEX;+ const originalServiceUrl = config.SEARCH_SERVICE_URL;+ const originalSampleRate = config.SEARCH_INDEX_SAMPLE_RATE;++ beforeEach(() => {+ // Force the transformer past the "enabled" and "sampling" early-returns so+ // only the lockdown / should-index checks can stop it.+ (config as any).ENABLE_SEARCH_INDEX = true;+ (config as any).SEARCH_SERVICE_URL = "https://search.internal";+ (config as any).SEARCH_INDEX_SAMPLE_RATE = 1;+ mockedIndex.mockClear();+ });++ afterAll(() => {+ (config as any).ENABLE_SEARCH_INDEX = originalEnabled;+ (config as any).SEARCH_SERVICE_URL = originalServiceUrl;+ (config as any).SEARCH_INDEX_SAMPLE_RATE = originalSampleRate;+ });++ const baseDocument: any = {+ markdown: "a".repeat(500),+ rawHtml: "<html><body>x</body></html>",+ metadata: {+ statusCode: 200,+ indexId: "idx-123",+ },+ };++ const baseMeta = (overrides: Partial<Record<string, unknown>> = {}) =>+ ({+ url: "https://example.com/page",+ options: {+ lockdown: false,+ headers: undefined,+ mobile: false,+ },+ internalOptions: {+ isParse: false,+ zeroDataRetention: false,+ },+ logger: {+ warn: jest.fn(),+ info: jest.fn(),+ debug: jest.fn(),+ error: jest.fn(),+ },+ ...overrides,+ }) as any;++ it("does not forward the URL to the search service when lockdown is true", async () => {+ const meta = baseMeta({+ options: { lockdown: true, headers: undefined, mobile: false },+ });++ await sendDocumentToSearchIndex(meta, { ...baseDocument });++ // Fire-and-forget promise: give it a tick to settle.+ await new Promise(r => setImmediate(r));++ expect(mockedIndex).not.toHaveBeenCalled();+ });++ it("still forwards normal (non-lockdown) documents to the search service", async () => {+ const meta = baseMeta();++ await sendDocumentToSearchIndex(meta, { ...baseDocument });++ await new Promise(r => setImmediate(r));++ expect(mockedIndex).toHaveBeenCalledTimes(1);+ expect(mockedIndex.mock.calls[0][0].url).toBe("https://example.com/page");+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.tsindex 553c00cd3..6076bd178 100644--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts@@ -42,6 +42,12 @@ export async function fetchAudio(return document;}+ // Lockdown forbids any outbound request touching the target URL. The avgrab+ // service fetches the source on our behalf, so we must skip it here.+ if (meta.options.lockdown) {+ return document;+ }+if (!config.AVGRAB_SERVICE_URL) {meta.logger.warn("AVGRAB_SERVICE_URL is not configured");document.warning =diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.tsindex d02dc0e10..7b350ea42 100644--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts@@ -27,6 +27,11 @@ function shouldIndexForSearch(meta: Meta, document: Document): boolean {return false;}+ // Lockdown must not forward the target URL to any external service.+ if (meta.options.lockdown) {+ return false;+ }+const statusCode = document.metadata.statusCode;if (statusCode < 200 || statusCode >= 300) {return false;