Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(lockdown): gate remaining outbound paths on lockdown mode (#3414)

Abimael Martell142b3ebced70
diff --git a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
index 37acc7c56..f9b8672cf 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
@@ -69,4 +69,36 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {
},
scrapeTimeout,
);
+
+ test(
+ "should serve cache and skip audio fetch even when audio format is requested",
+ async () => {
+ const id = crypto.randomUUID();
+ const url = "https://firecrawl.dev/?testAudioGate=" + id;
+
+ const seed = await scrape({ url }, identity);
+ expect(seed).toBeDefined();
+ expect(seed.metadata.cacheState).toBe("miss");
+
+ await new Promise(resolve => setTimeout(resolve, 20000));
+
+ // Without the lockdown audio gate this would either throw
+ // AudioUnsupportedUrlError (firecrawl.dev is not an audio source) or
+ // POST to AVGRAB_SERVICE_URL with the target URL. Success here implies
+ // the gate short-circuited before any outbound call.
+ const data = await scrape(
+ {
+ url,
+ lockdown: true,
+ formats: ["markdown", "audio"],
+ },
+ identity,
+ );
+
+ expect(data).toBeDefined();
+ expect(data.metadata.cacheState).toBe("hit");
+ expect(data.audio).toBeUndefined();
+ },
+ scrapeTimeout * 2 + 20000,
+ );
});
diff --git a/apps/api/src/scraper/scrapeURL/__tests__/shouldCheckRobots.test.ts b/apps/api/src/scraper/scrapeURL/__tests__/shouldCheckRobots.test.ts
new file mode 100644
index 000000000..710e21112
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/__tests__/shouldCheckRobots.test.ts
@@ -0,0 +1,37 @@
+import { shouldCheckRobots } from "../shouldCheckRobots";
+
+describe("shouldCheckRobots", () => {
+ it("returns false when lockdown is true, even if the team flag is set", () => {
+ expect(
+ shouldCheckRobots(
+ { lockdown: true },
+ { teamFlags: { checkRobotsOnScrape: true } as any },
+ ),
+ ).toBe(false);
+ });
+
+ it("returns false when the team flag is off, regardless of lockdown", () => {
+ expect(
+ shouldCheckRobots(
+ { lockdown: false },
+ { teamFlags: { checkRobotsOnScrape: false } as any },
+ ),
+ ).toBe(false);
+ expect(
+ shouldCheckRobots({ lockdown: true }, { teamFlags: {} as any }),
+ ).toBe(false);
+ });
+
+ it("returns false when teamFlags is missing", () => {
+ expect(shouldCheckRobots({ lockdown: false }, {})).toBe(false);
+ });
+
+ it("returns true only when the team flag is on and lockdown is off", () => {
+ expect(
+ shouldCheckRobots(
+ { lockdown: false },
+ { teamFlags: { checkRobotsOnScrape: true } as any },
+ ),
+ ).toBe(true);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index b44b49c4c..d79c2dc63 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -57,6 +57,7 @@ import { ScrapeRetryTracker } from "./retryTracker";
import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
import { urlSpecificParams } from "./lib/urlSpecificParams";
+import { shouldCheckRobots } from "./shouldCheckRobots";
import { loadMock, MockState } from "./lib/mock";
import { CostTracking } from "../../lib/cost-tracking";
import { getEngineForUrl } from "../WebScraper/utils/engine-forcing";
@@ -1052,7 +1053,7 @@ export async function scrapeURL(
});
}
- if (internalOptions.teamFlags?.checkRobotsOnScrape) {
+ if (shouldCheckRobots(options, internalOptions)) {
await withSpan("scrape.robots_check", async robotsSpan => {
const urlToCheck = meta.rewrittenUrl || meta.url;
meta.logger.info("Checking robots.txt", { url: urlToCheck });
diff --git a/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts
new file mode 100644
index 000000000..d2bc039f3
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts
@@ -0,0 +1,15 @@
+import type { TeamFlags } from "../../controllers/v1/types";
+
+// The invariant "lockdown never fetches robots.txt" is load-bearing for the
+// lockdown guarantee (robots.txt is a request to the target domain). Keep this
+// in its own file so it can be unit-tested without dragging in scrapeURL's
+// ESM-heavy module graph.
+export function shouldCheckRobots(
+ options: { lockdown?: boolean },
+ internalOptions: { teamFlags?: TeamFlags },
+): boolean {
+ if (options.lockdown) {
+ return false;
+ }
+ return !!internalOptions.teamFlags?.checkRobotsOnScrape;
+}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts
new file mode 100644
index 000000000..8ad8fac45
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts
@@ -0,0 +1,50 @@
+import { fetchAudio } from "../audio";
+
+describe("fetchAudio lockdown guard", () => {
+ const originalFetch = global.fetch;
+
+ afterEach(() => {
+ global.fetch = originalFetch;
+ jest.clearAllMocks();
+ });
+
+ it("does not issue any fetch when lockdown is true, even if audio format is requested", async () => {
+ const fetchSpy = jest.fn();
+ global.fetch = fetchSpy as any;
+
+ const meta: any = {
+ url: "https://example.com/audio",
+ options: {
+ lockdown: true,
+ formats: [{ type: "audio" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = { markdown: "cached" };
+
+ const result = await fetchAudio(meta, document);
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ expect(result).toBe(document);
+ expect(result.audio).toBeUndefined();
+ });
+
+ it("returns early when audio format is not requested regardless of lockdown", async () => {
+ const fetchSpy = jest.fn();
+ global.fetch = fetchSpy as any;
+
+ const meta: any = {
+ url: "https://example.com/audio",
+ options: {
+ lockdown: false,
+ formats: [{ type: "markdown" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = { markdown: "cached" };
+
+ await fetchAudio(meta, document);
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/sendToSearchIndex.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/sendToSearchIndex.test.ts
new file mode 100644
index 000000000..5456e20a6
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/sendToSearchIndex.test.ts
@@ -0,0 +1,86 @@
+import { sendDocumentToSearchIndex } from "../sendToSearchIndex";
+import { indexDocumentIfEnabled } from "../../../../lib/search-index-client";
+import { config } from "../../../../config";
+
+jest.mock("../../../../lib/search-index-client", () => ({
+ indexDocumentIfEnabled: jest.fn(),
+}));
+
+const mockedIndex = indexDocumentIfEnabled as jest.MockedFunction<
+ typeof indexDocumentIfEnabled
+>;
+
+describe("sendDocumentToSearchIndex lockdown guard", () => {
+ const originalEnabled = config.ENABLE_SEARCH_INDEX;
+ const originalServiceUrl = config.SEARCH_SERVICE_URL;
+ const originalSampleRate = config.SEARCH_INDEX_SAMPLE_RATE;
+
+ beforeEach(() => {
+ // Force the transformer past the "enabled" and "sampling" early-returns so
+ // only the lockdown / should-index checks can stop it.
+ (config as any).ENABLE_SEARCH_INDEX = true;
+ (config as any).SEARCH_SERVICE_URL = "https://search.internal";
+ (config as any).SEARCH_INDEX_SAMPLE_RATE = 1;
+ mockedIndex.mockClear();
+ });
+
+ afterAll(() => {
+ (config as any).ENABLE_SEARCH_INDEX = originalEnabled;
+ (config as any).SEARCH_SERVICE_URL = originalServiceUrl;
+ (config as any).SEARCH_INDEX_SAMPLE_RATE = originalSampleRate;
+ });
+
+ const baseDocument: any = {
+ markdown: "a".repeat(500),
+ rawHtml: "<html><body>x</body></html>",
+ metadata: {
+ statusCode: 200,
+ indexId: "idx-123",
+ },
+ };
+
+ const baseMeta = (overrides: Partial<Record<string, unknown>> = {}) =>
+ ({
+ url: "https://example.com/page",
+ options: {
+ lockdown: false,
+ headers: undefined,
+ mobile: false,
+ },
+ internalOptions: {
+ isParse: false,
+ zeroDataRetention: false,
+ },
+ logger: {
+ warn: jest.fn(),
+ info: jest.fn(),
+ debug: jest.fn(),
+ error: jest.fn(),
+ },
+ ...overrides,
+ }) as any;
+
+ it("does not forward the URL to the search service when lockdown is true", async () => {
+ const meta = baseMeta({
+ options: { lockdown: true, headers: undefined, mobile: false },
+ });
+
+ await sendDocumentToSearchIndex(meta, { ...baseDocument });
+
+ // Fire-and-forget promise: give it a tick to settle.
+ await new Promise(r => setImmediate(r));
+
+ expect(mockedIndex).not.toHaveBeenCalled();
+ });
+
+ it("still forwards normal (non-lockdown) documents to the search service", async () => {
+ const meta = baseMeta();
+
+ await sendDocumentToSearchIndex(meta, { ...baseDocument });
+
+ await new Promise(r => setImmediate(r));
+
+ expect(mockedIndex).toHaveBeenCalledTimes(1);
+ expect(mockedIndex.mock.calls[0][0].url).toBe("https://example.com/page");
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
index 553c00cd3..6076bd178 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
@@ -42,6 +42,12 @@ export async function fetchAudio(
return document;
}
+ // Lockdown forbids any outbound request touching the target URL. The avgrab
+ // service fetches the source on our behalf, so we must skip it here.
+ if (meta.options.lockdown) {
+ return document;
+ }
+
if (!config.AVGRAB_SERVICE_URL) {
meta.logger.warn("AVGRAB_SERVICE_URL is not configured");
document.warning =
diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
index d02dc0e10..7b350ea42 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
@@ -27,6 +27,11 @@ function shouldIndexForSearch(meta: Meta, document: Document): boolean {
return false;
}
+ // Lockdown must not forward the target URL to any external service.
+ if (meta.options.lockdown) {
+ return false;
+ }
+
const statusCode = document.metadata.statusCode;
if (statusCode < 200 || statusCode >= 300) {
return false;