Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): don't use the Wikipedia engine for zero data retention requests (#5551)

Gergő Móricz2f0ec359a691
diff --git a/apps/api/src/__tests__/snips/v2/wikipedia-zdr-routing.test.ts b/apps/api/src/__tests__/snips/v2/wikipedia-zdr-routing.test.ts
new file mode 100644
index 000000000..664d5a082
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/wikipedia-zdr-routing.test.ts
@@ -0,0 +1,118 @@
+/**
+ * Unit test for Wikipedia engine routing via buildFallbackList: zero data
+ * retention scrapes of Wikimedia URLs never use the Wikipedia engine.
+ */
+
+describe("Wikipedia engine routing under zero data retention (buildFallbackList)", () => {
+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;
+
+ const env = {
+ FIRE_ENGINE_BETA_URL: process.env.FIRE_ENGINE_BETA_URL,
+ WIKIPEDIA_ENTERPRISE_USERNAME: process.env.WIKIPEDIA_ENTERPRISE_USERNAME,
+ WIKIPEDIA_ENTERPRISE_PASSWORD: process.env.WIKIPEDIA_ENTERPRISE_PASSWORD,
+ };
+
+ beforeAll(async () => {
+ process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";
+ process.env.WIKIPEDIA_ENTERPRISE_USERNAME = "test-user";
+ process.env.WIKIPEDIA_ENTERPRISE_PASSWORD = "test-password";
+
+ // Re-import engines fresh so it reads the env vars set above at eval time.
+ vi.resetModules();
+ ({ buildFallbackList } = await import(
+ "../../../scraper/scrapeURL/engines/index.js"
+ ));
+ });
+
+ beforeEach(() => {
+ // Below 0.5, so a Wikimedia URL keeps the Wikipedia engine when allowed.
+ vi.spyOn(Math, "random").mockReturnValue(0);
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ afterAll(() => {
+ for (const [name, value] of Object.entries(env)) {
+ if (value === undefined) {
+ delete process.env[name];
+ } else {
+ process.env[name] = value;
+ }
+ }
+ });
+
+ const buildStubMeta = (
+ url: string,
+ zeroDataRetention: boolean,
+ forceEngine?: string[],
+ ) =>
+ ({
+ id: "test",
+ url,
+ options: {
+ formats: [],
+ maxAge: 0,
+ },
+ internalOptions: { teamId: "test", zeroDataRetention, forceEngine },
+ featureFlags: new Set(),
+ mock: null,
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ debug: vi.fn(),
+ error: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+ }) as any;
+
+ const enginesFor = async (url: string, zeroDataRetention: boolean) =>
+ (await buildFallbackList(buildStubMeta(url, zeroDataRetention))).map(
+ f => f.engine,
+ );
+
+ it("uses the Wikipedia engine for a Wikimedia URL without zero data retention", async () => {
+ const engines = await enginesFor(
+ "https://en.wikipedia.org/wiki/Web_scraping",
+ false,
+ );
+
+ expect(engines).toContain("wikipedia");
+ });
+
+ it("skips the Wikipedia engine for a zero data retention scrape of a Wikimedia URL", async () => {
+ const engines = await enginesFor(
+ "https://en.wikipedia.org/wiki/Web_scraping",
+ true,
+ );
+
+ expect(engines).not.toContain("wikipedia");
+ expect(engines).toContain("fire-engine;chrome-cdp");
+ });
+
+ it("skips a forced Wikipedia engine under zero data retention", async () => {
+ const meta = (zeroDataRetention: boolean) =>
+ buildStubMeta(
+ "https://en.wikipedia.org/wiki/Web_scraping",
+ zeroDataRetention,
+ ["wikipedia", "fire-engine;chrome-cdp"],
+ );
+
+ const forced = (await buildFallbackList(meta(false))).map(f => f.engine);
+ expect(forced).toContain("wikipedia");
+
+ const zdr = (await buildFallbackList(meta(true))).map(f => f.engine);
+ expect(zdr).not.toContain("wikipedia");
+ expect(zdr).toContain("fire-engine;chrome-cdp");
+ });
+
+ it("leaves non-Wikimedia URLs off the Wikipedia engine either way", async () => {
+ expect(await enginesFor("https://example.com/", false)).not.toContain(
+ "wikipedia",
+ );
+ expect(await enginesFor("https://example.com/", true)).not.toContain(
+ "wikipedia",
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index a5bb88efc..6ec6b3aa4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -990,6 +990,11 @@ export async function buildFallbackList(meta: Meta): Promise<
: _engines;
for (const engine of currentEngines) {
+ // Zero data retention scrapes, forced engines included, never use the
+ // Wikipedia engine: they stay on the engines that fetch the page directly.
+ if (engine === "wikipedia" && meta.internalOptions.zeroDataRetention)
+ continue;
+
// A profile must never fall back to fetching anonymous content. File parsers
// may consume bytes already fetched by the authenticated browser.
if (