Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(scrapeURL): remove the stealth-skip and 429 waterfall experiments (#5565)
rowinsg4591e46a8e09
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex a76c17016..80cbb929a 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -79,7 +79,6 @@ export class NoEnginesLeftError extends TransportableError {export class AddFeatureError extends Error {public featureFlags: FeatureFlag[];- public statusCode?: number;public pdfPrefetch: Meta["pdfPrefetch"];public documentPrefetch: Meta["documentPrefetch"];public imagePrefetch: Meta["imagePrefetch"];diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 1595b32dc..9a60d3525 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -800,9 +800,7 @@ async function scrapeURLLoopIter(length: engineResult.html?.trim().length ?? 0,},);- throw Object.assign(new AddFeatureError(["stealthProxy"]), {- statusCode: engineResult.statusCode,- });+ throw new AddFeatureError(["stealthProxy"]);}// NOTE: TODO: what to do when status code is bad is tough...@@ -1009,11 +1007,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error: error.error,});if (testArm) {- if (- experiment === "skip-enhanced-retry-after-site-protection"- ) {- remainingEngines.shift();- }enginePromises = enginePromises.filter(x => x.engine !== error.engine,);diff --git a/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts b/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.tsindex 8693904ec..657939f34 100644--- a/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts+++ b/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts@@ -1,4 +1,4 @@-import { AddFeatureError, EngineError, SiteError } from "./error";+import { SiteError } from "./error";import {inWaterfallExperimentArm,waterfallExperimentFor,@@ -32,81 +32,18 @@ describe("waterfallExperimentFor", () => {),).toBeUndefined();});-- it("takes a site_protection failure on the first enhanced attempt", () => {- const blocked = new EngineError("Scrape job failed", {- cause: { status: { failureReason: "site_protection" } },- });- expect(- waterfallExperimentFor(- "fire-engine;chrome-cdp;stealth",- blocked,- "fire-engine(retry);chrome-cdp;stealth",- ),- ).toBe("skip-enhanced-retry-after-site-protection");- expect(- waterfallExperimentFor(- "fire-engine;chrome-cdp;stealth",- new EngineError("Scrape job failed", { cause: { status: {} } }),- "fire-engine(retry);chrome-cdp;stealth",- ),- ).toBeUndefined();- });-- it("takes a 429 escalation on the first basic attempt only", () => {- const escalation = (statusCode: number) =>- Object.assign(new AddFeatureError(["stealthProxy"]), { statusCode });- expect(- waterfallExperimentFor(- "fire-engine;chrome-cdp",- escalation(429),- "fire-engine(retry);chrome-cdp",- ),- ).toBe("basic-retry-on-429");- expect(- waterfallExperimentFor(- "fire-engine;chrome-cdp",- escalation(403),- "fire-engine(retry);chrome-cdp",- ),- ).toBeUndefined();- expect(- waterfallExperimentFor(- "fire-engine(retry);chrome-cdp",- escalation(429),- "fire-engine;chrome-cdp;stealth",- ),- ).toBeUndefined();- });});describe("inWaterfallExperimentArm", () => {it("is stable per scrape and close to the rollout", () => {const ids = Array.from({ length: 4000 }, (_, i) => `scrape-${i}`);- const share = (- experiment: Parameters<typeof inWaterfallExperimentArm>[1],- ) =>- ids.filter(id => inWaterfallExperimentArm(id, experiment)).length /- ids.length;+ const share =+ ids.filter(id => inWaterfallExperimentArm(id, "tunnel-error-retry"))+ .length / ids.length;expect(inWaterfallExperimentArm("scrape-1", "tunnel-error-retry")).toBe(inWaterfallExperimentArm("scrape-1", "tunnel-error-retry"),);- expect(share("tunnel-error-retry")).toBeGreaterThan(0.45);- expect(share("tunnel-error-retry")).toBeLessThan(0.55);- expect(share("skip-enhanced-retry-after-site-protection")).toBeGreaterThan(- 0.16,- );- expect(share("skip-enhanced-retry-after-site-protection")).toBeLessThan(- 0.24,- );- expect(share("basic-retry-on-429")).toBeGreaterThan(0.45);- expect(share("basic-retry-on-429")).toBeLessThan(0.55);- const both = ids.filter(- id =>- inWaterfallExperimentArm(id, "tunnel-error-retry") &&- inWaterfallExperimentArm(id, "basic-retry-on-429"),- ).length;- expect(both / ids.length).toBeGreaterThan(0.2);- expect(both / ids.length).toBeLessThan(0.3);+ expect(share).toBeGreaterThan(0.45);+ expect(share).toBeLessThan(0.55);});});diff --git a/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts b/apps/api/src/scraper/scrapeURL/waterfallExperiments.tsindex b546fa288..0e4a9081c 100644--- a/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts+++ b/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts@@ -1,11 +1,9 @@import { createHash } from "node:crypto";import type { Engine } from "./engines";-import { AddFeatureError, EngineError, SiteError } from "./error";+import { SiteError } from "./error";const ROLLOUTS = {"tunnel-error-retry": 0.5,- "skip-enhanced-retry-after-site-protection": 0.2,- "basic-retry-on-429": 0.5,} as const;export type WaterfallExperiment = keyof typeof ROLLOUTS;@@ -25,23 +23,6 @@ export function waterfallExperimentFor() {return "tunnel-error-retry";}- if (- engine === "fire-engine;chrome-cdp;stealth" &&- nextEngine === "fire-engine(retry);chrome-cdp;stealth" &&- error instanceof EngineError &&- (error.cause as { status?: { failureReason?: string } } | undefined)?.status- ?.failureReason === "site_protection"- ) {- return "skip-enhanced-retry-after-site-protection";- }- if (- engine === "fire-engine;chrome-cdp" &&- nextEngine === "fire-engine(retry);chrome-cdp" &&- error instanceof AddFeatureError &&- error.statusCode === 429- ) {- return "basic-retry-on-429";- }return undefined;}