Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
test(api): stop extract snips at the extract sunset and check the 410 (#5688)
Gergő Móricz902a083b9532
diff --git a/apps/api/src/__tests__/snips/v1/billing.test.ts b/apps/api/src/__tests__/snips/v1/billing.test.tsindex 400f332dc..2f7486602 100644--- a/apps/api/src/__tests__/snips/v1/billing.test.ts+++ b/apps/api/src/__tests__/snips/v1/billing.test.ts@@ -1,9 +1,10 @@-import { TEST_PRODUCTION, TEST_SUITE_WEBSITE } from "../lib";+import { concurrentIf, TEST_PRODUCTION, TEST_SUITE_WEBSITE } from "../lib";import {batchScrape,crawl,creditUsage,describeV1,+ EXTRACT_LIVE,extract,idmux,map,@@ -350,7 +351,7 @@ describeV1(TEST_PRODUCTION)("Billing tests", () => {600000,);- it.concurrent(+ concurrentIf(EXTRACT_LIVE)("bills extract correctly",async () => {const identity = await idmux({diff --git a/apps/api/src/__tests__/snips/v1/extract.test.ts b/apps/api/src/__tests__/snips/v1/extract.test.tsindex 36bce2998..abdb8f001 100644--- a/apps/api/src/__tests__/snips/v1/extract.test.ts+++ b/apps/api/src/__tests__/snips/v1/extract.test.ts@@ -1,11 +1,20 @@+import request from "supertest";import {ALLOW_TEST_SUITE_WEBSITE,- describeIf,HAS_AI,+ itIf,+ TEST_API_URL,TEST_PRODUCTION,TEST_SUITE_WEBSITE,} from "../lib";-import { extract, idmux, Identity } from "./lib";+import {+ describeExtract,+ EXTRACT_LIVE,+ EXTRACT_SUNSET,+ extract,+ idmux,+ Identity,+} from "./lib";let identity: Identity;@@ -17,7 +26,7 @@ beforeAll(async () => {});}, 10000);-describeIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(+describeExtract(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))("Extract tests",() => {it.concurrent(@@ -86,3 +95,23 @@ describeIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))();},);++itIf(!EXTRACT_LIVE)(+ "POST /v1/extract answers 410 after the extract sunset",+ async () => {+ const res = await request(TEST_API_URL)+ .post("/v1/extract")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({ urls: [TEST_SUITE_WEBSITE], prompt: "Extract the page title." });++ expect(res.statusCode).toBe(410);+ expect(res.body.code).toBe("ENDPOINT_REMOVED");+ expect(res.body.replacement).toBe("/v2/scrape");+ expect(res.body.documentation_url).toBe(+ "https://docs.firecrawl.dev/migrating-from-extract",+ );+ expect(res.headers["sunset"]).toBe(new Date(EXTRACT_SUNSET).toUTCString());+ },+ 30000,+);diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.tsindex affc0d965..7bbb46099 100644--- a/apps/api/src/__tests__/snips/v1/lib.ts+++ b/apps/api/src/__tests__/snips/v1/lib.ts@@ -33,6 +33,12 @@ export { scrapeTimeout, indexCooldown, Identity, idmux };export const V1_LIVE = Date.now() < sunsetOf("v1_scrape");export const describeV1 = (cond = true) => describeIf(V1_LIVE && cond);+// From the extract sunset /v1/extract answers 410 instead of extracting.+export const EXTRACT_SUNSET = sunsetOf("v1_extract");+export const EXTRACT_LIVE = Date.now() < EXTRACT_SUNSET;+export const describeExtract = (cond = true) =>+ describeIf(EXTRACT_LIVE && cond);+// Deep research and llms.txt answer 410 from their own sunsets.export const DEEP_RESEARCH_LIVE = Date.now() < sunsetOf("v1_deep_research");export const LLMSTXT_LIVE = Date.now() < sunsetOf("v1_llmstxt");diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.tsindex 307aecb74..ebd997508 100644--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts@@ -24,6 +24,7 @@ import {scrapeRaw,extract,describeV1,+ EXTRACT_LIVE,V1_LIVE,} from "./lib";import crypto from "crypto";@@ -1626,7 +1627,9 @@ describeV1()("Scrape tests", () => {scrapeTimeout,);- itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(+ itIf(+ EXTRACT_LIVE && (TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE)),+ )("should normalize extract request with additionalProperties in schema",async () => {const identity = await idmux({ name: "schema-validation-test" });diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex e03081ae9..3360b5126 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -27,11 +27,15 @@ import {idmux,} from "../lib";import { SearchV2Response } from "../../../lib/entities";+import { sunsetOf } from "../../../lib/deprecations";// Re-export shared utilities for backwards compatibilityexport { scrapeTimeout, indexCooldown, Identity, idmux, TEST_API_URL };export default request;+// From the extract sunset /v2/extract answers 410 instead of extracting.+export const EXTRACT_LIVE = Date.now() < sunsetOf("v2_extract");+const pollSleep = async () => new Promise(r => setTimeout(r, 50));// =========================================diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 3990910f7..9ef8c4ce5 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -23,6 +23,7 @@ import {Identity,scrapeRaw,extractRaw,+ EXTRACT_LIVE,creditUsage,TEST_API_URL,} from "./lib";@@ -2226,7 +2227,9 @@ describe("Attribute formats", () => {scrapeTimeout,);- itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(+ itIf(+ EXTRACT_LIVE && (TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE)),+ )("should normalize extract request with additionalProperties in schema",async () => {const identity = await idmux({ name: "schema-validation-test" });diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.tsindex f43c7ab42..e93a5c125 100644--- a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts+++ b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts@@ -4,6 +4,7 @@ import {describeIf,idmux,Identity,+ itIf,TEST_API_URL,TEST_PRODUCTION,TEST_SUITE_WEBSITE,@@ -13,6 +14,7 @@ import {crawl,crawlStart,creditUsage,+ EXTRACT_LIVE,map,scrape,scrapeRaw,@@ -416,7 +418,7 @@ describeIf(TEST_PRODUCTION)("Threat protection enforcement", () => {scrapeTimeout,);- it(+ itIf(EXTRACT_LIVE)("extract: blocked target URLs are rejected before fetching",async () => {const res = await request(TEST_API_URL)diff --git a/apps/api/src/__tests__/snips/v2/us-only.test.ts b/apps/api/src/__tests__/snips/v2/us-only.test.tsindex 88c80ed34..05c9fef78 100644--- a/apps/api/src/__tests__/snips/v2/us-only.test.ts+++ b/apps/api/src/__tests__/snips/v2/us-only.test.ts@@ -9,6 +9,7 @@ import {Identity,crawlStart,extractRaw,+ EXTRACT_LIVE,map,browserCreateRaw,} from "./lib";@@ -182,7 +183,7 @@ describeIf(TEST_PRODUCTION)("US-only teams", () => {scrapeTimeout,);- itIf(!HAS_US_OPENAI)(+ itIf(EXTRACT_LIVE && !HAS_US_OPENAI)("refuses extract without a US route",async () => {const response = await extractRaw(