Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

test(api): stop extract snips at the extract sunset and check the 410 (#5688)

Gergő Móricz902a083b9532
diff --git a/apps/api/src/__tests__/snips/v1/billing.test.ts b/apps/api/src/__tests__/snips/v1/billing.test.ts
index 400f332dc..2f7486602 100644
--- a/apps/api/src/__tests__/snips/v1/billing.test.ts
+++ b/apps/api/src/__tests__/snips/v1/billing.test.ts
@@ -1,9 +1,10 @@
-import { TEST_PRODUCTION, TEST_SUITE_WEBSITE } from "../lib";
+import { concurrentIf, TEST_PRODUCTION, TEST_SUITE_WEBSITE } from "../lib";
import {
batchScrape,
crawl,
creditUsage,
describeV1,
+ EXTRACT_LIVE,
extract,
idmux,
map,
@@ -350,7 +351,7 @@ describeV1(TEST_PRODUCTION)("Billing tests", () => {
600000,
);
- it.concurrent(
+ concurrentIf(EXTRACT_LIVE)(
"bills extract correctly",
async () => {
const identity = await idmux({
diff --git a/apps/api/src/__tests__/snips/v1/extract.test.ts b/apps/api/src/__tests__/snips/v1/extract.test.ts
index 36bce2998..abdb8f001 100644
--- a/apps/api/src/__tests__/snips/v1/extract.test.ts
+++ b/apps/api/src/__tests__/snips/v1/extract.test.ts
@@ -1,11 +1,20 @@
+import request from "supertest";
import {
ALLOW_TEST_SUITE_WEBSITE,
- describeIf,
HAS_AI,
+ itIf,
+ TEST_API_URL,
TEST_PRODUCTION,
TEST_SUITE_WEBSITE,
} from "../lib";
-import { extract, idmux, Identity } from "./lib";
+import {
+ describeExtract,
+ EXTRACT_LIVE,
+ EXTRACT_SUNSET,
+ extract,
+ idmux,
+ Identity,
+} from "./lib";
let identity: Identity;
@@ -17,7 +26,7 @@ beforeAll(async () => {
});
}, 10000);
-describeIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
+describeExtract(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
"Extract tests",
() => {
it.concurrent(
@@ -86,3 +95,23 @@ describeIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
);
},
);
+
+itIf(!EXTRACT_LIVE)(
+ "POST /v1/extract answers 410 after the extract sunset",
+ async () => {
+ const res = await request(TEST_API_URL)
+ .post("/v1/extract")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({ urls: [TEST_SUITE_WEBSITE], prompt: "Extract the page title." });
+
+ expect(res.statusCode).toBe(410);
+ expect(res.body.code).toBe("ENDPOINT_REMOVED");
+ expect(res.body.replacement).toBe("/v2/scrape");
+ expect(res.body.documentation_url).toBe(
+ "https://docs.firecrawl.dev/migrating-from-extract",
+ );
+ expect(res.headers["sunset"]).toBe(new Date(EXTRACT_SUNSET).toUTCString());
+ },
+ 30000,
+);
diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.ts
index affc0d965..7bbb46099 100644
--- a/apps/api/src/__tests__/snips/v1/lib.ts
+++ b/apps/api/src/__tests__/snips/v1/lib.ts
@@ -33,6 +33,12 @@ export { scrapeTimeout, indexCooldown, Identity, idmux };
export const V1_LIVE = Date.now() < sunsetOf("v1_scrape");
export const describeV1 = (cond = true) => describeIf(V1_LIVE && cond);
+// From the extract sunset /v1/extract answers 410 instead of extracting.
+export const EXTRACT_SUNSET = sunsetOf("v1_extract");
+export const EXTRACT_LIVE = Date.now() < EXTRACT_SUNSET;
+export const describeExtract = (cond = true) =>
+ describeIf(EXTRACT_LIVE && cond);
+
// Deep research and llms.txt answer 410 from their own sunsets.
export const DEEP_RESEARCH_LIVE = Date.now() < sunsetOf("v1_deep_research");
export const LLMSTXT_LIVE = Date.now() < sunsetOf("v1_llmstxt");
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.ts
index 307aecb74..ebd997508 100644
--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts
@@ -24,6 +24,7 @@ import {
scrapeRaw,
extract,
describeV1,
+ EXTRACT_LIVE,
V1_LIVE,
} from "./lib";
import crypto from "crypto";
@@ -1626,7 +1627,9 @@ describeV1()("Scrape tests", () => {
scrapeTimeout,
);
- itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
+ itIf(
+ EXTRACT_LIVE && (TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE)),
+ )(
"should normalize extract request with additionalProperties in schema",
async () => {
const identity = await idmux({ name: "schema-validation-test" });
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index e03081ae9..3360b5126 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -27,11 +27,15 @@ import {
idmux,
} from "../lib";
import { SearchV2Response } from "../../../lib/entities";
+import { sunsetOf } from "../../../lib/deprecations";
// Re-export shared utilities for backwards compatibility
export { scrapeTimeout, indexCooldown, Identity, idmux, TEST_API_URL };
export default request;
+// From the extract sunset /v2/extract answers 410 instead of extracting.
+export const EXTRACT_LIVE = Date.now() < sunsetOf("v2_extract");
+
const pollSleep = async () => new Promise(r => setTimeout(r, 50));
// =========================================
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index 3990910f7..9ef8c4ce5 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -23,6 +23,7 @@ import {
Identity,
scrapeRaw,
extractRaw,
+ EXTRACT_LIVE,
creditUsage,
TEST_API_URL,
} from "./lib";
@@ -2226,7 +2227,9 @@ describe("Attribute formats", () => {
scrapeTimeout,
);
- itIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
+ itIf(
+ EXTRACT_LIVE && (TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE)),
+ )(
"should normalize extract request with additionalProperties in schema",
async () => {
const identity = await idmux({ name: "schema-validation-test" });
diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts
index f43c7ab42..e93a5c125 100644
--- a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts
+++ b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts
@@ -4,6 +4,7 @@ import {
describeIf,
idmux,
Identity,
+ itIf,
TEST_API_URL,
TEST_PRODUCTION,
TEST_SUITE_WEBSITE,
@@ -13,6 +14,7 @@ import {
crawl,
crawlStart,
creditUsage,
+ EXTRACT_LIVE,
map,
scrape,
scrapeRaw,
@@ -416,7 +418,7 @@ describeIf(TEST_PRODUCTION)("Threat protection enforcement", () => {
scrapeTimeout,
);
- it(
+ itIf(EXTRACT_LIVE)(
"extract: blocked target URLs are rejected before fetching",
async () => {
const res = await request(TEST_API_URL)
diff --git a/apps/api/src/__tests__/snips/v2/us-only.test.ts b/apps/api/src/__tests__/snips/v2/us-only.test.ts
index 88c80ed34..05c9fef78 100644
--- a/apps/api/src/__tests__/snips/v2/us-only.test.ts
+++ b/apps/api/src/__tests__/snips/v2/us-only.test.ts
@@ -9,6 +9,7 @@ import {
Identity,
crawlStart,
extractRaw,
+ EXTRACT_LIVE,
map,
browserCreateRaw,
} from "./lib";
@@ -182,7 +183,7 @@ describeIf(TEST_PRODUCTION)("US-only teams", () => {
scrapeTimeout,
);
- itIf(!HAS_US_OPENAI)(
+ itIf(EXTRACT_LIVE && !HAS_US_OPENAI)(
"refuses extract without a US route",
async () => {
const response = await extractRaw(