Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update error message (#2793)

firecrawl-spring[bot]6d07b91fb205
diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
index 3b9bd5151..d33507eee 100644
--- a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
@@ -1,7 +1,7 @@
import request from "supertest";
import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
// const TEST_URL = 'http://localhost:3002'
const TEST_URL = "http://127.0.0.1:3002";
@@ -44,15 +44,15 @@ describe("E2E Tests for API Routes", () => {
},
);
- it.concurrent("should return an error for a blocklisted URL", async () => {
- const blocklistedUrl = "https://facebook.com/fake-test";
+ it.concurrent("should return an error for a unsupported URL", async () => {
+ const unsupportedUrl = "https://facebook.com/fake-test";
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
- .send({ url: blocklistedUrl });
+ .send({ url: unsupportedUrl });
expect(response.statusCode).toBe(403);
- expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
+ expect(response.body.error).toContain(UNSUPPORTED_SITE_MESSAGE);
});
it.concurrent(
@@ -454,15 +454,15 @@ describe("E2E Tests for API Routes", () => {
},
);
- it.concurrent("should return an error for a blocklisted URL", async () => {
- const blocklistedUrl = "https://twitter.com/fake-test";
+ it.concurrent("should return an error for a unsupported URL", async () => {
+ const unsupportedUrl = "https://twitter.com/fake-test";
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
- .send({ url: blocklistedUrl });
+ .send({ url: unsupportedUrl });
expect(response.statusCode).toBe(403);
- expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
+ expect(response.body.error).toContain(UNSUPPORTED_SITE_MESSAGE);
});
it.concurrent(
@@ -1045,13 +1045,13 @@ describe("E2E Tests for API Routes", () => {
},
);
- // it.concurrent("should return an error for a blocklisted URL", async () => {
- // const blocklistedUrl = "https://instagram.com/fake-test";
+ // it.concurrent("should return an error for a unsupported URL", async () => {
+ // const unsupportedUrl = "https://instagram.com/fake-test";
// const response = await request(TEST_URL)
// .post("/v0/crawlWebsitePreview")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
- // .send({ url: blocklistedUrl });
+ // .send({ url: unsupportedUrl });
// // is returning 429 instead of 403
// expect(response.statusCode).toBe(403);
// expect(response.body.error).toContain("Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.");
diff --git a/apps/api/src/__tests__/e2e_noAuth/index.test.ts b/apps/api/src/__tests__/e2e_noAuth/index.test.ts
index 428202a85..65bf979f0 100644
--- a/apps/api/src/__tests__/e2e_noAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_noAuth/index.test.ts
@@ -1,6 +1,6 @@
import request from "supertest";
import { config } from "../../config";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
const fs = require("fs");
const path = require("path");
@@ -52,14 +52,14 @@ describe("E2E Tests for API Routes with No Authentication", () => {
expect(response.statusCode).not.toBe(401);
});
- it("should return an error for a blocklisted URL without requiring authorization", async () => {
- const blocklistedUrl = "https://facebook.com/fake-test";
+ it("should return an error for a unsupported URL without requiring authorization", async () => {
+ const unsupportedUrl = "https://facebook.com/fake-test";
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Content-Type", "application/json")
- .send({ url: blocklistedUrl });
+ .send({ url: unsupportedUrl });
expect(response.statusCode).toBe(403);
- expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
+ expect(response.body.error).toContain(UNSUPPORTED_SITE_MESSAGE);
});
it("should return a successful response", async () => {
@@ -77,14 +77,14 @@ describe("E2E Tests for API Routes with No Authentication", () => {
expect(response.statusCode).not.toBe(401);
});
- it("should return an error for a blocklisted URL", async () => {
- const blocklistedUrl = "https://twitter.com/fake-test";
+ it("should return an error for a unsupported URL", async () => {
+ const unsupportedUrl = "https://twitter.com/fake-test";
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Content-Type", "application/json")
- .send({ url: blocklistedUrl });
+ .send({ url: unsupportedUrl });
expect(response.statusCode).toBe(403);
- expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
+ expect(response.body.error).toContain(UNSUPPORTED_SITE_MESSAGE);
});
it("should return a successful response", async () => {
@@ -106,14 +106,14 @@ describe("E2E Tests for API Routes with No Authentication", () => {
expect(response.statusCode).not.toBe(401);
});
- it("should return an error for a blocklisted URL", async () => {
- const blocklistedUrl = "https://instagram.com/fake-test";
+ it("should return an error for a unsupported URL", async () => {
+ const unsupportedUrl = "https://instagram.com/fake-test";
const response = await request(TEST_URL)
.post("/v0/crawlWebsitePreview")
.set("Content-Type", "application/json")
- .send({ url: blocklistedUrl });
+ .send({ url: unsupportedUrl });
expect(response.statusCode).toBe(403);
- expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
+ expect(response.body.error).toContain(UNSUPPORTED_SITE_MESSAGE);
});
it("should return a successful response", async () => {
diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
index ffb3eab49..4d1011434 100644
--- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
@@ -2,7 +2,7 @@ import request from "supertest";
import { config } from "../../config";
import { configDotenv } from "dotenv";
import { ScrapeRequestInput } from "../../controllers/v1/types";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
configDotenv();
const TEST_URL = "http://127.0.0.1:3002";
@@ -40,7 +40,7 @@ describe("E2E Tests for v1 API Routes", () => {
expect(response.statusCode).toBe(401);
});
- it.concurrent("should throw error for blocklisted URL", async () => {
+ it.concurrent("should throw error for unsupported URL", async () => {
const scrapeRequest: ScrapeRequestInput = {
url: "https://facebook.com/fake-test",
};
@@ -52,7 +52,7 @@ describe("E2E Tests for v1 API Routes", () => {
.send(scrapeRequest);
expect(response.statusCode).toBe(403);
- expect(response.body.error).toBe(BLOCKLISTED_URL_MESSAGE);
+ expect(response.body.error).toBe(UNSUPPORTED_SITE_MESSAGE);
});
it.concurrent(
@@ -736,7 +736,7 @@ describe("E2E Tests for v1 API Routes", () => {
expect(response.statusCode).toBe(401);
});
- it.concurrent("should throw error for blocklisted URL", async () => {
+ it.concurrent("should throw error for unsupported URL", async () => {
const scrapeRequest: ScrapeRequestInput = {
url: "https://facebook.com/fake-test",
};
@@ -748,7 +748,7 @@ describe("E2E Tests for v1 API Routes", () => {
.send(scrapeRequest);
expect(response.statusCode).toBe(403);
- expect(response.body.error).toBe(BLOCKLISTED_URL_MESSAGE);
+ expect(response.body.error).toBe(UNSUPPORTED_SITE_MESSAGE);
});
it.concurrent(
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 1caa079a0..865c83cfc 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -30,7 +30,7 @@ import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
import { url as urlSchema } from "../v1/types";
import { ZodError } from "zod";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { fromV0ScrapeOptions } from "../v2/types";
import { isSelfHosted } from "../../lib/deployment";
import { crawlGroup } from "../../services/worker/nuq";
@@ -156,7 +156,7 @@ export async function crawlController(req: Request, res: Response) {
if (isUrlBlocked(url, auth.chunk?.flags ?? null)) {
return res.status(403).json({
- error: BLOCKLISTED_URL_MESSAGE,
+ error: UNSUPPORTED_SITE_MESSAGE,
});
}
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index ca3957720..1e7d420d1 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -19,7 +19,7 @@ import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
import { ZodError } from "zod";
import { Document as V0Document } from "./../../lib/entities";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { fromV0Combo } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
@@ -64,7 +64,7 @@ async function scrapeHelper(
if (isUrlBlocked(url, flags)) {
return {
success: false,
- error: BLOCKLISTED_URL_MESSAGE,
+ error: UNSUPPORTED_SITE_MESSAGE,
returnCode: 403,
};
}
diff --git a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts b/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts
index 6fe982007..df7ff3080 100644
--- a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts
+++ b/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts
@@ -1,5 +1,5 @@
import { url } from "../types";
-import { BLOCKLISTED_URL_MESSAGE } from "../../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../../lib/strings";
describe("URL Schema Validation", () => {
beforeEach(() => {
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 57b36d595..fbeb0425b 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -22,7 +22,7 @@ import { getJobPriority } from "../../lib/job-priority";
import { addScrapeJobs } from "../../services/queue-jobs";
import { createWebhookSender, WebhookEvent } from "../../services/webhook";
import { logger as _logger } from "../../lib/logger";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
@@ -93,7 +93,7 @@ export async function batchScrapeController(
if (!res.headersSent) {
return res.status(403).json({
success: false,
- error: BLOCKLISTED_URL_MESSAGE,
+ error: UNSUPPORTED_SITE_MESSAGE,
});
}
}
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 9662c64c0..1d0e947ca 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -11,7 +11,7 @@ import { saveExtract } from "../../lib/extract/extract-redis";
import { getTeamIdSyncB } from "../../lib/extract/team-id-sync";
import { ExtractResult } from "../../lib/extract/extraction-service";
import { performExtraction_F0 } from "../../lib/extract/fire-0/extraction-service-f0";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { logger as _logger } from "../../lib/logger";
import {
@@ -124,7 +124,7 @@ export async function extractController(
if (!res.headersSent) {
return res.status(403).json({
success: false,
- error: BLOCKLISTED_URL_MESSAGE,
+ error: UNSUPPORTED_SITE_MESSAGE,
});
}
}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 0ab6df618..c161731e8 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -74,7 +74,7 @@ export const url = z.preprocess(
return false;
}
}, "Invalid URL"),
- // .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),
+ // .refine((x) => !isUrlBlocked(x as string), UNSUPPORTED_SITE_MESSAGE),
);
const agentExtractModelValue = "fire-1";
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index d53948fad..805b2519a 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -14,7 +14,7 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logSearch, logRequest } from "../../services/logging/log_job";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
import { CostTracking } from "../../lib/cost-tracking";
@@ -56,7 +56,7 @@ async function scrapeX402SearchResult(
try {
if (isUrlBlocked(searchResult.url, flags)) {
- throw new Error("Could not scrape url: " + BLOCKLISTED_URL_MESSAGE);
+ throw new Error("Could not scrape url: " + UNSUPPORTED_SITE_MESSAGE);
}
logger.info("Adding scrape job [x402]", {
scrapeId: jobId,
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index e2e8a48f8..50da40aa0 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -23,7 +23,7 @@ import { getJobPriority } from "../../lib/job-priority";
import { addScrapeJobs } from "../../services/queue-jobs";
import { createWebhookSender, WebhookEvent } from "../../services/webhook";
import { logger as _logger } from "../../lib/logger";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { checkPermissions } from "../../lib/permissions";
import { crawlGroup } from "../../services/worker/nuq";
@@ -109,7 +109,7 @@ export async function batchScrapeController(
if (!res.headersSent) {
return res.status(403).json({
success: false,
- error: BLOCKLISTED_URL_MESSAGE,
+ error: UNSUPPORTED_SITE_MESSAGE,
});
}
}
diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts
index f5f550407..44b69a5b7 100644
--- a/apps/api/src/controllers/v2/extract.ts
+++ b/apps/api/src/controllers/v2/extract.ts
@@ -8,7 +8,7 @@ import {
} from "./types";
import { addExtractJobToQueue } from "../../services/queue-service";
import { saveExtract } from "../../lib/extract/extract-redis";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { logger as _logger } from "../../lib/logger";
import { logRequest } from "../../services/logging/log_job";
@@ -51,7 +51,8 @@ export async function extractController(
if (req.body.agent?.model === "v3-beta") {
return res.status(400).json({
success: false,
- error: "Use the new /agent endpoint instead of passing agent.model=v3-beta into /extract.",
+ error:
+ "Use the new /agent endpoint instead of passing agent.model=v3-beta into /extract.",
});
}
@@ -64,7 +65,7 @@ export async function extractController(
if (!res.headersSent) {
return res.status(403).json({
success: false,
- error: BLOCKLISTED_URL_MESSAGE,
+ error: UNSUPPORTED_SITE_MESSAGE,
});
}
}
@@ -113,8 +114,8 @@ export async function extractController(
urlTrace: [],
...(invalidURLs.length > 0 && req.body.ignoreInvalidURLs
? {
- invalidURLs,
- }
+ invalidURLs,
+ }
: {}),
});
}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 2712b40e4..b3004a813 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -71,7 +71,7 @@ export const URL = z.preprocess(
return false;
}
}, "Invalid URL"),
- // .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),
+ // .refine((x) => !isUrlBlocked(x as string), UNSUPPORTED_SITE_MESSAGE),
);
const strictMessage =
@@ -771,7 +771,7 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({
auth: z.string(),
requestId: z.string(),
shouldBill: z.boolean(),
- boostConcurrency: z.boolean().optional()
+ boostConcurrency: z.boolean().optional(),
})
.optional(),
});
diff --git a/apps/api/src/lib/strings.ts b/apps/api/src/lib/strings.ts
index 18b59000a..1a04d60f2 100644
--- a/apps/api/src/lib/strings.ts
+++ b/apps/api/src/lib/strings.ts
@@ -1,5 +1,2 @@
-import { isSelfHosted } from "./deployment";
-
-export const BLOCKLISTED_URL_MESSAGE = isSelfHosted()
- ? "This website has been blocklisted and cannot be scraped. Websites may be blocklisted due to: (1) Terms of service restrictions, (2) Legal requirements, (3) Technical limitations that prevent reliable scraping, or (4) Site owner requests. Please check your server configuration and logs for more details about why this specific domain is blocked."
- : "This website has been blocklisted and cannot be scraped. Websites may be blocklisted due to: (1) Terms of service restrictions, (2) Legal requirements, (3) Technical limitations that prevent reliable scraping, or (4) Site owner requests. If you are part of an enterprise plan and believe this site should be accessible for your use case, please reach out to [email protected] to discuss the possibility of getting it activated on your account.";
+export const UNSUPPORTED_SITE_MESSAGE =
+ "We apologize for the inconvenience but we do not support this site. If you are part of an enterprise and want to have a further conversation about this, please fill out our intake form here: https://fk4bvu0n5qp.typeform.com/to/Ej6oydlg";
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 9d261d373..92079bf72 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -13,7 +13,7 @@ import { validateIdempotencyKey } from "../services/idempotency/validate";
import { checkTeamCredits } from "../services/billing/credit_billing";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
import { logger } from "../lib/logger";
-import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../lib/strings";
import { addDomainFrequencyJob } from "../services";
import * as geoip from "geoip-country";
import { isSelfHosted } from "../lib/deployment";
@@ -218,7 +218,7 @@ export function blocklistMiddleware(
if (!res.headersSent) {
return res.status(403).json({
success: false,
- error: BLOCKLISTED_URL_MESSAGE,
+ error: UNSUPPORTED_SITE_MESSAGE,
});
}
}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 35df31154..965cb7261 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -45,7 +45,7 @@ import { startWebScraperPipeline } from "../../main/runWebScraper";
import { CostTracking } from "../../lib/cost-tracking";
import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { generateURLSplits, queryIndexAtSplitLevel } from "../index";
import { WebCrawler } from "../../scraper/WebScraper/crawler";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
@@ -181,10 +181,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
const abortTimeoutHandle =
remainingTime !== undefined
? setTimeout(
- () =>
- abortController.abort(
- new ScrapeJobTimeoutError(),
- ),
+ () => abortController.abort(new ScrapeJobTimeoutError()),
remainingTime,
)
: undefined;
@@ -322,7 +319,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
)
) {
- throw new CrawlDenialError(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
+ throw new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
}
const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v1/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v1/index.test.ts
index 95d6852de..a0c9cf94b 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v1/index.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v1/index.test.ts
@@ -33,10 +33,10 @@ describe('FirecrawlApp E2E Tests', () => {
}
});
- test.concurrent('should throw error for blocklisted URL on scrape', async () => {
+ test.concurrent('should throw error for unsupported URL on scrape', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
- const blocklistedUrl = "https://facebook.com/fake-test";
- await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("This website is not currently supported");
+ const unsupportedUrl = "https://facebook.com/fake-test";
+ await expect(app.scrapeUrl(unsupportedUrl)).rejects.toThrow("do not support this site");
});
test.concurrent('should return successful response for valid scrape', async () => {
@@ -243,10 +243,10 @@ describe('FirecrawlApp E2E Tests', () => {
}
});
- test.concurrent('should throw error for blocklisted URL on map', async () => {
+ test.concurrent('should throw error for unsupported URL on map', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
- const blocklistedUrl = "https://facebook.com/fake-test";
- await expect(app.mapUrl(blocklistedUrl)).rejects.toThrow("403");
+ const unsupportedUrl = "https://facebook.com/fake-test";
+ await expect(app.mapUrl(unsupportedUrl)).rejects.toThrow("403");
});
test.concurrent('should return successful response for valid map', async () => {
diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs
index 05c6d0080..d6a09088b 100644
--- a/apps/rust-sdk/tests/e2e_with_auth.rs
+++ b/apps/rust-sdk/tests/e2e_with_auth.rs
@@ -5,13 +5,13 @@ use serde_json::json;
use std::env;
// #[tokio::test]
-// async fn test_blocklisted_url() {
+// async fn test_unsupported_url() {
// dotenv().ok();
// let api_url = env::var("API_URL").unwrap();
// let api_key = env::var("TEST_API_KEY").ok();
// let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
-// let blocklisted_url = "https://facebook.com/fake-test";
-// let result = app.scrape_url(blocklisted_url, None).await;
+// let unsupported_url = "https://facebook.com/fake-test";
+// let result = app.scrape_url(unsupported_url, None).await;
// assert_matches!(
// result,
@@ -102,13 +102,13 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explici
}
// #[tokio::test]
-// async fn test_should_return_error_for_blocklisted_url() {
+// async fn test_should_return_error_for_unsupported_url() {
// dotenv().ok();
// let api_url = env::var("API_URL").unwrap();
// let api_key = env::var("TEST_API_KEY").ok();
// let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
-// let blocklisted_url = "https://twitter.com/fake-test";
-// let result = app.crawl_url(blocklisted_url, None).await;
+// let unsupported_url = "https://twitter.com/fake-test";
+// let result = app.crawl_url(unsupported_url, None).await;
// assert_matches!(
// result,