Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: blocklist string
Nicolasd1f3e26f9ed8
diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.tsindex 45b3c31e5..40686c45d 100644--- a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts@@ -1,6 +1,7 @@import request from "supertest";import dotenv from "dotenv";import { v4 as uuidv4 } from "uuid";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";dotenv.config();@@ -58,9 +59,7 @@ describe("E2E Tests for API Routes", () => {.set("Content-Type", "application/json").send({ url: blocklistedUrl });expect(response.statusCode).toBe(403);- expect(response.body.error).toContain(- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",- );+ expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);});// tested on rate limit test@@ -480,9 +479,7 @@ describe("E2E Tests for API Routes", () => {.set("Content-Type", "application/json").send({ url: blocklistedUrl });expect(response.statusCode).toBe(403);- expect(response.body.error).toContain(- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",- );+ expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);});it.concurrent(diff --git a/apps/api/src/__tests__/e2e_noAuth/index.test.ts b/apps/api/src/__tests__/e2e_noAuth/index.test.tsindex e30352a55..9d5dc5546 100644--- a/apps/api/src/__tests__/e2e_noAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_noAuth/index.test.ts@@ -1,5 +1,6 @@import request from "supertest";import dotenv from "dotenv";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";const fs = require("fs");const path = require("path");@@ -61,9 +62,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {.set("Content-Type", "application/json").send({ url: blocklistedUrl });expect(response.statusCode).toBe(403);- expect(response.body.error).toContain(- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",- );+ expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);});it("should return a successful response", async () => {@@ -88,9 +87,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {.set("Content-Type", "application/json").send({ url: blocklistedUrl });expect(response.statusCode).toBe(403);- expect(response.body.error).toContain(- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",- );+ expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);});it("should return a successful response", async () => {@@ -119,9 +116,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {.set("Content-Type", "application/json").send({ url: blocklistedUrl });expect(response.statusCode).toBe(403);- expect(response.body.error).toContain(- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",- );+ expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);});it("should return a successful response", async () => {diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.tsindex 35ee2d89f..39e0aa855 100644--- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts@@ -4,6 +4,7 @@ import {ScrapeRequestInput,ScrapeResponseRequestTest,} from "../../controllers/v1/types";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";configDotenv();const TEST_URL = "http://127.0.0.1:3002";@@ -57,9 +58,7 @@ describe("E2E Tests for v1 API Routes", () => {.send(scrapeRequest);expect(response.statusCode).toBe(403);- expect(response.body.error).toBe(- "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.",- );+ expect(response.body.error).toBe(BLOCKLISTED_URL_MESSAGE);});it.concurrent(@@ -756,9 +755,7 @@ describe("E2E Tests for v1 API Routes", () => {.send(scrapeRequest);expect(response.statusCode).toBe(403);- expect(response.body.error).toBe(- "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.",- );+ expect(response.body.error).toBe(BLOCKLISTED_URL_MESSAGE);});it.concurrent(diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 36b8309fa..ceeaa4367 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -29,6 +29,7 @@ import * as Sentry from "@sentry/node";import { getJobPriority } from "../../lib/job-priority";import { fromLegacyScrapeOptions, url as urlSchema } from "../v1/types";import { ZodError } from "zod";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";export async function crawlController(req: Request, res: Response) {try {@@ -112,8 +113,7 @@ export async function crawlController(req: Request, res: Response) {if (isUrlBlocked(url)) {return res.status(403).json({- error:- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",+ error: BLOCKLISTED_URL_MESSAGE,});}diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.tsindex 405e49c27..f9462c3d2 100644--- a/apps/api/src/controllers/v0/crawlPreview.ts+++ b/apps/api/src/controllers/v0/crawlPreview.ts@@ -15,6 +15,7 @@ import { addScrapeJob } from "../../../src/services/queue-jobs";import { checkAndUpdateURL } from "../../../src/lib/validateUrl";import * as Sentry from "@sentry/node";import { fromLegacyScrapeOptions } from "../v1/types";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";export async function crawlPreviewController(req: Request, res: Response) {try {@@ -42,8 +43,7 @@ export async function crawlPreviewController(req: Request, res: Response) {if (isUrlBlocked(url)) {return res.status(403).json({- error:- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",+ error: BLOCKLISTED_URL_MESSAGE,});}diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 2d8acf5f8..05bf364bd 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -29,6 +29,7 @@ import { getJobPriority } from "../../lib/job-priority";import { fromLegacyScrapeOptions } from "../v1/types";import { ZodError } from "zod";import { Document as V0Document } from "./../../lib/entities";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";export async function scrapeHelper(jobId: string,@@ -53,8 +54,7 @@ export async function scrapeHelper(if (isUrlBlocked(url)) {return {success: false,- error:- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",+ error: BLOCKLISTED_URL_MESSAGE,returnCode: 403,};}@@ -265,13 +265,15 @@ export async function scrapeController(req: Request, res: Response) {}if (creditsToBeBilled > 0) {// billing for doc done on queue end, bill only for llm extraction- billTeam(team_id, chunk?.sub_id, creditsToBeBilled, logger).catch((error) => {- logger.error(- `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,- { error }- );- // Optionally, you could notify an admin or add to a retry queue here- });+ billTeam(team_id, chunk?.sub_id, creditsToBeBilled, logger).catch(+ (error) => {+ logger.error(+ `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,+ { error },+ );+ // Optionally, you could notify an admin or add to a retry queue here+ },+ );}}diff --git a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts b/apps/api/src/controllers/v1/__tests__/urlValidation.test.tsindex b455e5ab2..afa44e58d 100644--- a/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts+++ b/apps/api/src/controllers/v1/__tests__/urlValidation.test.ts@@ -1,4 +1,5 @@import { url } from "../types";+import { BLOCKLISTED_URL_MESSAGE } from "../../../lib/strings";describe("URL Schema Validation", () => {beforeEach(() => {@@ -31,7 +32,7 @@ describe("URL Schema Validation", () => {it("should reject blocked URLs", () => {expect(() => url.parse("https://facebook.com")).toThrow(- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",+ BLOCKLISTED_URL_MESSAGE,);});@@ -47,16 +48,16 @@ describe("URL Schema Validation", () => {it("should handle URLs with subdomains that are blocked", () => {expect(() => url.parse("https://sub.facebook.com")).toThrow(- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",+ BLOCKLISTED_URL_MESSAGE,);});it("should handle URLs with paths that are blocked", () => {expect(() => url.parse("http://facebook.com/path")).toThrow(- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",+ BLOCKLISTED_URL_MESSAGE,);expect(() => url.parse("https://facebook.com/another/path")).toThrow(- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",+ BLOCKLISTED_URL_MESSAGE,);});diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 1188bafb4..58e757517 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -92,19 +92,18 @@ export async function extractController(let mappedLinks = mapResults.mapResults as MapDocument[];// Remove duplicates between mapResults.links and mappedLinks- const allUrls = [...mappedLinks.map(m => m.url), ...mapResults.links];+ const allUrls = [...mappedLinks.map((m) => m.url), ...mapResults.links];const uniqueUrls = removeDuplicateUrls(allUrls);-+// Only add URLs from mapResults.links that aren't already in mappedLinks- const existingUrls = new Set(mappedLinks.map(m => m.url));- const newUrls = uniqueUrls.filter(url => !existingUrls.has(url));-+ const existingUrls = new Set(mappedLinks.map((m) => m.url));+ const newUrls = uniqueUrls.filter((url) => !existingUrls.has(url));+mappedLinks = [...mappedLinks,- ...newUrls.map(url => ({ url, title: "", description: "" }))+ ...newUrls.map((url) => ({ url, title: "", description: "" })),];-if (mappedLinks.length === 0) {mappedLinks = [{ url: baseUrl, title: "", description: "" }];}@@ -117,7 +116,6 @@ export async function extractController(`url: ${x.url}, title: ${x.title}, description: ${x.description}`,);-if (req.body.prompt) {let searchQuery =req.body.prompt && allowExternalLinksdiff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 06605eb95..114c115e8 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -11,6 +11,7 @@ import {Document as V0Document,} from "../../lib/entities";import { InternalOptions } from "../../scraper/scrapeURL";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";export type Format =| "markdown"@@ -44,10 +45,7 @@ export const url = z.preprocess(return false;}}, "Invalid URL")- .refine(- (x) => !isUrlBlocked(x as string),- "Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",- ),+ .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),);const strictMessage =diff --git a/apps/api/src/lib/strings.ts b/apps/api/src/lib/strings.tsnew file mode 100644index 000000000..4e278d2b3--- /dev/null+++ b/apps/api/src/lib/strings.ts@@ -0,0 +1,2 @@+export const BLOCKLISTED_URL_MESSAGE =+ "This website is no longer supported, please reach out to [email protected] for more info on how to activate it on your account.";diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex c50ab9c9b..0f3b85244 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -71,7 +71,7 @@ export async function runWebScraper({module: "runWebscraper",scrapeId: bull_job_id,jobId: bull_job_id,- })+ });const tries = is_crawl ? 3 : 1;let response: ScrapeUrlResponse | undefined = undefined;@@ -176,7 +176,7 @@ export async function runWebScraper({billTeam(team_id, undefined, creditsToBeBilled, logger).catch((error) => {logger.error(`Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,- { error }+ { error },);// Optionally, you could notify an admin or add to a retry queue here});diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 764271145..1ee191efd 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -32,6 +32,7 @@ import { extractController } from "../controllers/v1/extract";// import { livenessController } from "../controllers/v1/liveness";// import { readinessController } from "../controllers/v1/readiness";import { creditUsageController } from "../controllers/v1/credit-usage";+import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";function checkCreditsMiddleware(minimum?: number,@@ -123,8 +124,7 @@ function blocklistMiddleware(req: Request, res: Response, next: NextFunction) {if (!res.headersSent) {return res.status(403).json({success: false,- error:- "URL is blocked intentionally. Firecrawl currently does not support scraping this site due to policy restrictions.",+ error: BLOCKLISTED_URL_MESSAGE,});}}@@ -231,4 +231,3 @@ v1Router.get(authMiddleware(RateLimiterMode.CrawlStatus),wrap(creditUsageController),);-diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex c2671034b..5eb541fd4 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -44,7 +44,10 @@ export async function supaBillTeam(if (team_id === "preview") {return { success: true, message: "Preview team, no credits used" };}- _logger.info(`Billing team ${team_id} for ${credits} credits`, { team_id, credits });+ _logger.info(`Billing team ${team_id} for ${credits} credits`, {+ team_id,+ credits,+ });const { data, error } = await supabase_service.rpc("bill_team", {_team_id: team_id,diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.tsindex dea558462..81b0a523b 100644--- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts@@ -159,12 +159,6 @@ describe('FirecrawlApp E2E Tests', () => {await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");});- test.concurrent('should throw error for blocklisted URL on crawl', async () => {- const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });- const blocklistedUrl = "https://twitter.com/fake-test";- await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.");- });-test.concurrent('should return successful response for crawl and wait for completion', async () => {const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });const response = await app.crawlUrl('https://roastmywebsite.ai', {}, 30) as CrawlStatusResponse;diff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.pyindex 8945d74dc..50d5306f8 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py@@ -29,12 +29,12 @@ def test_scrape_url_invalid_api_key():invalid_app.scrape_url('https://firecrawl.dev')assert "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)-def test_blocklisted_url():- blocklisted_url = "https://facebook.com/fake-test"- app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')- with pytest.raises(Exception) as excinfo:- app.scrape_url(blocklisted_url)- assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)+# def test_blocklisted_url():+# blocklisted_url = "https://facebook.com/fake-test"+# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')+# with pytest.raises(Exception) as excinfo:+# app.scrape_url(blocklisted_url)+# assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)def test_successful_response_with_valid_preview_token():app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0')@@ -90,12 +90,12 @@ def test_crawl_url_invalid_api_key():invalid_app.crawl_url('https://firecrawl.dev')assert "Unexpected error during start crawl job: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)-def test_should_return_error_for_blocklisted_url():- app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')- blocklisted_url = "https://twitter.com/fake-test"- with pytest.raises(Exception) as excinfo:- app.crawl_url(blocklisted_url)- assert "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)+# def test_should_return_error_for_blocklisted_url():+# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')+# blocklisted_url = "https://twitter.com/fake-test"+# with pytest.raises(Exception) as excinfo:+# app.crawl_url(blocklisted_url)+# assert "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)def test_crawl_url_wait_for_completion_e2e():app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')diff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.pyindex 12fa10ce2..0ada6c1d8 100644--- a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py+++ b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py@@ -30,12 +30,12 @@ def test_scrape_url_invalid_api_key():invalid_app.scrape_url('https://firecrawl.dev')assert "Unauthorized: Invalid token" in str(excinfo.value)-def test_blocklisted_url():- blocklisted_url = "https://facebook.com/fake-test"- app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)- with pytest.raises(Exception) as excinfo:- app.scrape_url(blocklisted_url)- assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)+# def test_blocklisted_url():+# blocklisted_url = "https://facebook.com/fake-test"+# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)+# with pytest.raises(Exception) as excinfo:+# app.scrape_url(blocklisted_url)+# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)def test_successful_response_with_valid_preview_token():app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")@@ -136,12 +136,12 @@ def test_crawl_url_invalid_api_key():invalid_app.crawl_url('https://firecrawl.dev')assert "Unauthorized: Invalid token" in str(excinfo.value)-def test_should_return_error_for_blocklisted_url():- app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)- blocklisted_url = "https://twitter.com/fake-test"- with pytest.raises(Exception) as excinfo:- app.crawl_url(blocklisted_url)- assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)+# def test_should_return_error_for_blocklisted_url():+# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)+# blocklisted_url = "https://twitter.com/fake-test"+# with pytest.raises(Exception) as excinfo:+# app.crawl_url(blocklisted_url)+# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)def test_crawl_url_wait_for_completion_e2e():app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)@@ -296,12 +296,12 @@ def test_invalid_api_key_on_map():invalid_app.map_url('https://roastmywebsite.ai')assert "Unauthorized: Invalid token" in str(excinfo.value)-def test_blocklisted_url_on_map():- app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)- blocklisted_url = "https://facebook.com/fake-test"- with pytest.raises(Exception) as excinfo:- app.map_url(blocklisted_url)- assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)+# def test_blocklisted_url_on_map():+# app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)+# blocklisted_url = "https://facebook.com/fake-test"+# with pytest.raises(Exception) as excinfo:+# app.map_url(blocklisted_url)+# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)def test_successful_response_with_valid_preview_token_on_map():app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL)diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rsindex 75568f92b..92b202cbe 100644--- a/apps/rust-sdk/tests/e2e_with_auth.rs+++ b/apps/rust-sdk/tests/e2e_with_auth.rs@@ -5,20 +5,20 @@ use firecrawl::FirecrawlApp;use serde_json::json;use std::env;-#[tokio::test]-async fn test_blocklisted_url() {- dotenv().ok();- let api_url = env::var("API_URL").unwrap();- let api_key = env::var("TEST_API_KEY").ok();- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();- let blocklisted_url = "https://facebook.com/fake-test";- let result = app.scrape_url(blocklisted_url, None).await;+// #[tokio::test]+// async fn test_blocklisted_url() {+// dotenv().ok();+// let api_url = env::var("API_URL").unwrap();+// let api_key = env::var("TEST_API_KEY").ok();+// let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();+// let blocklisted_url = "https://facebook.com/fake-test";+// let result = app.scrape_url(blocklisted_url, None).await;- assert_matches!(- result,- Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions")- );-}+// assert_matches!(+// result,+// Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions")+// );+// }#[tokio::test]async fn test_successful_response_with_valid_preview_token() {@@ -103,20 +103,21 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explici.contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));}-#[tokio::test]-async fn test_should_return_error_for_blocklisted_url() {- dotenv().ok();- let api_url = env::var("API_URL").unwrap();- let api_key = env::var("TEST_API_KEY").ok();- let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();- let blocklisted_url = "https://twitter.com/fake-test";- let result = app.crawl_url(blocklisted_url, None).await;- assert_matches!(- result,- Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions.")- );-}+// #[tokio::test]+// async fn test_should_return_error_for_blocklisted_url() {+// dotenv().ok();+// let api_url = env::var("API_URL").unwrap();+// let api_key = env::var("TEST_API_KEY").ok();+// let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();+// let blocklisted_url = "https://twitter.com/fake-test";+// let result = app.crawl_url(blocklisted_url, None).await;++// assert_matches!(+// result,+// Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions.")+// );+// }#[tokio::test]async fn test_llm_extraction() {