Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/fire-engine): default to separate US-generic proxy list if no location is specified (FIR-728) (#1104)
Gergő Móricz5733b82e9dde
diff --git a/apps/api/src/__tests__/snips/mocks/mocking-works-properly.json b/apps/api/src/__tests__/snips/mocks/mocking-works-properly.jsonindex 5609e6c21..6cba173f6 100644--- a/apps/api/src/__tests__/snips/mocks/mocking-works-properly.json+++ b/apps/api/src/__tests__/snips/mocks/mocking-works-properly.json@@ -2,7 +2,7 @@{"time": 1735911273239,"options": {- "url": "http://default-fire-engine-api-service:8080/scrape",+ "url": "<fire-engine>/scrape","method": "POST","body": {"url": "http://firecrawl.dev",@@ -27,7 +27,7 @@{"time": 1735911273354,"options": {- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp","method": "GET","headers": {},"ignoreResponse": false,@@ -43,7 +43,7 @@{"time": 1735911273720,"options": {- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp","method": "GET","headers": {},"ignoreResponse": false,@@ -59,7 +59,7 @@{"time": 1735911274092,"options": {- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp","method": "GET","headers": {},"ignoreResponse": false,@@ -75,7 +75,7 @@{"time": 1735911274467,"options": {- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp","method": "GET","headers": {},"ignoreResponse": false,@@ -91,7 +91,7 @@{"time": 1735911274947,"options": {- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp","method": "GET","headers": {},"ignoreResponse": false,diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex fd28c25d0..8a570ec68 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -35,4 +35,24 @@ describe("Scrape tests", () => {"this is fake data coming from the mocking system!",);});++ describe("Location API", () => {+ it.concurrent("works without specifying an explicit location", async () => {+ const response = await scrape({+ url: "https://iplocation.com",+ });++ expectScrapeToSucceed(response);+ });++ it.concurrent("works with country US", async () => {+ const response = await scrape({+ url: "https://iplocation.com",+ location: { country: "US" },+ });++ expectScrapeToSucceed(response);+ expect(response.body.data.markdown).toContain("| Country | United States |");+ });+ })});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex cdf2a2d44..6ad9288bf 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -154,13 +154,13 @@ export const scrapeOptions = z.string().optional().refine(- (val) => !val || Object.keys(countries).includes(val.toUpperCase()),+ (val) => !val || Object.keys(countries).includes(val.toUpperCase()) || val === "US-generic",{message:"Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",},)- .transform((val) => (val ? val.toUpperCase() : "US")),+ .transform((val) => (val ? val.toUpperCase() : "US-generic")),languages: z.string().array().optional(),}).optional(),@@ -178,7 +178,7 @@ export const scrapeOptions = z"Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",},)- .transform((val) => (val ? val.toUpperCase() : "US")),+ .transform((val) => (val ? val.toUpperCase() : "US-generic")),languages: z.string().array().optional(),}).optional(),diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex 73b8f9be4..c9513bfe3 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -126,10 +126,14 @@ export async function robustFetch<const makeRequestTypeId = (request: (typeof mock)["requests"][number]["options"],) => {- let out = request.url + ";" + request.method;+ let trueUrl = (process.env.FIRE_ENGINE_BETA_URL && request.url.startsWith(process.env.FIRE_ENGINE_BETA_URL))+ ? request.url.replace(process.env.FIRE_ENGINE_BETA_URL, "<fire-engine>")+ : request.url;++ let out = trueUrl + ";" + request.method;if (process.env.FIRE_ENGINE_BETA_URL &&- url.startsWith(process.env.FIRE_ENGINE_BETA_URL) &&+ (trueUrl.startsWith("<fire-engine>")) &&request.method === "POST") {out += "f-e;" + request.body?.engine + ";" + request.body?.url;diff --git a/apps/api/src/scraper/scrapeURL/lib/mock.ts b/apps/api/src/scraper/scrapeURL/lib/mock.tsindex e57256d65..853e3a4f1 100644--- a/apps/api/src/scraper/scrapeURL/lib/mock.ts+++ b/apps/api/src/scraper/scrapeURL/lib/mock.ts@@ -3,7 +3,7 @@ import * as path from "path";import { logger as _logger } from "../../../lib/logger";import { Logger } from "winston";const saveMocksDirPath = path.join(__dirname, "../mocks/").replace("dist/", "");-const loadMocksDirPath = path.join(__dirname, "../../../__tests__/snips/mocks");+const loadMocksDirPath = path.join(__dirname, "../../../__tests__/snips/mocks").replace("dist/", "");export async function saveMock(options: unknown, result: unknown) {if (process.env.FIRECRAWL_SAVE_MOCKS !== "true") return;