Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrapeURL/fire-engine): default to separate US-generic proxy list if no location is specified (FIR-728) (#1104)

Gergő Móricz5733b82e9dde
diff --git a/apps/api/src/__tests__/snips/mocks/mocking-works-properly.json b/apps/api/src/__tests__/snips/mocks/mocking-works-properly.json
index 5609e6c21..6cba173f6 100644
--- a/apps/api/src/__tests__/snips/mocks/mocking-works-properly.json
+++ b/apps/api/src/__tests__/snips/mocks/mocking-works-properly.json
@@ -2,7 +2,7 @@
{
"time": 1735911273239,
"options": {
- "url": "http://default-fire-engine-api-service:8080/scrape",
+ "url": "<fire-engine>/scrape",
"method": "POST",
"body": {
"url": "http://firecrawl.dev",
@@ -27,7 +27,7 @@
{
"time": 1735911273354,
"options": {
- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
"method": "GET",
"headers": {},
"ignoreResponse": false,
@@ -43,7 +43,7 @@
{
"time": 1735911273720,
"options": {
- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
"method": "GET",
"headers": {},
"ignoreResponse": false,
@@ -59,7 +59,7 @@
{
"time": 1735911274092,
"options": {
- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
"method": "GET",
"headers": {},
"ignoreResponse": false,
@@ -75,7 +75,7 @@
{
"time": 1735911274467,
"options": {
- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
"method": "GET",
"headers": {},
"ignoreResponse": false,
@@ -91,7 +91,7 @@
{
"time": 1735911274947,
"options": {
- "url": "http://default-fire-engine-api-service:8080/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
+ "url": "<fire-engine>/scrape/ede37286-90db-4f60-8efb-76217dfcfa35!chrome-cdp",
"method": "GET",
"headers": {},
"ignoreResponse": false,
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index fd28c25d0..8a570ec68 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -35,4 +35,24 @@ describe("Scrape tests", () => {
"this is fake data coming from the mocking system!",
);
});
+
+ describe("Location API", () => {
+ it.concurrent("works without specifying an explicit location", async () => {
+ const response = await scrape({
+ url: "https://iplocation.com",
+ });
+
+ expectScrapeToSucceed(response);
+ });
+
+ it.concurrent("works with country US", async () => {
+ const response = await scrape({
+ url: "https://iplocation.com",
+ location: { country: "US" },
+ });
+
+ expectScrapeToSucceed(response);
+ expect(response.body.data.markdown).toContain("| Country | United States |");
+ });
+ })
});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index cdf2a2d44..6ad9288bf 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -154,13 +154,13 @@ export const scrapeOptions = z
.string()
.optional()
.refine(
- (val) => !val || Object.keys(countries).includes(val.toUpperCase()),
+ (val) => !val || Object.keys(countries).includes(val.toUpperCase()) || val === "US-generic",
{
message:
"Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",
},
)
- .transform((val) => (val ? val.toUpperCase() : "US")),
+ .transform((val) => (val ? val.toUpperCase() : "US-generic")),
languages: z.string().array().optional(),
})
.optional(),
@@ -178,7 +178,7 @@ export const scrapeOptions = z
"Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",
},
)
- .transform((val) => (val ? val.toUpperCase() : "US")),
+ .transform((val) => (val ? val.toUpperCase() : "US-generic")),
languages: z.string().array().optional(),
})
.optional(),
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index 73b8f9be4..c9513bfe3 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -126,10 +126,14 @@ export async function robustFetch<
const makeRequestTypeId = (
request: (typeof mock)["requests"][number]["options"],
) => {
- let out = request.url + ";" + request.method;
+ let trueUrl = (process.env.FIRE_ENGINE_BETA_URL && request.url.startsWith(process.env.FIRE_ENGINE_BETA_URL))
+ ? request.url.replace(process.env.FIRE_ENGINE_BETA_URL, "<fire-engine>")
+ : request.url;
+
+ let out = trueUrl + ";" + request.method;
if (
process.env.FIRE_ENGINE_BETA_URL &&
- url.startsWith(process.env.FIRE_ENGINE_BETA_URL) &&
+ (trueUrl.startsWith("<fire-engine>")) &&
request.method === "POST"
) {
out += "f-e;" + request.body?.engine + ";" + request.body?.url;
diff --git a/apps/api/src/scraper/scrapeURL/lib/mock.ts b/apps/api/src/scraper/scrapeURL/lib/mock.ts
index e57256d65..853e3a4f1 100644
--- a/apps/api/src/scraper/scrapeURL/lib/mock.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/mock.ts
@@ -3,7 +3,7 @@ import * as path from "path";
import { logger as _logger } from "../../../lib/logger";
import { Logger } from "winston";
const saveMocksDirPath = path.join(__dirname, "../mocks/").replace("dist/", "");
-const loadMocksDirPath = path.join(__dirname, "../../../__tests__/snips/mocks");
+const loadMocksDirPath = path.join(__dirname, "../../../__tests__/snips/mocks").replace("dist/", "");
export async function saveMock(options: unknown, result: unknown) {
if (process.env.FIRECRAWL_SAVE_MOCKS !== "true") return;