Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add proxy location support to crawl and map endpoints (ENG-3361) (#2092)

tom9a3bd6ca5017
diff --git a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts b/apps/api/src/__tests__/snips/v2/robots-txt.test.ts
index b9c9c5f1d..e96275c21 100644
--- a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts
+++ b/apps/api/src/__tests__/snips/v2/robots-txt.test.ts
@@ -8,7 +8,16 @@ describe("Robots.txt tests", () => {
let checker: ReturnType<typeof createRobotsChecker>;
beforeAll(async () => {
- robotsTxt = await fetchRobotsTxt(robotsUrl);
+ const logger = {
+ error: jest.fn(),
+ } as any;
+ robotsTxt = (
+ await fetchRobotsTxt(
+ { url: robotsUrl, zeroDataRetention: true },
+ "test-scrape",
+ logger,
+ )
+ ).content;
checker = createRobotsChecker(siteUrl, robotsTxt);
}, scrapeTimeout);
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 51cac4173..45593c2ca 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -6,7 +6,7 @@ import {
RequestWithAuth,
TeamFlags,
} from "./types";
-import { scrapeOptions } from "../v2/types";
+import { scrapeOptions, ScrapeOptions } from "../v2/types";
import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
import { MapResponse, MapRequest, MAX_MAP_LIMIT } from "./types";
import { configDotenv } from "dotenv";
@@ -94,6 +94,7 @@ export async function getMapResults({
flags,
useIndex = true,
timeout,
+ location,
}: {
url: string;
search?: string;
@@ -111,6 +112,7 @@ export async function getMapResults({
flags: TeamFlags;
useIndex?: boolean;
timeout?: number;
+ location?: ScrapeOptions["location"];
}): Promise<MapResult> {
const id = uuidv4();
let links: string[] = [url];
@@ -125,7 +127,9 @@ export async function getMapResults({
limit: crawlerOptions.sitemapOnly ? 10000000 : limit,
scrapeOptions: undefined,
},
- scrapeOptions: scrapeOptions.parse({}),
+ scrapeOptions: scrapeOptions.parse({
+ ...(location ? { location } : {}),
+ }),
internalOptions: { teamId },
team_id: teamId,
createdAt: Date.now(),
@@ -381,6 +385,7 @@ export async function mapController(
flags: req.acuc?.flags ?? null,
useIndex: req.body.useIndex,
timeout: req.body.timeout,
+ location: req.body.location,
}),
...(req.body.timeout !== undefined
? [
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 46159a83b..89a8a8e11 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -22,7 +22,7 @@ import * as Sentry from "@sentry/node";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
-import { CostTracking } from "../../lib/extract/extraction-service";
+import { CostTracking } from "../../lib/cost-tracking";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
import { fromV1ScrapeOptions } from "../v2/types";
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 7d1ee1c77..b3458abd6 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -284,6 +284,26 @@ export const actionsSchema = z
},
);
+export const locationSchema = z
+ .object({
+ country: z
+ .string()
+ .optional()
+ .refine(
+ val =>
+ !val ||
+ Object.keys(countries).includes(val.toUpperCase()) ||
+ val === "US-generic",
+ {
+ message:
+ "Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",
+ },
+ )
+ .transform(val => (val ? val.toUpperCase() : "US-generic")),
+ languages: z.string().array().optional(),
+ })
+ .optional();
+
function transformIframeSelector(selector: string): string {
return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, match => {
const prefix = match.match(/^[\s,]/)?.[0] || "";
@@ -370,25 +390,7 @@ const baseScrapeOptions = z
parsePDF: z.boolean().default(true),
actions: actionsSchema.optional(),
// New
- location: z
- .object({
- country: z
- .string()
- .optional()
- .refine(
- val =>
- !val ||
- Object.keys(countries).includes(val.toUpperCase()) ||
- val === "US-generic",
- {
- message:
- "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",
- },
- )
- .transform(val => (val ? val.toUpperCase() : "US-generic")),
- languages: z.string().array().optional(),
- })
- .optional(),
+ location: locationSchema,
// Deprecated
geolocation: z
@@ -866,6 +868,7 @@ export const mapRequestSchema = crawlerOptions
useMock: z.string().optional(),
filterByPath: z.boolean().default(true),
useIndex: z.boolean().default(true),
+ location: locationSchema,
})
.strict(strictMessage);
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index 6345f6856..706fed9de 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -20,7 +20,7 @@ import * as Sentry from "@sentry/node";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
-import { CostTracking } from "../../lib/extract/extraction-service";
+import { CostTracking } from "../../lib/cost-tracking";
import { supabase_service } from "../../services/supabase";
import { fromV1ScrapeOptions } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
index 9c5d65ecc..2b556a20a 100644
--- a/apps/api/src/controllers/v2/crawl-params-preview.ts
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -3,7 +3,7 @@ import { z } from "zod";
import { ErrorResponse, RequestWithAuth, crawlRequestSchema } from "./types";
import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
-import { CostTracking } from "../../lib/extract/extraction-service";
+import { CostTracking } from "../../lib/cost-tracking";
// Define the request schema for params preview
// Only url and prompt are required/relevant for preview
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index aad3cae4b..337e6f806 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -11,7 +11,7 @@ import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";
import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
-import { CostTracking } from "../../lib/extract/extraction-service";
+import { CostTracking } from "../../lib/cost-tracking";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index a7e565981..7df9e6fcf 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -4,6 +4,7 @@ import {
mapRequestSchema,
RequestWithAuth,
scrapeOptions,
+ ScrapeOptions,
TeamFlags,
MapRequest,
MapDocument,
@@ -96,6 +97,7 @@ export async function getMapResults({
filterByPath = true,
flags,
useIndex = true,
+ location,
}: {
url: string;
search?: string;
@@ -111,6 +113,7 @@ export async function getMapResults({
filterByPath?: boolean;
flags: TeamFlags;
useIndex?: boolean;
+ location?: ScrapeOptions["location"];
}): Promise<MapResult> {
const id = uuidv4();
let mapResults: MapDocument[] = [];
@@ -123,7 +126,9 @@ export async function getMapResults({
limit: crawlerOptions.sitemapOnly ? 10000000 : limit,
scrapeOptions: undefined,
},
- scrapeOptions: scrapeOptions.parse({}),
+ scrapeOptions: scrapeOptions.parse({
+ ...(location ? { location } : {}),
+ }),
internalOptions: { teamId },
team_id: teamId,
createdAt: Date.now(),
@@ -375,6 +380,7 @@ export async function mapController(
filterByPath: req.body.filterByPath !== false,
flags: req.acuc?.flags ?? null,
useIndex: req.body.useIndex,
+ location: req.body.location,
}),
...(req.body.timeout !== undefined
? [
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 052167f00..c958cf63b 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -20,7 +20,7 @@ import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
-import { CostTracking } from "../../lib/extract/extraction-service";
+import { CostTracking } from "../../lib/cost-tracking";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
import { SearchResult, SearchV2Response } from "../../lib/entities";
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 8ccff9d89..09348bfd8 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -332,6 +332,26 @@ function transformIframeSelector(selector: string): string {
});
}
+const locationSchema = z
+ .object({
+ country: z
+ .string()
+ .optional()
+ .refine(
+ val =>
+ !val ||
+ Object.keys(countries).includes(val.toUpperCase()) ||
+ val === "US-generic",
+ {
+ message:
+ "Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",
+ },
+ )
+ .transform(val => (val ? val.toUpperCase() : "US-generic")),
+ languages: z.string().array().optional(),
+ })
+ .optional();
+
const baseScrapeOptions = z
.object({
formats: z
@@ -395,25 +415,7 @@ const baseScrapeOptions = z
parsers: parsersSchema.optional(),
actions: actionsSchema.optional(),
- location: z
- .object({
- country: z
- .string()
- .optional()
- .refine(
- val =>
- !val ||
- Object.keys(countries).includes(val.toUpperCase()) ||
- val === "US-generic",
- {
- message:
- "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",
- },
- )
- .transform(val => (val ? val.toUpperCase() : "US-generic")),
- languages: z.string().array().optional(),
- })
- .optional(),
+ location: locationSchema,
skipTlsVerification: z.boolean().default(true),
removeBase64Images: z.boolean().default(true),
@@ -732,6 +734,7 @@ export const mapRequestSchema = crawlerOptions
useMock: z.string().optional(),
filterByPath: z.boolean().default(true),
useIndex: z.boolean().default(true),
+ location: locationSchema,
})
.strict(strictMessage);
diff --git a/apps/api/src/lib/cost-tracking.ts b/apps/api/src/lib/cost-tracking.ts
new file mode 100644
index 000000000..a41d7f6a4
--- /dev/null
+++ b/apps/api/src/lib/cost-tracking.ts
@@ -0,0 +1,57 @@
+export class CostLimitExceededError extends Error {
+ constructor() {
+ super("Cost limit exceeded");
+ this.message = "Cost limit exceeded";
+ this.name = "CostLimitExceededError";
+ }
+}
+
+const nanProof = (n: number | null | undefined) =>
+ isNaN(n ?? 0) ? 0 : (n ?? 0);
+
+export class CostTracking {
+ calls: {
+ type: "smartScrape" | "other";
+ metadata: Record<string, any>;
+ cost: number;
+ model: string;
+ tokens?: {
+ input: number;
+ output: number;
+ };
+ stack: string;
+ }[] = [];
+ limit: number | null = null;
+
+ constructor(limit: number | null = null) {
+ this.limit = limit;
+ }
+
+ public addCall(call: Omit<(typeof this.calls)[number], "stack">) {
+ this.calls.push({
+ ...call,
+ stack: new Error().stack!.split("\n").slice(2).join("\n"),
+ });
+
+ if (this.limit !== null && this.toJSON().totalCost > this.limit) {
+ throw new CostLimitExceededError();
+ }
+ }
+
+ public toJSON() {
+ return {
+ calls: this.calls,
+
+ smartScrapeCallCount: this.calls.filter(c => c.type === "smartScrape")
+ .length,
+ smartScrapeCost: this.calls
+ .filter(c => c.type === "smartScrape")
+ .reduce((acc, c) => acc + nanProof(c.cost), 0),
+ otherCallCount: this.calls.filter(c => c.type === "other").length,
+ otherCost: this.calls
+ .filter(c => c.type === "other")
+ .reduce((acc, c) => acc + nanProof(c.cost), 0),
+ totalCost: this.calls.reduce((acc, c) => acc + nanProof(c.cost), 0),
+ };
+ }
+}
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index ed1a5a621..0ffc6a97a 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -530,6 +530,7 @@ export function crawlToCrawler(
maxDiscoveryDepth: sc.crawlerOptions?.maxDiscoveryDepth,
currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,
zeroDataRetention: (teamFlags?.forceZDR || sc.zeroDataRetention) ?? false,
+ location: sc.scrapeOptions?.location,
});
if (sc.robots !== undefined) {
diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.ts
index d3bfaca88..38d67ae56 100644
--- a/apps/api/src/lib/deep-research/deep-research-service.ts
+++ b/apps/api/src/lib/deep-research/deep-research-service.ts
@@ -5,7 +5,7 @@ import { ResearchLLMService, ResearchStateManager } from "./research-manager";
import { logJob } from "../../services/logging/log_job";
import { billTeam } from "../../services/billing/credit_billing";
import { ExtractOptions } from "../../controllers/v1/types";
-import { CostTracking } from "../extract/extraction-service";
+import { CostTracking } from "../cost-tracking";
import { getACUCTeam } from "../../controllers/auth";
export interface DeepResearchServiceOptions {
researchId: string;
diff --git a/apps/api/src/lib/deep-research/research-manager.ts b/apps/api/src/lib/deep-research/research-manager.ts
index 8b7b28436..81389267d 100644
--- a/apps/api/src/lib/deep-research/research-manager.ts
+++ b/apps/api/src/lib/deep-research/research-manager.ts
@@ -12,7 +12,7 @@ import {
import { ExtractOptions } from "../../controllers/v1/types";
import { getModel } from "../generic-ai";
-import { CostTracking } from "../extract/extraction-service";
+import { CostTracking } from "../cost-tracking";
interface AnalysisResult {
gaps: string[];
nextSteps: string[];
diff --git a/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts b/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts
index 96bc7dca5..b126993c9 100644
--- a/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts
+++ b/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts
@@ -10,7 +10,7 @@ import {
} from "../build-prompts";
import { getModel } from "../../../lib/generic-ai";
import { Logger } from "winston";
-import { CostTracking } from "../extraction-service";
+import { CostTracking } from "../../cost-tracking";
export async function analyzeSchemaAndPrompt(
urls: string[],
schema: any,
diff --git a/apps/api/src/lib/extract/completions/batchExtract.ts b/apps/api/src/lib/extract/completions/batchExtract.ts
index 8895ab655..864bc6cf6 100644
--- a/apps/api/src/lib/extract/completions/batchExtract.ts
+++ b/apps/api/src/lib/extract/completions/batchExtract.ts
@@ -10,7 +10,7 @@ import {
buildBatchExtractSystemPrompt,
} from "../build-prompts";
import { getModel } from "../../generic-ai";
-import { CostTracking, CostLimitExceededError } from "../extraction-service";
+import { CostTracking, CostLimitExceededError } from "../../cost-tracking";
import fs from "fs/promises";
import { extractData } from "../../../scraper/scrapeURL/lib/extractSmartScrape";
import type { Logger } from "winston";
diff --git a/apps/api/src/lib/extract/completions/checkShouldExtract.ts b/apps/api/src/lib/extract/completions/checkShouldExtract.ts
index 82781b062..7120a7675 100644
--- a/apps/api/src/lib/extract/completions/checkShouldExtract.ts
+++ b/apps/api/src/lib/extract/completions/checkShouldExtract.ts
@@ -7,7 +7,7 @@ import {
buildShouldExtractUserPrompt,
} from "../build-prompts";
import { getModel } from "../../../lib/generic-ai";
-import { CostTracking } from "../extraction-service";
+import { CostTracking } from "../../cost-tracking";
export async function checkShouldExtract(
prompt: string,
diff --git a/apps/api/src/lib/extract/completions/singleAnswer.ts b/apps/api/src/lib/extract/completions/singleAnswer.ts
index ba3948965..019474102 100644
--- a/apps/api/src/lib/extract/completions/singleAnswer.ts
+++ b/apps/api/src/lib/extract/completions/singleAnswer.ts
@@ -7,7 +7,7 @@ import { buildDocument } from "../build-document";
import { Document, TokenUsage } from "../../../controllers/v1/types";
import { getModel } from "../../../lib/generic-ai";
import { extractData } from "../../../scraper/scrapeURL/lib/extractSmartScrape";
-import { CostTracking } from "../extraction-service";
+import { CostTracking } from "../../cost-tracking";
export async function singleAnswerCompletion({
singleAnswerDocs,
diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts
index 9f5962033..be9ac4a6b 100644
--- a/apps/api/src/lib/extract/extract-redis.ts
+++ b/apps/api/src/lib/extract/extract-redis.ts
@@ -1,6 +1,6 @@
import { redisEvictConnection } from "../../services/redis";
import { logger as _logger } from "../logger";
-import { CostTracking } from "./extraction-service";
+import { CostTracking } from "../cost-tracking";
export enum ExtractStep {
INITIAL = "initial",
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index 22b4e1da5..fca7153d6 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -24,7 +24,6 @@ const ajv = new Ajv();
import { ExtractStep, updateExtract } from "./extract-redis";
import { deduplicateObjectsArray } from "./helpers/deduplicate-objs-array";
import { mergeNullValObjs } from "./helpers/merge-null-val-objs";
-import { areMergeable } from "./helpers/merge-null-val-objs";
import { CUSTOM_U_TEAMS } from "./config";
import {
calculateFinalResultCost,
@@ -40,8 +39,8 @@ import { normalizeUrl } from "../canonical-url";
import { search } from "../../search";
import { buildRephraseToSerpPrompt } from "./build-prompts";
import { getACUCTeam } from "../../controllers/auth";
-import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { langfuse } from "../../services/langfuse";
+import { CostLimitExceededError, CostTracking } from "../cost-tracking";
interface ExtractServiceOptions {
request: ExtractRequest;
@@ -74,64 +73,6 @@ type completions = {
sources?: string[];
};
-export class CostLimitExceededError extends Error {
- constructor() {
- super("Cost limit exceeded");
- this.message = "Cost limit exceeded";
- this.name = "CostLimitExceededError";
- }
-}
-
-const nanProof = (n: number | null | undefined) =>
- isNaN(n ?? 0) ? 0 : (n ?? 0);
-
-export class CostTracking {
- calls: {
- type: "smartScrape" | "other";
- metadata: Record<string, any>;
- cost: number;
- model: string;
- tokens?: {
- input: number;
- output: number;
- };
- stack: string;
- }[] = [];
- limit: number | null = null;
-
- constructor(limit: number | null = null) {
- this.limit = limit;
- }
-
- public addCall(call: Omit<(typeof this.calls)[number], "stack">) {
- this.calls.push({
- ...call,
- stack: new Error().stack!.split("\n").slice(2).join("\n"),
- });
-
- if (this.limit !== null && this.toJSON().totalCost > this.limit) {
- throw new CostLimitExceededError();
- }
- }
-
- public toJSON() {
- return {
- calls: this.calls,
-
- smartScrapeCallCount: this.calls.filter(c => c.type === "smartScrape")
- .length,
- smartScrapeCost: this.calls
- .filter(c => c.type === "smartScrape")
- .reduce((acc, c) => acc + nanProof(c.cost), 0),
- otherCallCount: this.calls.filter(c => c.type === "other").length,
- otherCost: this.calls
- .filter(c => c.type === "other")
- .reduce((acc, c) => acc + nanProof(c.cost), 0),
- totalCost: this.calls.reduce((acc, c) => acc + nanProof(c.cost), 0),
- };
- }
-}
-
export async function performExtraction(
extractId: string,
options: ExtractServiceOptions,
diff --git a/apps/api/src/lib/extract/fire-0/reranker-f0.ts b/apps/api/src/lib/extract/fire-0/reranker-f0.ts
index bf2ec2e99..d9e8cb5c0 100644
--- a/apps/api/src/lib/extract/fire-0/reranker-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/reranker-f0.ts
@@ -13,7 +13,7 @@ import {
buildRerankerSystemPrompt_F0,
buildRerankerUserPrompt_F0,
} from "./build-prompts-f0";
-import { CostTracking } from "../extraction-service";
+import { CostTracking } from "../../cost-tracking";
const cohere = new CohereClient({
token: process.env.COHERE_API_KEY,
});
diff --git a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts b/apps/api/src/lib/extract/fire-0/url-processor-f0.ts
index dd6a425ff..8f59f516a 100644
--- a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/url-processor-f0.ts
@@ -12,7 +12,7 @@ import { extractConfig } from "../config";
import type { Logger } from "winston";
import { generateText } from "ai";
import { getModel } from "../../generic-ai";
-import { CostTracking } from "../extraction-service";
+import { CostTracking } from "../../cost-tracking";
import { getACUCTeam } from "../../../controllers/auth";
export async function generateBasicCompletion_FO(
diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.ts
index 3b6242795..c06949ab4 100644
--- a/apps/api/src/lib/extract/reranker.ts
+++ b/apps/api/src/lib/extract/reranker.ts
@@ -10,7 +10,7 @@ import { buildRerankerSystemPrompt } from "./build-prompts";
import { dumpToFile } from "./helpers/dump-to-file";
import { getModel } from "../generic-ai";
import fs from "fs/promises";
-import { CostTracking } from "./extraction-service";
+import { CostTracking } from "../cost-tracking";
const THRESHOLD_FOR_SINGLEPAGE = 0.6;
const THRESHOLD_FOR_MULTIENTITY = 0.45;
diff --git a/apps/api/src/lib/extract/url-processor.ts b/apps/api/src/lib/extract/url-processor.ts
index 017604282..7875ff085 100644
--- a/apps/api/src/lib/extract/url-processor.ts
+++ b/apps/api/src/lib/extract/url-processor.ts
@@ -9,7 +9,7 @@ import type { Logger } from "winston";
import { generateText } from "ai";
import { getModel } from "../generic-ai";
import { calculateCost } from "../../scraper/scrapeURL/transformers/llmExtract";
-import type { CostTracking } from "./extraction-service";
+import type { CostTracking } from "../cost-tracking";
export async function generateBasicCompletion(
prompt: string,
diff --git a/apps/api/src/lib/extract/usage/llm-cost.ts b/apps/api/src/lib/extract/usage/llm-cost.ts
index 46dde72f5..4e675acef 100644
--- a/apps/api/src/lib/extract/usage/llm-cost.ts
+++ b/apps/api/src/lib/extract/usage/llm-cost.ts
@@ -1,6 +1,6 @@
import { TokenUsage } from "../../../controllers/v1/types";
import { logger } from "../../../lib/logger";
-import { CostTracking } from "../extraction-service";
+import { CostTracking } from "../../cost-tracking";
import { modelPrices } from "./model-prices";
interface ModelPricing {
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
index 7e16fde7d..92d681e90 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
@@ -11,7 +11,7 @@ import { billTeam } from "../../services/billing/credit_billing";
import { logJob } from "../../services/logging/log_job";
import { getModel } from "../generic-ai";
import { generateCompletions } from "../../scraper/scrapeURL/transformers/llmExtract";
-import { CostTracking } from "../extract/extraction-service";
+import { CostTracking } from "../cost-tracking";
import { getACUCTeam } from "../../controllers/auth";
interface GenerateLLMsTextServiceOptions {
generationId: string;
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 300909b4a..40712f183 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -1,7 +1,13 @@
import robotsParser, { Robot } from "robots-parser";
-import * as undici from "undici";
import { Logger } from "winston";
-import { getSecureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";
+import { ScrapeOptions, scrapeOptions } from "../controllers/v2/types";
+import { scrapeURL } from "../scraper/scrapeURL";
+import { Engine } from "../scraper/scrapeURL/engines";
+import { CostTracking } from "./cost-tracking";
+
+const useFireEngine =
+ process.env.FIRE_ENGINE_BETA_URL !== "" &&
+ process.env.FIRE_ENGINE_BETA_URL !== undefined;
export interface RobotsTxtChecker {
robotsTxtUrl: string;
@@ -10,36 +16,88 @@ export interface RobotsTxtChecker {
}
export async function fetchRobotsTxt(
- url: string,
- skipTlsVerification: boolean = false,
+ {
+ url,
+ zeroDataRetention,
+ location,
+ }: {
+ url: string;
+ zeroDataRetention: boolean;
+ location?: ScrapeOptions["location"];
+ },
+ scrapeId: string,
+ logger: Logger,
abort?: AbortSignal,
-): Promise<string> {
+): Promise<{ content: string; url: string }> {
const urlObj = new URL(url);
const robotsTxtUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
- const response = await undici.fetch(robotsTxtUrl, {
- signal: abort,
- dispatcher: getSecureDispatcher(skipTlsVerification),
- });
-
- if (!response.ok) {
- throw new Error(
- `Failed to fetch robots.txt: ${response.status} ${response.statusText}`,
- );
- }
-
- const content = await response.text();
- const contentType = response.headers.get("content-type") || "";
+ const shouldPrioritizeFireEngine = location && useFireEngine;
+
+ const forceEngine: Engine[] = [
+ ...(shouldPrioritizeFireEngine
+ ? [
+ "fire-engine;tlsclient" as const,
+ "fire-engine;tlsclient;stealth" as const,
+ ]
+ : []),
+ "fetch",
+ ...(!shouldPrioritizeFireEngine && useFireEngine
+ ? [
+ "fire-engine;tlsclient" as const,
+ "fire-engine;tlsclient;stealth" as const,
+ ]
+ : []),
+ ];
+
+ let content: string = "";
+ const response = await scrapeURL(
+ "robots-txt;" + scrapeId,
+ robotsTxtUrl,
+ scrapeOptions.parse({
+ formats: ["rawHtml"],
+ ...(location ? { location } : {}),
+ }),
+ {
+ forceEngine,
+ v0DisableJsDom: true,
+ externalAbort: abort
+ ? {
+ signal: abort,
+ tier: "external",
+ throwable() {
+ return new Error("Robots.txt fetch aborted");
+ },
+ }
+ : undefined,
+ teamId: "robots-txt",
+ zeroDataRetention,
+ },
+ new CostTracking(),
+ );
if (
- (contentType.includes("text/html") && content.trim().startsWith("<")) ||
- contentType.includes("application/json") ||
- contentType.includes("application/xml")
+ response.success &&
+ response.document.metadata.statusCode >= 200 &&
+ response.document.metadata.statusCode < 300
) {
- return "";
+ content = response.document.rawHtml!;
+ } else {
+ logger.error(`Request failed for robots.txt fetch`, {
+ method: "fetchRobotsTxt",
+ robotsTxtUrl,
+ error: response.success
+ ? response.document.metadata.statusCode
+ : response.error,
+ });
+ return { content: "", url: robotsTxtUrl };
}
- return content;
+ // return URL in case we've been redirected
+ return {
+ content: content,
+ url: response.document.metadata.url || robotsTxtUrl,
+ };
}
export function createRobotsChecker(
@@ -99,23 +157,3 @@ export function isUrlAllowedByRobots(
return false;
}
-
-export async function checkRobotsTxt(
- url: string,
- skipTlsVerification: boolean = false,
- logger?: Logger,
- abort?: AbortSignal,
-): Promise<boolean> {
- try {
- const robotsTxt = await fetchRobotsTxt(url, skipTlsVerification, abort);
- const checker = createRobotsChecker(url, robotsTxt);
- return isUrlAllowedByRobots(url, checker.robots);
- } catch (error) {
- // If we can't fetch robots.txt, assume it's allowed
- logger?.debug("Failed to fetch robots.txt, allowing scrape", {
- error,
- url,
- });
- return true;
- }
-}
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index c2358558c..67f174af4 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -5,7 +5,7 @@ import {
TeamFlags,
shouldParsePDF,
} from "../controllers/v2/types";
-import { CostTracking } from "./extract/extraction-service";
+import { CostTracking } from "./cost-tracking";
import { hasFormatOfType } from "./format-utils";
const creditsPerPDFPage = 1;
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index b7a5ad560..e500ebd4f 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -4,7 +4,7 @@ import { supabase_service } from "../services/supabase";
import { logger as _logger } from "../lib/logger";
import { configDotenv } from "dotenv";
import { scrapeURL, ScrapeUrlResponse } from "../scraper/scrapeURL";
-import { CostTracking } from "../lib/extract/extraction-service";
+import { CostTracking } from "../lib/cost-tracking";
configDotenv();
export async function startWebScraperPipeline({
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 0f796ea98..0f060e5ca 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -15,6 +15,8 @@ import {
isUrlAllowedByRobots,
} from "../../lib/robots-txt";
import { ScrapeJobTimeoutError } from "../../lib/error";
+import { Logger } from "winston";
+import { ScrapeOptions } from "../../controllers/v2/types";
export const SITEMAP_LIMIT = 100;
@@ -67,6 +69,7 @@ export class WebCrawler {
private maxDiscoveryDepth: number | undefined;
private currentDiscoveryDepth: number;
private zeroDataRetention: boolean;
+ private location?: ScrapeOptions["location"];
constructor({
jobId,
@@ -86,6 +89,7 @@ export class WebCrawler {
maxDiscoveryDepth,
currentDiscoveryDepth,
zeroDataRetention,
+ location,
}: {
jobId: string;
initialUrl: string;
@@ -104,6 +108,7 @@ export class WebCrawler {
maxDiscoveryDepth?: number;
currentDiscoveryDepth?: number;
zeroDataRetention?: boolean;
+ location?: ScrapeOptions["location"];
}) {
this.jobId = jobId;
this.initialUrl = initialUrl;
@@ -131,6 +136,7 @@ export class WebCrawler {
});
this.maxDiscoveryDepth = maxDiscoveryDepth;
this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0;
+ this.location = location;
}
public async filterLinks(
@@ -340,9 +346,14 @@ export class WebCrawler {
skipTlsVerification,
});
- const robotsTxt = await fetchRobotsTxt(
- this.initialUrl,
- skipTlsVerification,
+ const { content: robotsTxt, url } = await fetchRobotsTxt(
+ {
+ url: this.robotsTxtUrl,
+ zeroDataRetention: this.zeroDataRetention,
+ location: this.location,
+ },
+ this.jobId,
+ this.logger,
abort,
);
@@ -351,6 +362,7 @@ export class WebCrawler {
initialUrl: this.initialUrl,
robotsTxtLength: robotsTxt.length,
hasContent: robotsTxt.length > 0,
+ finalUrl: url,
});
return robotsTxt;
@@ -843,6 +855,7 @@ export class WebCrawler {
mode: "fire-engine",
maxAge,
zeroDataRetention: this.zeroDataRetention,
+ location: this.location,
},
this.logger,
this.jobId,
@@ -893,6 +906,7 @@ export class WebCrawler {
mode: "fire-engine",
maxAge,
zeroDataRetention: this.zeroDataRetention,
+ location: this.location,
},
this.logger,
this.jobId,
@@ -934,6 +948,7 @@ export class WebCrawler {
mode: "fire-engine",
maxAge,
zeroDataRetention: this.zeroDataRetention,
+ location: this.location,
},
this.logger,
this.jobId,
@@ -960,6 +975,7 @@ export class WebCrawler {
mode: "fire-engine",
maxAge,
zeroDataRetention: this.zeroDataRetention,
+ location: this.location,
},
this.logger,
this.jobId,
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 43621aaa9..4616df85b 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -3,9 +3,11 @@ import { WebCrawler, SITEMAP_LIMIT } from "./crawler";
import { scrapeURL } from "../scrapeURL";
import { scrapeOptions } from "../../controllers/v2/types";
import type { Logger } from "winston";
-import { CostTracking } from "../../lib/extract/extraction-service";
+import { CostTracking } from "../../lib/cost-tracking";
import { parseSitemapXml, processSitemap } from "../../lib/crawler";
import { ScrapeJobTimeoutError } from "../../lib/error";
+import type { ScrapeOptions } from "../../controllers/v2/types";
+import { Engine } from "../scrapeURL/engines";
const useFireEngine =
process.env.FIRE_ENGINE_BETA_URL !== "" &&
process.env.FIRE_ENGINE_BETA_URL !== undefined;
@@ -17,12 +19,14 @@ export async function getLinksFromSitemap(
mode = "axios",
maxAge = 0,
zeroDataRetention,
+ location,
}: {
sitemapUrl: string;
urlsHandler(urls: string[]): unknown;
mode?: "axios" | "fire-engine";
maxAge?: number;
zeroDataRetention: boolean;
+ location?: ScrapeOptions["location"];
},
logger: Logger,
crawlId: string,
@@ -44,21 +48,39 @@ export async function getLinksFromSitemap(
try {
let content: string = "";
try {
+ const shouldPrioritizeFireEngine =
+ location && mode === "fire-engine" && useFireEngine;
+
+ const forceEngine: Engine[] = [
+ ...(maxAge > 0 ? ["index" as const] : []),
+ ...(shouldPrioritizeFireEngine
+ ? [
+ "fire-engine;tlsclient" as const,
+ "fire-engine;tlsclient;stealth" as const,
+ ]
+ : []),
+ "fetch",
+ ...(!shouldPrioritizeFireEngine &&
+ mode === "fire-engine" &&
+ useFireEngine
+ ? [
+ "fire-engine;tlsclient" as const,
+ "fire-engine;tlsclient;stealth" as const,
+ ]
+ : []),
+ ];
+
const response = await scrapeURL(
"sitemap;" + crawlId,
sitemapUrl,
- scrapeOptions.parse({ formats: ["rawHtml"], useMock: mock, maxAge }),
+ scrapeOptions.parse({
+ formats: ["rawHtml"],
+ useMock: mock,
+ maxAge,
+ ...(location ? { location } : {}),
+ }),
{
- forceEngine: [
- ...(maxAge > 0 ? ["index" as const] : []),
- "fetch",
- ...(mode === "fire-engine" && useFireEngine
- ? [
- "fire-engine;tlsclient" as const,
- "fire-engine;tlsclient;stealth" as const,
- ]
- : []),
- ],
+ forceEngine,
v0DisableJsDom: true,
externalAbort: abort
? {
@@ -145,7 +167,7 @@ export async function getLinksFromSitemap(
const sitemapPromises: Promise<number>[] = sitemapUrls.map(sitemapUrl =>
getLinksFromSitemap(
- { sitemapUrl, urlsHandler, mode, zeroDataRetention },
+ { sitemapUrl, urlsHandler, mode, zeroDataRetention, location },
logger,
crawlId,
sitemapsHit,
@@ -169,7 +191,13 @@ export async function getLinksFromSitemap(
if (xmlSitemaps.length > 0) {
const sitemapPromises = xmlSitemaps.map(sitemapUrl =>
getLinksFromSitemap(
- { sitemapUrl: sitemapUrl, urlsHandler, mode, zeroDataRetention },
+ {
+ sitemapUrl: sitemapUrl,
+ urlsHandler,
+ mode,
+ zeroDataRetention,
+ location,
+ },
logger,
crawlId,
sitemapsHit,
@@ -209,7 +237,7 @@ export async function getLinksFromSitemap(
const sitemapPromises: Promise<number>[] = instruction.urls.map(
sitemapUrl =>
getLinksFromSitemap(
- { sitemapUrl, urlsHandler, mode, zeroDataRetention },
+ { sitemapUrl, urlsHandler, mode, zeroDataRetention, location },
logger,
crawlId,
sitemapsHit,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 5821a3615..c1d81fa79 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -43,7 +43,7 @@ import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
import { urlSpecificParams } from "./lib/urlSpecificParams";
import { loadMock, MockState } from "./lib/mock";
-import { CostTracking } from "../../lib/extract/extraction-service";
+import { CostTracking } from "../../lib/cost-tracking";
import {
addIndexRFInsertJob,
generateDomainSplits,
@@ -52,7 +52,11 @@ import {
normalizeURLForIndex,
useIndex,
} from "../../services/index";
-import { checkRobotsTxt } from "../../lib/robots-txt";
+import {
+ fetchRobotsTxt,
+ createRobotsChecker,
+ isUrlAllowedByRobots,
+} from "../../lib/robots-txt";
import {
AbortInstance,
AbortManager,
@@ -674,24 +678,41 @@ export async function scrapeURL(
}
if (internalOptions.teamFlags?.checkRobotsOnScrape) {
- meta.logger.info("Checking robots.txt", {
- checkRobotsOnScrape: internalOptions.teamFlags?.checkRobotsOnScrape,
- url: meta.rewrittenUrl || meta.url,
- });
const urlToCheck = meta.rewrittenUrl || meta.url;
- const isAllowed = await checkRobotsTxt(
- urlToCheck,
- options.skipTlsVerification,
- meta.logger,
- meta.abort.asSignal(),
- );
+ meta.logger.info("Checking robots.txt", { url: urlToCheck });
+
+ const urlObj = new URL(urlToCheck);
+ const isRobotsTxtPath = urlObj.pathname === "/robots.txt";
+
+ if (!isRobotsTxtPath) {
+ try {
+ const { content: robotsTxt } = await fetchRobotsTxt(
+ {
+ url: urlToCheck,
+ zeroDataRetention: internalOptions.zeroDataRetention || false,
+ location: options.location,
+ },
+ id,
+ meta.logger,
+ meta.abort.asSignal(),
+ );
- if (!isAllowed) {
- meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });
- return {
- success: false,
- error: new Error("URL blocked by robots.txt"),
- };
+ const checker = createRobotsChecker(urlToCheck, robotsTxt);
+ const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);
+
+ if (!isAllowed) {
+ meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });
+ return {
+ success: false,
+ error: new Error("URL blocked by robots.txt"),
+ };
+ }
+ } catch (error) {
+ meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {
+ error,
+ url: urlToCheck,
+ });
+ }
}
}
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 6becda9eb..e04a93f46 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -13,7 +13,7 @@ import type { SmartScrapeResult } from "./smartScrape";
import {
CostLimitExceededError,
CostTracking,
-} from "../../../lib/extract/extraction-service";
+} from "../../../lib/cost-tracking";
const commonSmartScrapeProperties = {
shouldUseSmartscrape: {
type: "boolean",
diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
index 7c4cbe341..f46c1a753 100644
--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
@@ -6,7 +6,7 @@ import { configDotenv } from "dotenv";
import {
CostLimitExceededError,
CostTracking,
-} from "../../../lib/extract/extraction-service";
+} from "../../../lib/cost-tracking";
configDotenv();
// Define schemas outside the function scope
diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
index 79a8dc477..c379b92ab 100644
--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
@@ -5,7 +5,7 @@ process.env.ENV = "test";
import { scrapeURL } from ".";
import { scrapeOptions } from "../../controllers/v2/types";
import { Engine } from "./engines";
-import { CostTracking } from "../../lib/extract/extraction-service";
+import { CostTracking } from "../../lib/cost-tracking";
const testEngines: (Engine | undefined)[] = [
undefined,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index aaede1997..9a559c6d1 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -22,7 +22,7 @@ import { z } from "zod";
import fs from "fs/promises";
import Ajv from "ajv";
import { extractData } from "../lib/extractSmartScrape";
-import { CostTracking } from "../../../lib/extract/extraction-service";
+import { CostTracking } from "../../../lib/cost-tracking";
import { isAgentExtractModelValid } from "../../../controllers/v1/types";
import { hasFormatOfType } from "../../../lib/format-utils";
// TODO: fix this, it's horrible
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 664761323..1c79ccd4a 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -34,7 +34,7 @@ import { getACUCTeam } from "../../controllers/auth";
import { callWebhook } from "../webhook";
import { CustomError } from "../../lib/custom-error";
import { startWebScraperPipeline } from "../../main/runWebScraper";
-import { CostTracking } from "../../lib/extract/extraction-service";
+import { CostTracking } from "../../lib/cost-tracking";
import { redisEvictConnection } from "../redis";
import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 4a46da35b..834e81dd7 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -9,7 +9,7 @@ import {
import { AuthCreditUsageChunk } from "./controllers/v1/types";
import { ExtractorOptions, Document } from "./lib/entities";
import { InternalOptions } from "./scraper/scrapeURL";
-import type { CostTracking } from "./lib/extract/extraction-service";
+import type { CostTracking } from "./lib/cost-tracking";
type Mode = "crawl" | "single_urls" | "sitemap" | "kickoff";
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 081c80e68..888c1cc75 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.3.1",
+ "version": "4.3.2",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index dfa24cd72..659f6133e 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -102,6 +102,14 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult
description?: string;
}
+/**
+ * Location configuration for proxy location
+ */
+export interface LocationConfig {
+ country?: string;
+ languages?: string[];
+}
+
/**
* Parameters for scraping operations.
* Defines the options and configurations available for scraping web content.
@@ -114,10 +122,7 @@ export interface CrawlScrapeOptions {
onlyMainContent?: boolean;
waitFor?: number;
timeout?: number;
- location?: {
- country?: string;
- languages?: string[];
- };
+ location?: LocationConfig;
mobile?: boolean;
skipTlsVerification?: boolean;
removeBase64Images?: boolean;
@@ -300,6 +305,7 @@ export interface MapParams {
limit?: number;
timeout?: number;
useIndex?: boolean;
+ location?: LocationConfig;
}
/**
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/map.ts b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
index 8b41f772e..7eb5e42c7 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/map.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
@@ -11,6 +11,7 @@ function prepareMapPayload(url: string, options?: MapOptions): Record<string, un
if (options.includeSubdomains != null) payload.includeSubdomains = options.includeSubdomains;
if (options.limit != null) payload.limit = options.limit;
if (options.timeout != null) payload.timeout = options.timeout;
+ if (options.location != null) payload.location = options.location;
}
return payload;
}
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 31434a1ff..6f9ad92eb 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -319,6 +319,7 @@ export interface MapOptions {
includeSubdomains?: boolean;
limit?: number;
timeout?: number;
+ location?: LocationConfig;
}
export interface ExtractResponse {
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index adba0b03e..6778ceaf3 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.3.1"
+__version__ = "4.3.2"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.py
index b6c94eca2..bc0af1ebf 100644
--- a/apps/python-sdk/firecrawl/v1/client.py
+++ b/apps/python-sdk/firecrawl/v1/client.py
@@ -309,6 +309,7 @@ class V1MapParams(pydantic.BaseModel):
limit: Optional[int] = None
timeout: Optional[int] = 30000
useIndex: Optional[bool] = None
+ location: Optional[V1LocationConfig] = None
class V1MapResponse(pydantic.BaseModel):
"""Response from mapping operations."""
@@ -1333,6 +1334,7 @@ class V1FirecrawlApp:
limit: Optional[int] = None,
timeout: Optional[int] = 30000,
use_index: Optional[bool] = None,
+ location: Optional[V1LocationConfig] = None,
**kwargs) -> V1MapResponse:
"""
Map and discover links from a URL.
@@ -1377,6 +1379,8 @@ class V1FirecrawlApp:
map_params['timeout'] = timeout
if use_index is not None:
map_params['useIndex'] = use_index
+ if location is not None:
+ map_params['location'] = location.dict(by_alias=True, exclude_none=True)
# Add any additional kwargs
map_params.update(kwargs)
@@ -3910,6 +3914,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
sitemap_only: Optional[bool] = None,
limit: Optional[int] = None,
timeout: Optional[int] = 30000,
+ location: Optional[V1LocationConfig] = None,
params: Optional[V1MapParams] = None) -> V1MapResponse:
"""
Asynchronously map and discover links from a URL.
@@ -3952,6 +3957,8 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
map_params['limit'] = limit
if timeout is not None:
map_params['timeout'] = timeout
+ if location is not None:
+ map_params['location'] = location.dict(by_alias=True, exclude_none=True)
# Create final params object
final_params = V1MapParams(**map_params)
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 5c4e3f867..9073460ee 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -421,6 +421,7 @@ class FirecrawlClient:
limit: Optional[int] = None,
sitemap: Optional[Literal["only", "include", "skip"]] = None,
timeout: Optional[int] = None,
+ location: Optional[Location] = None,
) -> MapData:
"""Map a URL and return discovered links.
@@ -441,7 +442,8 @@ class FirecrawlClient:
limit=limit,
sitemap=sitemap if sitemap is not None else "include",
timeout=timeout,
- ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, timeout]) else None
+ location=location
+ ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, timeout, location]) else None
return map_module.map(self.http_client, url, options)
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/map.py b/apps/python-sdk/firecrawl/v2/methods/aio/map.py
index b9ca22f58..46d9fa898 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/map.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/map.py
@@ -20,6 +20,8 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict
data["limit"] = options.limit
if options.timeout is not None:
data["timeout"] = options.timeout
+ if options.location is not None:
+ data["location"] = options.location.model_dump(exclude_none=True)
payload.update(data)
return payload
diff --git a/apps/python-sdk/firecrawl/v2/methods/map.py b/apps/python-sdk/firecrawl/v2/methods/map.py
index 2c08e5be6..83343d24f 100644
--- a/apps/python-sdk/firecrawl/v2/methods/map.py
+++ b/apps/python-sdk/firecrawl/v2/methods/map.py
@@ -27,6 +27,8 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict
data["limit"] = options.limit
if options.timeout is not None:
data["timeout"] = options.timeout
+ if options.location is not None:
+ data["location"] = options.location.model_dump(exclude_none=True)
payload.update(data)
return payload
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index e0a8a2794..77f61c4a5 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -445,6 +445,7 @@ class MapOptions(BaseModel):
include_subdomains: Optional[bool] = None
limit: Optional[int] = None
timeout: Optional[int] = None
+ location: Optional['Location'] = None
class MapRequest(BaseModel):
"""Request for mapping a website."""
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index 11637503b..a34122c4c 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "firecrawl"
author= "Mendable.ai"
-version = "1.2.1"
+version = "1.2.2"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rs
index 44016064d..386cb098e 100644
--- a/apps/rust-sdk/src/map.rs
+++ b/apps/rust-sdk/src/map.rs
@@ -2,6 +2,17 @@ use serde::{Deserialize, Serialize};
use crate::{FirecrawlApp, FirecrawlError, API_VERSION};
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Location {
+ /// Country code (ISO 3166-1 alpha-2)
+ pub country: Option<String>,
+
+ /// List of language codes
+ pub languages: Option<Vec<String>>,
+}
+
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
@@ -17,6 +28,9 @@ pub struct MapOptions {
/// Maximum number of links to return (default: `5000`)
pub limit: Option<u32>,
+
+ /// Location configuration for proxy location
+ pub location: Option<Location>,
}
#[derive(Deserialize, Serialize, Debug, Default)]