Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add proxy location support to crawl and map endpoints (ENG-3361) (#2092)
tom9a3bd6ca5017
diff --git a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts b/apps/api/src/__tests__/snips/v2/robots-txt.test.tsindex b9c9c5f1d..e96275c21 100644--- a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts+++ b/apps/api/src/__tests__/snips/v2/robots-txt.test.ts@@ -8,7 +8,16 @@ describe("Robots.txt tests", () => {let checker: ReturnType<typeof createRobotsChecker>;beforeAll(async () => {- robotsTxt = await fetchRobotsTxt(robotsUrl);+ const logger = {+ error: jest.fn(),+ } as any;+ robotsTxt = (+ await fetchRobotsTxt(+ { url: robotsUrl, zeroDataRetention: true },+ "test-scrape",+ logger,+ )+ ).content;checker = createRobotsChecker(siteUrl, robotsTxt);}, scrapeTimeout);diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 51cac4173..45593c2ca 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -6,7 +6,7 @@ import {RequestWithAuth,TeamFlags,} from "./types";-import { scrapeOptions } from "../v2/types";+import { scrapeOptions, ScrapeOptions } from "../v2/types";import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";import { MapResponse, MapRequest, MAX_MAP_LIMIT } from "./types";import { configDotenv } from "dotenv";@@ -94,6 +94,7 @@ export async function getMapResults({flags,useIndex = true,timeout,+ location,}: {url: string;search?: string;@@ -111,6 +112,7 @@ export async function getMapResults({flags: TeamFlags;useIndex?: boolean;timeout?: number;+ location?: ScrapeOptions["location"];}): Promise<MapResult> {const id = uuidv4();let links: string[] = [url];@@ -125,7 +127,9 @@ export async function getMapResults({limit: crawlerOptions.sitemapOnly ? 10000000 : limit,scrapeOptions: undefined,},- scrapeOptions: scrapeOptions.parse({}),+ scrapeOptions: scrapeOptions.parse({+ ...(location ? { location } : {}),+ }),internalOptions: { teamId },team_id: teamId,createdAt: Date.now(),@@ -381,6 +385,7 @@ export async function mapController(flags: req.acuc?.flags ?? null,useIndex: req.body.useIndex,timeout: req.body.timeout,+ location: req.body.location,}),...(req.body.timeout !== undefined? [diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 46159a83b..89a8a8e11 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -22,7 +22,7 @@ import * as Sentry from "@sentry/node";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";-import { CostTracking } from "../../lib/extract/extraction-service";+import { CostTracking } from "../../lib/cost-tracking";import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";import { fromV1ScrapeOptions } from "../v2/types";diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 7d1ee1c77..b3458abd6 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -284,6 +284,26 @@ export const actionsSchema = z},);+export const locationSchema = z+ .object({+ country: z+ .string()+ .optional()+ .refine(+ val =>+ !val ||+ Object.keys(countries).includes(val.toUpperCase()) ||+ val === "US-generic",+ {+ message:+ "Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",+ },+ )+ .transform(val => (val ? val.toUpperCase() : "US-generic")),+ languages: z.string().array().optional(),+ })+ .optional();+function transformIframeSelector(selector: string): string {return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, match => {const prefix = match.match(/^[\s,]/)?.[0] || "";@@ -370,25 +390,7 @@ const baseScrapeOptions = zparsePDF: z.boolean().default(true),actions: actionsSchema.optional(),// New- location: z- .object({- country: z- .string()- .optional()- .refine(- val =>- !val ||- Object.keys(countries).includes(val.toUpperCase()) ||- val === "US-generic",- {- message:- "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",- },- )- .transform(val => (val ? val.toUpperCase() : "US-generic")),- languages: z.string().array().optional(),- })- .optional(),+ location: locationSchema,// Deprecatedgeolocation: z@@ -866,6 +868,7 @@ export const mapRequestSchema = crawlerOptionsuseMock: z.string().optional(),filterByPath: z.boolean().default(true),useIndex: z.boolean().default(true),+ location: locationSchema,}).strict(strictMessage);diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 6345f6856..706fed9de 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -20,7 +20,7 @@ import * as Sentry from "@sentry/node";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";-import { CostTracking } from "../../lib/extract/extraction-service";+import { CostTracking } from "../../lib/cost-tracking";import { supabase_service } from "../../services/supabase";import { fromV1ScrapeOptions } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsindex 9c5d65ecc..2b556a20a 100644--- a/apps/api/src/controllers/v2/crawl-params-preview.ts+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -3,7 +3,7 @@ import { z } from "zod";import { ErrorResponse, RequestWithAuth, crawlRequestSchema } from "./types";import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";-import { CostTracking } from "../../lib/extract/extraction-service";+import { CostTracking } from "../../lib/cost-tracking";// Define the request schema for params preview// Only url and prompt are required/relevant for previewdiff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex aad3cae4b..337e6f806 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -11,7 +11,7 @@ import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";-import { CostTracking } from "../../lib/extract/extraction-service";+import { CostTracking } from "../../lib/cost-tracking";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex a7e565981..7df9e6fcf 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -4,6 +4,7 @@ import {mapRequestSchema,RequestWithAuth,scrapeOptions,+ ScrapeOptions,TeamFlags,MapRequest,MapDocument,@@ -96,6 +97,7 @@ export async function getMapResults({filterByPath = true,flags,useIndex = true,+ location,}: {url: string;search?: string;@@ -111,6 +113,7 @@ export async function getMapResults({filterByPath?: boolean;flags: TeamFlags;useIndex?: boolean;+ location?: ScrapeOptions["location"];}): Promise<MapResult> {const id = uuidv4();let mapResults: MapDocument[] = [];@@ -123,7 +126,9 @@ export async function getMapResults({limit: crawlerOptions.sitemapOnly ? 10000000 : limit,scrapeOptions: undefined,},- scrapeOptions: scrapeOptions.parse({}),+ scrapeOptions: scrapeOptions.parse({+ ...(location ? { location } : {}),+ }),internalOptions: { teamId },team_id: teamId,createdAt: Date.now(),@@ -375,6 +380,7 @@ export async function mapController(filterByPath: req.body.filterByPath !== false,flags: req.acuc?.flags ?? null,useIndex: req.body.useIndex,+ location: req.body.location,}),...(req.body.timeout !== undefined? [diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 052167f00..c958cf63b 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -20,7 +20,7 @@ import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";-import { CostTracking } from "../../lib/extract/extraction-service";+import { CostTracking } from "../../lib/cost-tracking";import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";import { SearchResult, SearchV2Response } from "../../lib/entities";diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 8ccff9d89..09348bfd8 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -332,6 +332,26 @@ function transformIframeSelector(selector: string): string {});}+const locationSchema = z+ .object({+ country: z+ .string()+ .optional()+ .refine(+ val =>+ !val ||+ Object.keys(countries).includes(val.toUpperCase()) ||+ val === "US-generic",+ {+ message:+ "Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",+ },+ )+ .transform(val => (val ? val.toUpperCase() : "US-generic")),+ languages: z.string().array().optional(),+ })+ .optional();+const baseScrapeOptions = z.object({formats: z@@ -395,25 +415,7 @@ const baseScrapeOptions = zparsers: parsersSchema.optional(),actions: actionsSchema.optional(),- location: z- .object({- country: z- .string()- .optional()- .refine(- val =>- !val ||- Object.keys(countries).includes(val.toUpperCase()) ||- val === "US-generic",- {- message:- "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",- },- )- .transform(val => (val ? val.toUpperCase() : "US-generic")),- languages: z.string().array().optional(),- })- .optional(),+ location: locationSchema,skipTlsVerification: z.boolean().default(true),removeBase64Images: z.boolean().default(true),@@ -732,6 +734,7 @@ export const mapRequestSchema = crawlerOptionsuseMock: z.string().optional(),filterByPath: z.boolean().default(true),useIndex: z.boolean().default(true),+ location: locationSchema,}).strict(strictMessage);diff --git a/apps/api/src/lib/cost-tracking.ts b/apps/api/src/lib/cost-tracking.tsnew file mode 100644index 000000000..a41d7f6a4--- /dev/null+++ b/apps/api/src/lib/cost-tracking.ts@@ -0,0 +1,57 @@+export class CostLimitExceededError extends Error {+ constructor() {+ super("Cost limit exceeded");+ this.message = "Cost limit exceeded";+ this.name = "CostLimitExceededError";+ }+}++const nanProof = (n: number | null | undefined) =>+ isNaN(n ?? 0) ? 0 : (n ?? 0);++export class CostTracking {+ calls: {+ type: "smartScrape" | "other";+ metadata: Record<string, any>;+ cost: number;+ model: string;+ tokens?: {+ input: number;+ output: number;+ };+ stack: string;+ }[] = [];+ limit: number | null = null;++ constructor(limit: number | null = null) {+ this.limit = limit;+ }++ public addCall(call: Omit<(typeof this.calls)[number], "stack">) {+ this.calls.push({+ ...call,+ stack: new Error().stack!.split("\n").slice(2).join("\n"),+ });++ if (this.limit !== null && this.toJSON().totalCost > this.limit) {+ throw new CostLimitExceededError();+ }+ }++ public toJSON() {+ return {+ calls: this.calls,++ smartScrapeCallCount: this.calls.filter(c => c.type === "smartScrape")+ .length,+ smartScrapeCost: this.calls+ .filter(c => c.type === "smartScrape")+ .reduce((acc, c) => acc + nanProof(c.cost), 0),+ otherCallCount: this.calls.filter(c => c.type === "other").length,+ otherCost: this.calls+ .filter(c => c.type === "other")+ .reduce((acc, c) => acc + nanProof(c.cost), 0),+ totalCost: this.calls.reduce((acc, c) => acc + nanProof(c.cost), 0),+ };+ }+}diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex ed1a5a621..0ffc6a97a 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -530,6 +530,7 @@ export function crawlToCrawler(maxDiscoveryDepth: sc.crawlerOptions?.maxDiscoveryDepth,currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,zeroDataRetention: (teamFlags?.forceZDR || sc.zeroDataRetention) ?? false,+ location: sc.scrapeOptions?.location,});if (sc.robots !== undefined) {diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.tsindex d3bfaca88..38d67ae56 100644--- a/apps/api/src/lib/deep-research/deep-research-service.ts+++ b/apps/api/src/lib/deep-research/deep-research-service.ts@@ -5,7 +5,7 @@ import { ResearchLLMService, ResearchStateManager } from "./research-manager";import { logJob } from "../../services/logging/log_job";import { billTeam } from "../../services/billing/credit_billing";import { ExtractOptions } from "../../controllers/v1/types";-import { CostTracking } from "../extract/extraction-service";+import { CostTracking } from "../cost-tracking";import { getACUCTeam } from "../../controllers/auth";export interface DeepResearchServiceOptions {researchId: string;diff --git a/apps/api/src/lib/deep-research/research-manager.ts b/apps/api/src/lib/deep-research/research-manager.tsindex 8b7b28436..81389267d 100644--- a/apps/api/src/lib/deep-research/research-manager.ts+++ b/apps/api/src/lib/deep-research/research-manager.ts@@ -12,7 +12,7 @@ import {import { ExtractOptions } from "../../controllers/v1/types";import { getModel } from "../generic-ai";-import { CostTracking } from "../extract/extraction-service";+import { CostTracking } from "../cost-tracking";interface AnalysisResult {gaps: string[];nextSteps: string[];diff --git a/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts b/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.tsindex 96bc7dca5..b126993c9 100644--- a/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts+++ b/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts@@ -10,7 +10,7 @@ import {} from "../build-prompts";import { getModel } from "../../../lib/generic-ai";import { Logger } from "winston";-import { CostTracking } from "../extraction-service";+import { CostTracking } from "../../cost-tracking";export async function analyzeSchemaAndPrompt(urls: string[],schema: any,diff --git a/apps/api/src/lib/extract/completions/batchExtract.ts b/apps/api/src/lib/extract/completions/batchExtract.tsindex 8895ab655..864bc6cf6 100644--- a/apps/api/src/lib/extract/completions/batchExtract.ts+++ b/apps/api/src/lib/extract/completions/batchExtract.ts@@ -10,7 +10,7 @@ import {buildBatchExtractSystemPrompt,} from "../build-prompts";import { getModel } from "../../generic-ai";-import { CostTracking, CostLimitExceededError } from "../extraction-service";+import { CostTracking, CostLimitExceededError } from "../../cost-tracking";import fs from "fs/promises";import { extractData } from "../../../scraper/scrapeURL/lib/extractSmartScrape";import type { Logger } from "winston";diff --git a/apps/api/src/lib/extract/completions/checkShouldExtract.ts b/apps/api/src/lib/extract/completions/checkShouldExtract.tsindex 82781b062..7120a7675 100644--- a/apps/api/src/lib/extract/completions/checkShouldExtract.ts+++ b/apps/api/src/lib/extract/completions/checkShouldExtract.ts@@ -7,7 +7,7 @@ import {buildShouldExtractUserPrompt,} from "../build-prompts";import { getModel } from "../../../lib/generic-ai";-import { CostTracking } from "../extraction-service";+import { CostTracking } from "../../cost-tracking";export async function checkShouldExtract(prompt: string,diff --git a/apps/api/src/lib/extract/completions/singleAnswer.ts b/apps/api/src/lib/extract/completions/singleAnswer.tsindex ba3948965..019474102 100644--- a/apps/api/src/lib/extract/completions/singleAnswer.ts+++ b/apps/api/src/lib/extract/completions/singleAnswer.ts@@ -7,7 +7,7 @@ import { buildDocument } from "../build-document";import { Document, TokenUsage } from "../../../controllers/v1/types";import { getModel } from "../../../lib/generic-ai";import { extractData } from "../../../scraper/scrapeURL/lib/extractSmartScrape";-import { CostTracking } from "../extraction-service";+import { CostTracking } from "../../cost-tracking";export async function singleAnswerCompletion({singleAnswerDocs,diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.tsindex 9f5962033..be9ac4a6b 100644--- a/apps/api/src/lib/extract/extract-redis.ts+++ b/apps/api/src/lib/extract/extract-redis.ts@@ -1,6 +1,6 @@import { redisEvictConnection } from "../../services/redis";import { logger as _logger } from "../logger";-import { CostTracking } from "./extraction-service";+import { CostTracking } from "../cost-tracking";export enum ExtractStep {INITIAL = "initial",diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 22b4e1da5..fca7153d6 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -24,7 +24,6 @@ const ajv = new Ajv();import { ExtractStep, updateExtract } from "./extract-redis";import { deduplicateObjectsArray } from "./helpers/deduplicate-objs-array";import { mergeNullValObjs } from "./helpers/merge-null-val-objs";-import { areMergeable } from "./helpers/merge-null-val-objs";import { CUSTOM_U_TEAMS } from "./config";import {calculateFinalResultCost,@@ -40,8 +39,8 @@ import { normalizeUrl } from "../canonical-url";import { search } from "../../search";import { buildRephraseToSerpPrompt } from "./build-prompts";import { getACUCTeam } from "../../controllers/auth";-import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { langfuse } from "../../services/langfuse";+import { CostLimitExceededError, CostTracking } from "../cost-tracking";interface ExtractServiceOptions {request: ExtractRequest;@@ -74,64 +73,6 @@ type completions = {sources?: string[];};-export class CostLimitExceededError extends Error {- constructor() {- super("Cost limit exceeded");- this.message = "Cost limit exceeded";- this.name = "CostLimitExceededError";- }-}--const nanProof = (n: number | null | undefined) =>- isNaN(n ?? 0) ? 0 : (n ?? 0);--export class CostTracking {- calls: {- type: "smartScrape" | "other";- metadata: Record<string, any>;- cost: number;- model: string;- tokens?: {- input: number;- output: number;- };- stack: string;- }[] = [];- limit: number | null = null;-- constructor(limit: number | null = null) {- this.limit = limit;- }-- public addCall(call: Omit<(typeof this.calls)[number], "stack">) {- this.calls.push({- ...call,- stack: new Error().stack!.split("\n").slice(2).join("\n"),- });-- if (this.limit !== null && this.toJSON().totalCost > this.limit) {- throw new CostLimitExceededError();- }- }-- public toJSON() {- return {- calls: this.calls,-- smartScrapeCallCount: this.calls.filter(c => c.type === "smartScrape")- .length,- smartScrapeCost: this.calls- .filter(c => c.type === "smartScrape")- .reduce((acc, c) => acc + nanProof(c.cost), 0),- otherCallCount: this.calls.filter(c => c.type === "other").length,- otherCost: this.calls- .filter(c => c.type === "other")- .reduce((acc, c) => acc + nanProof(c.cost), 0),- totalCost: this.calls.reduce((acc, c) => acc + nanProof(c.cost), 0),- };- }-}-export async function performExtraction(extractId: string,options: ExtractServiceOptions,diff --git a/apps/api/src/lib/extract/fire-0/reranker-f0.ts b/apps/api/src/lib/extract/fire-0/reranker-f0.tsindex bf2ec2e99..d9e8cb5c0 100644--- a/apps/api/src/lib/extract/fire-0/reranker-f0.ts+++ b/apps/api/src/lib/extract/fire-0/reranker-f0.ts@@ -13,7 +13,7 @@ import {buildRerankerSystemPrompt_F0,buildRerankerUserPrompt_F0,} from "./build-prompts-f0";-import { CostTracking } from "../extraction-service";+import { CostTracking } from "../../cost-tracking";const cohere = new CohereClient({token: process.env.COHERE_API_KEY,});diff --git a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts b/apps/api/src/lib/extract/fire-0/url-processor-f0.tsindex dd6a425ff..8f59f516a 100644--- a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts+++ b/apps/api/src/lib/extract/fire-0/url-processor-f0.ts@@ -12,7 +12,7 @@ import { extractConfig } from "../config";import type { Logger } from "winston";import { generateText } from "ai";import { getModel } from "../../generic-ai";-import { CostTracking } from "../extraction-service";+import { CostTracking } from "../../cost-tracking";import { getACUCTeam } from "../../../controllers/auth";export async function generateBasicCompletion_FO(diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.tsindex 3b6242795..c06949ab4 100644--- a/apps/api/src/lib/extract/reranker.ts+++ b/apps/api/src/lib/extract/reranker.ts@@ -10,7 +10,7 @@ import { buildRerankerSystemPrompt } from "./build-prompts";import { dumpToFile } from "./helpers/dump-to-file";import { getModel } from "../generic-ai";import fs from "fs/promises";-import { CostTracking } from "./extraction-service";+import { CostTracking } from "../cost-tracking";const THRESHOLD_FOR_SINGLEPAGE = 0.6;const THRESHOLD_FOR_MULTIENTITY = 0.45;diff --git a/apps/api/src/lib/extract/url-processor.ts b/apps/api/src/lib/extract/url-processor.tsindex 017604282..7875ff085 100644--- a/apps/api/src/lib/extract/url-processor.ts+++ b/apps/api/src/lib/extract/url-processor.ts@@ -9,7 +9,7 @@ import type { Logger } from "winston";import { generateText } from "ai";import { getModel } from "../generic-ai";import { calculateCost } from "../../scraper/scrapeURL/transformers/llmExtract";-import type { CostTracking } from "./extraction-service";+import type { CostTracking } from "../cost-tracking";export async function generateBasicCompletion(prompt: string,diff --git a/apps/api/src/lib/extract/usage/llm-cost.ts b/apps/api/src/lib/extract/usage/llm-cost.tsindex 46dde72f5..4e675acef 100644--- a/apps/api/src/lib/extract/usage/llm-cost.ts+++ b/apps/api/src/lib/extract/usage/llm-cost.ts@@ -1,6 +1,6 @@import { TokenUsage } from "../../../controllers/v1/types";import { logger } from "../../../lib/logger";-import { CostTracking } from "../extraction-service";+import { CostTracking } from "../../cost-tracking";import { modelPrices } from "./model-prices";interface ModelPricing {diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.tsindex 7e16fde7d..92d681e90 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts@@ -11,7 +11,7 @@ import { billTeam } from "../../services/billing/credit_billing";import { logJob } from "../../services/logging/log_job";import { getModel } from "../generic-ai";import { generateCompletions } from "../../scraper/scrapeURL/transformers/llmExtract";-import { CostTracking } from "../extract/extraction-service";+import { CostTracking } from "../cost-tracking";import { getACUCTeam } from "../../controllers/auth";interface GenerateLLMsTextServiceOptions {generationId: string;diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 300909b4a..40712f183 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -1,7 +1,13 @@import robotsParser, { Robot } from "robots-parser";-import * as undici from "undici";import { Logger } from "winston";-import { getSecureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";+import { ScrapeOptions, scrapeOptions } from "../controllers/v2/types";+import { scrapeURL } from "../scraper/scrapeURL";+import { Engine } from "../scraper/scrapeURL/engines";+import { CostTracking } from "./cost-tracking";++const useFireEngine =+ process.env.FIRE_ENGINE_BETA_URL !== "" &&+ process.env.FIRE_ENGINE_BETA_URL !== undefined;export interface RobotsTxtChecker {robotsTxtUrl: string;@@ -10,36 +16,88 @@ export interface RobotsTxtChecker {}export async function fetchRobotsTxt(- url: string,- skipTlsVerification: boolean = false,+ {+ url,+ zeroDataRetention,+ location,+ }: {+ url: string;+ zeroDataRetention: boolean;+ location?: ScrapeOptions["location"];+ },+ scrapeId: string,+ logger: Logger,abort?: AbortSignal,-): Promise<string> {+): Promise<{ content: string; url: string }> {const urlObj = new URL(url);const robotsTxtUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;- const response = await undici.fetch(robotsTxtUrl, {- signal: abort,- dispatcher: getSecureDispatcher(skipTlsVerification),- });-- if (!response.ok) {- throw new Error(- `Failed to fetch robots.txt: ${response.status} ${response.statusText}`,- );- }-- const content = await response.text();- const contentType = response.headers.get("content-type") || "";+ const shouldPrioritizeFireEngine = location && useFireEngine;++ const forceEngine: Engine[] = [+ ...(shouldPrioritizeFireEngine+ ? [+ "fire-engine;tlsclient" as const,+ "fire-engine;tlsclient;stealth" as const,+ ]+ : []),+ "fetch",+ ...(!shouldPrioritizeFireEngine && useFireEngine+ ? [+ "fire-engine;tlsclient" as const,+ "fire-engine;tlsclient;stealth" as const,+ ]+ : []),+ ];++ let content: string = "";+ const response = await scrapeURL(+ "robots-txt;" + scrapeId,+ robotsTxtUrl,+ scrapeOptions.parse({+ formats: ["rawHtml"],+ ...(location ? { location } : {}),+ }),+ {+ forceEngine,+ v0DisableJsDom: true,+ externalAbort: abort+ ? {+ signal: abort,+ tier: "external",+ throwable() {+ return new Error("Robots.txt fetch aborted");+ },+ }+ : undefined,+ teamId: "robots-txt",+ zeroDataRetention,+ },+ new CostTracking(),+ );if (- (contentType.includes("text/html") && content.trim().startsWith("<")) ||- contentType.includes("application/json") ||- contentType.includes("application/xml")+ response.success &&+ response.document.metadata.statusCode >= 200 &&+ response.document.metadata.statusCode < 300) {- return "";+ content = response.document.rawHtml!;+ } else {+ logger.error(`Request failed for robots.txt fetch`, {+ method: "fetchRobotsTxt",+ robotsTxtUrl,+ error: response.success+ ? response.document.metadata.statusCode+ : response.error,+ });+ return { content: "", url: robotsTxtUrl };}- return content;+ // return URL in case we've been redirected+ return {+ content: content,+ url: response.document.metadata.url || robotsTxtUrl,+ };}export function createRobotsChecker(@@ -99,23 +157,3 @@ export function isUrlAllowedByRobots(return false;}--export async function checkRobotsTxt(- url: string,- skipTlsVerification: boolean = false,- logger?: Logger,- abort?: AbortSignal,-): Promise<boolean> {- try {- const robotsTxt = await fetchRobotsTxt(url, skipTlsVerification, abort);- const checker = createRobotsChecker(url, robotsTxt);- return isUrlAllowedByRobots(url, checker.robots);- } catch (error) {- // If we can't fetch robots.txt, assume it's allowed- logger?.debug("Failed to fetch robots.txt, allowing scrape", {- error,- url,- });- return true;- }-}diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex c2358558c..67f174af4 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -5,7 +5,7 @@ import {TeamFlags,shouldParsePDF,} from "../controllers/v2/types";-import { CostTracking } from "./extract/extraction-service";+import { CostTracking } from "./cost-tracking";import { hasFormatOfType } from "./format-utils";const creditsPerPDFPage = 1;diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex b7a5ad560..e500ebd4f 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -4,7 +4,7 @@ import { supabase_service } from "../services/supabase";import { logger as _logger } from "../lib/logger";import { configDotenv } from "dotenv";import { scrapeURL, ScrapeUrlResponse } from "../scraper/scrapeURL";-import { CostTracking } from "../lib/extract/extraction-service";+import { CostTracking } from "../lib/cost-tracking";configDotenv();export async function startWebScraperPipeline({diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 0f796ea98..0f060e5ca 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -15,6 +15,8 @@ import {isUrlAllowedByRobots,} from "../../lib/robots-txt";import { ScrapeJobTimeoutError } from "../../lib/error";+import { Logger } from "winston";+import { ScrapeOptions } from "../../controllers/v2/types";export const SITEMAP_LIMIT = 100;@@ -67,6 +69,7 @@ export class WebCrawler {private maxDiscoveryDepth: number | undefined;private currentDiscoveryDepth: number;private zeroDataRetention: boolean;+ private location?: ScrapeOptions["location"];constructor({jobId,@@ -86,6 +89,7 @@ export class WebCrawler {maxDiscoveryDepth,currentDiscoveryDepth,zeroDataRetention,+ location,}: {jobId: string;initialUrl: string;@@ -104,6 +108,7 @@ export class WebCrawler {maxDiscoveryDepth?: number;currentDiscoveryDepth?: number;zeroDataRetention?: boolean;+ location?: ScrapeOptions["location"];}) {this.jobId = jobId;this.initialUrl = initialUrl;@@ -131,6 +136,7 @@ export class WebCrawler {});this.maxDiscoveryDepth = maxDiscoveryDepth;this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0;+ this.location = location;}public async filterLinks(@@ -340,9 +346,14 @@ export class WebCrawler {skipTlsVerification,});- const robotsTxt = await fetchRobotsTxt(- this.initialUrl,- skipTlsVerification,+ const { content: robotsTxt, url } = await fetchRobotsTxt(+ {+ url: this.robotsTxtUrl,+ zeroDataRetention: this.zeroDataRetention,+ location: this.location,+ },+ this.jobId,+ this.logger,abort,);@@ -351,6 +362,7 @@ export class WebCrawler {initialUrl: this.initialUrl,robotsTxtLength: robotsTxt.length,hasContent: robotsTxt.length > 0,+ finalUrl: url,});return robotsTxt;@@ -843,6 +855,7 @@ export class WebCrawler {mode: "fire-engine",maxAge,zeroDataRetention: this.zeroDataRetention,+ location: this.location,},this.logger,this.jobId,@@ -893,6 +906,7 @@ export class WebCrawler {mode: "fire-engine",maxAge,zeroDataRetention: this.zeroDataRetention,+ location: this.location,},this.logger,this.jobId,@@ -934,6 +948,7 @@ export class WebCrawler {mode: "fire-engine",maxAge,zeroDataRetention: this.zeroDataRetention,+ location: this.location,},this.logger,this.jobId,@@ -960,6 +975,7 @@ export class WebCrawler {mode: "fire-engine",maxAge,zeroDataRetention: this.zeroDataRetention,+ location: this.location,},this.logger,this.jobId,diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 43621aaa9..4616df85b 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -3,9 +3,11 @@ import { WebCrawler, SITEMAP_LIMIT } from "./crawler";import { scrapeURL } from "../scrapeURL";import { scrapeOptions } from "../../controllers/v2/types";import type { Logger } from "winston";-import { CostTracking } from "../../lib/extract/extraction-service";+import { CostTracking } from "../../lib/cost-tracking";import { parseSitemapXml, processSitemap } from "../../lib/crawler";import { ScrapeJobTimeoutError } from "../../lib/error";+import type { ScrapeOptions } from "../../controllers/v2/types";+import { Engine } from "../scrapeURL/engines";const useFireEngine =process.env.FIRE_ENGINE_BETA_URL !== "" &&process.env.FIRE_ENGINE_BETA_URL !== undefined;@@ -17,12 +19,14 @@ export async function getLinksFromSitemap(mode = "axios",maxAge = 0,zeroDataRetention,+ location,}: {sitemapUrl: string;urlsHandler(urls: string[]): unknown;mode?: "axios" | "fire-engine";maxAge?: number;zeroDataRetention: boolean;+ location?: ScrapeOptions["location"];},logger: Logger,crawlId: string,@@ -44,21 +48,39 @@ export async function getLinksFromSitemap(try {let content: string = "";try {+ const shouldPrioritizeFireEngine =+ location && mode === "fire-engine" && useFireEngine;++ const forceEngine: Engine[] = [+ ...(maxAge > 0 ? ["index" as const] : []),+ ...(shouldPrioritizeFireEngine+ ? [+ "fire-engine;tlsclient" as const,+ "fire-engine;tlsclient;stealth" as const,+ ]+ : []),+ "fetch",+ ...(!shouldPrioritizeFireEngine &&+ mode === "fire-engine" &&+ useFireEngine+ ? [+ "fire-engine;tlsclient" as const,+ "fire-engine;tlsclient;stealth" as const,+ ]+ : []),+ ];+const response = await scrapeURL("sitemap;" + crawlId,sitemapUrl,- scrapeOptions.parse({ formats: ["rawHtml"], useMock: mock, maxAge }),+ scrapeOptions.parse({+ formats: ["rawHtml"],+ useMock: mock,+ maxAge,+ ...(location ? { location } : {}),+ }),{- forceEngine: [- ...(maxAge > 0 ? ["index" as const] : []),- "fetch",- ...(mode === "fire-engine" && useFireEngine- ? [- "fire-engine;tlsclient" as const,- "fire-engine;tlsclient;stealth" as const,- ]- : []),- ],+ forceEngine,v0DisableJsDom: true,externalAbort: abort? {@@ -145,7 +167,7 @@ export async function getLinksFromSitemap(const sitemapPromises: Promise<number>[] = sitemapUrls.map(sitemapUrl =>getLinksFromSitemap(- { sitemapUrl, urlsHandler, mode, zeroDataRetention },+ { sitemapUrl, urlsHandler, mode, zeroDataRetention, location },logger,crawlId,sitemapsHit,@@ -169,7 +191,13 @@ export async function getLinksFromSitemap(if (xmlSitemaps.length > 0) {const sitemapPromises = xmlSitemaps.map(sitemapUrl =>getLinksFromSitemap(- { sitemapUrl: sitemapUrl, urlsHandler, mode, zeroDataRetention },+ {+ sitemapUrl: sitemapUrl,+ urlsHandler,+ mode,+ zeroDataRetention,+ location,+ },logger,crawlId,sitemapsHit,@@ -209,7 +237,7 @@ export async function getLinksFromSitemap(const sitemapPromises: Promise<number>[] = instruction.urls.map(sitemapUrl =>getLinksFromSitemap(- { sitemapUrl, urlsHandler, mode, zeroDataRetention },+ { sitemapUrl, urlsHandler, mode, zeroDataRetention, location },logger,crawlId,sitemapsHit,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 5821a3615..c1d81fa79 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -43,7 +43,7 @@ import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";import { urlSpecificParams } from "./lib/urlSpecificParams";import { loadMock, MockState } from "./lib/mock";-import { CostTracking } from "../../lib/extract/extraction-service";+import { CostTracking } from "../../lib/cost-tracking";import {addIndexRFInsertJob,generateDomainSplits,@@ -52,7 +52,11 @@ import {normalizeURLForIndex,useIndex,} from "../../services/index";-import { checkRobotsTxt } from "../../lib/robots-txt";+import {+ fetchRobotsTxt,+ createRobotsChecker,+ isUrlAllowedByRobots,+} from "../../lib/robots-txt";import {AbortInstance,AbortManager,@@ -674,24 +678,41 @@ export async function scrapeURL(}if (internalOptions.teamFlags?.checkRobotsOnScrape) {- meta.logger.info("Checking robots.txt", {- checkRobotsOnScrape: internalOptions.teamFlags?.checkRobotsOnScrape,- url: meta.rewrittenUrl || meta.url,- });const urlToCheck = meta.rewrittenUrl || meta.url;- const isAllowed = await checkRobotsTxt(- urlToCheck,- options.skipTlsVerification,- meta.logger,- meta.abort.asSignal(),- );+ meta.logger.info("Checking robots.txt", { url: urlToCheck });++ const urlObj = new URL(urlToCheck);+ const isRobotsTxtPath = urlObj.pathname === "/robots.txt";++ if (!isRobotsTxtPath) {+ try {+ const { content: robotsTxt } = await fetchRobotsTxt(+ {+ url: urlToCheck,+ zeroDataRetention: internalOptions.zeroDataRetention || false,+ location: options.location,+ },+ id,+ meta.logger,+ meta.abort.asSignal(),+ );- if (!isAllowed) {- meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });- return {- success: false,- error: new Error("URL blocked by robots.txt"),- };+ const checker = createRobotsChecker(urlToCheck, robotsTxt);+ const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);++ if (!isAllowed) {+ meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });+ return {+ success: false,+ error: new Error("URL blocked by robots.txt"),+ };+ }+ } catch (error) {+ meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {+ error,+ url: urlToCheck,+ });+ }}}diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 6becda9eb..e04a93f46 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -13,7 +13,7 @@ import type { SmartScrapeResult } from "./smartScrape";import {CostLimitExceededError,CostTracking,-} from "../../../lib/extract/extraction-service";+} from "../../../lib/cost-tracking";const commonSmartScrapeProperties = {shouldUseSmartscrape: {type: "boolean",diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.tsindex 7c4cbe341..f46c1a753 100644--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts@@ -6,7 +6,7 @@ import { configDotenv } from "dotenv";import {CostLimitExceededError,CostTracking,-} from "../../../lib/extract/extraction-service";+} from "../../../lib/cost-tracking";configDotenv();// Define schemas outside the function scopediff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.tsindex 79a8dc477..c379b92ab 100644--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts@@ -5,7 +5,7 @@ process.env.ENV = "test";import { scrapeURL } from ".";import { scrapeOptions } from "../../controllers/v2/types";import { Engine } from "./engines";-import { CostTracking } from "../../lib/extract/extraction-service";+import { CostTracking } from "../../lib/cost-tracking";const testEngines: (Engine | undefined)[] = [undefined,diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex aaede1997..9a559c6d1 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -22,7 +22,7 @@ import { z } from "zod";import fs from "fs/promises";import Ajv from "ajv";import { extractData } from "../lib/extractSmartScrape";-import { CostTracking } from "../../../lib/extract/extraction-service";+import { CostTracking } from "../../../lib/cost-tracking";import { isAgentExtractModelValid } from "../../../controllers/v1/types";import { hasFormatOfType } from "../../../lib/format-utils";// TODO: fix this, it's horriblediff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 664761323..1c79ccd4a 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -34,7 +34,7 @@ import { getACUCTeam } from "../../controllers/auth";import { callWebhook } from "../webhook";import { CustomError } from "../../lib/custom-error";import { startWebScraperPipeline } from "../../main/runWebScraper";-import { CostTracking } from "../../lib/extract/extraction-service";+import { CostTracking } from "../../lib/cost-tracking";import { redisEvictConnection } from "../redis";import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 4a46da35b..834e81dd7 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -9,7 +9,7 @@ import {import { AuthCreditUsageChunk } from "./controllers/v1/types";import { ExtractorOptions, Document } from "./lib/entities";import { InternalOptions } from "./scraper/scrapeURL";-import type { CostTracking } from "./lib/extract/extraction-service";+import type { CostTracking } from "./lib/cost-tracking";type Mode = "crawl" | "single_urls" | "sitemap" | "kickoff";diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 081c80e68..888c1cc75 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.3.1",+ "version": "4.3.2","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex dfa24cd72..659f6133e 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -102,6 +102,14 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResultdescription?: string;}+/**+ * Location configuration for proxy location+ */+export interface LocationConfig {+ country?: string;+ languages?: string[];+}+/*** Parameters for scraping operations.* Defines the options and configurations available for scraping web content.@@ -114,10 +122,7 @@ export interface CrawlScrapeOptions {onlyMainContent?: boolean;waitFor?: number;timeout?: number;- location?: {- country?: string;- languages?: string[];- };+ location?: LocationConfig;mobile?: boolean;skipTlsVerification?: boolean;removeBase64Images?: boolean;@@ -300,6 +305,7 @@ export interface MapParams {limit?: number;timeout?: number;useIndex?: boolean;+ location?: LocationConfig;}/**diff --git a/apps/js-sdk/firecrawl/src/v2/methods/map.ts b/apps/js-sdk/firecrawl/src/v2/methods/map.tsindex 8b41f772e..7eb5e42c7 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/map.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/map.ts@@ -11,6 +11,7 @@ function prepareMapPayload(url: string, options?: MapOptions): Record<string, unif (options.includeSubdomains != null) payload.includeSubdomains = options.includeSubdomains;if (options.limit != null) payload.limit = options.limit;if (options.timeout != null) payload.timeout = options.timeout;+ if (options.location != null) payload.location = options.location;}return payload;}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 31434a1ff..6f9ad92eb 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -319,6 +319,7 @@ export interface MapOptions {includeSubdomains?: boolean;limit?: number;timeout?: number;+ location?: LocationConfig;}export interface ExtractResponse {diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex adba0b03e..6778ceaf3 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.3.1"+__version__ = "4.3.2"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.pyindex b6c94eca2..bc0af1ebf 100644--- a/apps/python-sdk/firecrawl/v1/client.py+++ b/apps/python-sdk/firecrawl/v1/client.py@@ -309,6 +309,7 @@ class V1MapParams(pydantic.BaseModel):limit: Optional[int] = Nonetimeout: Optional[int] = 30000useIndex: Optional[bool] = None+ location: Optional[V1LocationConfig] = Noneclass V1MapResponse(pydantic.BaseModel):"""Response from mapping operations."""@@ -1333,6 +1334,7 @@ class V1FirecrawlApp:limit: Optional[int] = None,timeout: Optional[int] = 30000,use_index: Optional[bool] = None,+ location: Optional[V1LocationConfig] = None,**kwargs) -> V1MapResponse:"""Map and discover links from a URL.@@ -1377,6 +1379,8 @@ class V1FirecrawlApp:map_params['timeout'] = timeoutif use_index is not None:map_params['useIndex'] = use_index+ if location is not None:+ map_params['location'] = location.dict(by_alias=True, exclude_none=True)# Add any additional kwargsmap_params.update(kwargs)@@ -3910,6 +3914,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):sitemap_only: Optional[bool] = None,limit: Optional[int] = None,timeout: Optional[int] = 30000,+ location: Optional[V1LocationConfig] = None,params: Optional[V1MapParams] = None) -> V1MapResponse:"""Asynchronously map and discover links from a URL.@@ -3952,6 +3957,8 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):map_params['limit'] = limitif timeout is not None:map_params['timeout'] = timeout+ if location is not None:+ map_params['location'] = location.dict(by_alias=True, exclude_none=True)# Create final params objectfinal_params = V1MapParams(**map_params)diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 5c4e3f867..9073460ee 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -421,6 +421,7 @@ class FirecrawlClient:limit: Optional[int] = None,sitemap: Optional[Literal["only", "include", "skip"]] = None,timeout: Optional[int] = None,+ location: Optional[Location] = None,) -> MapData:"""Map a URL and return discovered links.@@ -441,7 +442,8 @@ class FirecrawlClient:limit=limit,sitemap=sitemap if sitemap is not None else "include",timeout=timeout,- ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, timeout]) else None+ location=location+ ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, timeout, location]) else Nonereturn map_module.map(self.http_client, url, options)diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/map.py b/apps/python-sdk/firecrawl/v2/methods/aio/map.pyindex b9ca22f58..46d9fa898 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/map.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/map.py@@ -20,6 +20,8 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dictdata["limit"] = options.limitif options.timeout is not None:data["timeout"] = options.timeout+ if options.location is not None:+ data["location"] = options.location.model_dump(exclude_none=True)payload.update(data)return payloaddiff --git a/apps/python-sdk/firecrawl/v2/methods/map.py b/apps/python-sdk/firecrawl/v2/methods/map.pyindex 2c08e5be6..83343d24f 100644--- a/apps/python-sdk/firecrawl/v2/methods/map.py+++ b/apps/python-sdk/firecrawl/v2/methods/map.py@@ -27,6 +27,8 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dictdata["limit"] = options.limitif options.timeout is not None:data["timeout"] = options.timeout+ if options.location is not None:+ data["location"] = options.location.model_dump(exclude_none=True)payload.update(data)return payloaddiff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex e0a8a2794..77f61c4a5 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -445,6 +445,7 @@ class MapOptions(BaseModel):include_subdomains: Optional[bool] = Nonelimit: Optional[int] = Nonetimeout: Optional[int] = None+ location: Optional['Location'] = Noneclass MapRequest(BaseModel):"""Request for mapping a website."""diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex 11637503b..a34122c4c 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,7 +1,7 @@[package]name = "firecrawl"author= "Mendable.ai"-version = "1.2.1"+version = "1.2.2"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rsindex 44016064d..386cb098e 100644--- a/apps/rust-sdk/src/map.rs+++ b/apps/rust-sdk/src/map.rs@@ -2,6 +2,17 @@ use serde::{Deserialize, Serialize};use crate::{FirecrawlApp, FirecrawlError, API_VERSION};+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct Location {+ /// Country code (ISO 3166-1 alpha-2)+ pub country: Option<String>,++ /// List of language codes+ pub languages: Option<Vec<String>>,+}+#[serde_with::skip_serializing_none]#[derive(Deserialize, Serialize, Debug, Default)]#[serde(rename_all = "camelCase")]@@ -17,6 +28,9 @@ pub struct MapOptions {/// Maximum number of links to return (default: `5000`)pub limit: Option<u32>,++ /// Location configuration for proxy location+ pub location: Option<Location>,}#[derive(Deserialize, Serialize, Debug, Default)]