Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add ignoreCache parameter to map endpoint (#2686)

firecrawl-spring[bot]f2767b7271f5
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 2d385385d..d24b08ca6 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -95,6 +95,7 @@ export async function getMapResults({
filterByPath = true,
flags,
useIndex = true,
+ ignoreCache = false,
timeout,
location,
headers,
@@ -115,6 +116,7 @@ export async function getMapResults({
filterByPath?: boolean;
flags: TeamFlags;
useIndex?: boolean;
+ ignoreCache?: boolean;
timeout?: number;
location?: ScrapeOptions["location"];
headers?: Record<string, string>;
@@ -162,6 +164,7 @@ export async function getMapResults({
timeout ?? 30000,
abort,
mock,
+ ignoreCache ? 0 : undefined,
);
if (sitemap > 0) {
links = links
@@ -192,7 +195,7 @@ export async function getMapResults({
);
const cacheKey = `fireEngineMap:${mapUrl}`;
- const cachedResult = await redis.get(cacheKey);
+ const cachedResult = ignoreCache ? null : await redis.get(cacheKey);
let allResults: any[] = [];
let pagePromises: Promise<any>[] = [];
@@ -248,6 +251,8 @@ export async function getMapResults({
false,
timeout ?? 30000,
abort,
+ undefined,
+ ignoreCache ? 0 : undefined,
);
} catch (e) {
logger.warn("tryGetSitemap threw an error", { error: e });
@@ -423,6 +428,7 @@ export async function mapController(
filterByPath: req.body.filterByPath !== false,
flags: req.acuc?.flags ?? null,
useIndex: req.body.useIndex,
+ ignoreCache: req.body.ignoreCache,
timeout: req.body.timeout,
location: req.body.location,
headers: req.body.headers,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 72c9679ad..f2546b589 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -945,6 +945,7 @@ const mapRequestSchemaBase = crawlerOptions
useMock: z.string().optional(),
filterByPath: z.boolean().prefault(true),
useIndex: z.boolean().prefault(true),
+ ignoreCache: z.boolean().prefault(false),
location: locationSchema,
headers: z.record(z.string(), z.string()).optional(),
});
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 14599f0f6..4c5adcf13 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -90,6 +90,7 @@ export async function mapController(
filterByPath: req.body.filterByPath !== false,
flags: req.acuc?.flags ?? null,
useIndex: req.body.useIndex,
+ ignoreCache: req.body.ignoreCache,
location: req.body.location,
headers: req.body.headers,
id: mapId,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 42bcac35b..a599b82eb 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -942,6 +942,7 @@ const mapRequestSchemaBase = crawlerOptions
useMock: z.string().optional(),
filterByPath: z.boolean().prefault(true),
useIndex: z.boolean().prefault(true),
+ ignoreCache: z.boolean().prefault(false),
location: locationSchema,
headers: z.record(z.string(), z.string()).optional(),
});
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 4df034292..f8858939c 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -90,6 +90,7 @@ export async function getMapResults({
filterByPath = true,
flags,
useIndex = true,
+ ignoreCache = false,
location,
headers,
maxFireEngineResults = MAX_FIRE_ENGINE_RESULTS,
@@ -109,6 +110,7 @@ export async function getMapResults({
filterByPath?: boolean;
flags: TeamFlags | null;
useIndex?: boolean;
+ ignoreCache?: boolean;
location?: ScrapeOptions["location"];
headers?: Record<string, string>;
maxFireEngineResults?: number;
@@ -171,6 +173,7 @@ export async function getMapResults({
crawlerOptions.timeout ?? 30000,
abort,
crawlerOptions.useMock,
+ ignoreCache ? 0 : undefined,
);
if (sitemap > 0) {
@@ -203,7 +206,9 @@ export async function getMapResults({
);
const cacheKey = `fireEngineMap:${mapUrl}`;
- const cachedResult = await redisEvictConnection.get(cacheKey);
+ const cachedResult = ignoreCache
+ ? null
+ : await redisEvictConnection.get(cacheKey);
let pagePromises: (Promise<any> | any)[];
@@ -258,6 +263,8 @@ export async function getMapResults({
false,
crawlerOptions.timeout ?? 30000,
abort,
+ undefined,
+ ignoreCache ? 0 : undefined,
);
} catch (e) {
// Silently handle sitemap errors
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 31d33f18c..5172ff986 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -441,6 +441,7 @@ export class WebCrawler {
timeout: number = 120000,
abort?: AbortSignal,
mock?: string,
+ maxAge: number = SITEMAP_MAX_AGE,
): Promise<number> {
this.logger.debug(`Fetching sitemap links from ${this.initialUrl}`, {
method: "tryGetSitemap",
@@ -523,7 +524,7 @@ export class WebCrawler {
_urlsHandler,
abort,
mock,
- SITEMAP_MAX_AGE,
+ maxAge,
),
...robotsSitemaps.map(x =>
this.tryFetchSitemapLinks(
@@ -531,7 +532,7 @@ export class WebCrawler {
_urlsHandler,
abort,
mock,
- SITEMAP_MAX_AGE,
+ maxAge,
),
),
]).then(results => results.reduce((a, x) => a + x, 0)),