Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add ignoreCache parameter to map endpoint (#2686)
firecrawl-spring[bot]f2767b7271f5
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 2d385385d..d24b08ca6 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -95,6 +95,7 @@ export async function getMapResults({filterByPath = true,flags,useIndex = true,+ ignoreCache = false,timeout,location,headers,@@ -115,6 +116,7 @@ export async function getMapResults({filterByPath?: boolean;flags: TeamFlags;useIndex?: boolean;+ ignoreCache?: boolean;timeout?: number;location?: ScrapeOptions["location"];headers?: Record<string, string>;@@ -162,6 +164,7 @@ export async function getMapResults({timeout ?? 30000,abort,mock,+ ignoreCache ? 0 : undefined,);if (sitemap > 0) {links = links@@ -192,7 +195,7 @@ export async function getMapResults({);const cacheKey = `fireEngineMap:${mapUrl}`;- const cachedResult = await redis.get(cacheKey);+ const cachedResult = ignoreCache ? null : await redis.get(cacheKey);let allResults: any[] = [];let pagePromises: Promise<any>[] = [];@@ -248,6 +251,8 @@ export async function getMapResults({false,timeout ?? 30000,abort,+ undefined,+ ignoreCache ? 0 : undefined,);} catch (e) {logger.warn("tryGetSitemap threw an error", { error: e });@@ -423,6 +428,7 @@ export async function mapController(filterByPath: req.body.filterByPath !== false,flags: req.acuc?.flags ?? null,useIndex: req.body.useIndex,+ ignoreCache: req.body.ignoreCache,timeout: req.body.timeout,location: req.body.location,headers: req.body.headers,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 72c9679ad..f2546b589 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -945,6 +945,7 @@ const mapRequestSchemaBase = crawlerOptionsuseMock: z.string().optional(),filterByPath: z.boolean().prefault(true),useIndex: z.boolean().prefault(true),+ ignoreCache: z.boolean().prefault(false),location: locationSchema,headers: z.record(z.string(), z.string()).optional(),});diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 14599f0f6..4c5adcf13 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -90,6 +90,7 @@ export async function mapController(filterByPath: req.body.filterByPath !== false,flags: req.acuc?.flags ?? null,useIndex: req.body.useIndex,+ ignoreCache: req.body.ignoreCache,location: req.body.location,headers: req.body.headers,id: mapId,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 42bcac35b..a599b82eb 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -942,6 +942,7 @@ const mapRequestSchemaBase = crawlerOptionsuseMock: z.string().optional(),filterByPath: z.boolean().prefault(true),useIndex: z.boolean().prefault(true),+ ignoreCache: z.boolean().prefault(false),location: locationSchema,headers: z.record(z.string(), z.string()).optional(),});diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 4df034292..f8858939c 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -90,6 +90,7 @@ export async function getMapResults({filterByPath = true,flags,useIndex = true,+ ignoreCache = false,location,headers,maxFireEngineResults = MAX_FIRE_ENGINE_RESULTS,@@ -109,6 +110,7 @@ export async function getMapResults({filterByPath?: boolean;flags: TeamFlags | null;useIndex?: boolean;+ ignoreCache?: boolean;location?: ScrapeOptions["location"];headers?: Record<string, string>;maxFireEngineResults?: number;@@ -171,6 +173,7 @@ export async function getMapResults({crawlerOptions.timeout ?? 30000,abort,crawlerOptions.useMock,+ ignoreCache ? 0 : undefined,);if (sitemap > 0) {@@ -203,7 +206,9 @@ export async function getMapResults({);const cacheKey = `fireEngineMap:${mapUrl}`;- const cachedResult = await redisEvictConnection.get(cacheKey);+ const cachedResult = ignoreCache+ ? null+ : await redisEvictConnection.get(cacheKey);let pagePromises: (Promise<any> | any)[];@@ -258,6 +263,8 @@ export async function getMapResults({false,crawlerOptions.timeout ?? 30000,abort,+ undefined,+ ignoreCache ? 0 : undefined,);} catch (e) {// Silently handle sitemap errorsdiff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 31d33f18c..5172ff986 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -441,6 +441,7 @@ export class WebCrawler {timeout: number = 120000,abort?: AbortSignal,mock?: string,+ maxAge: number = SITEMAP_MAX_AGE,): Promise<number> {this.logger.debug(`Fetching sitemap links from ${this.initialUrl}`, {method: "tryGetSitemap",@@ -523,7 +524,7 @@ export class WebCrawler {_urlsHandler,abort,mock,- SITEMAP_MAX_AGE,+ maxAge,),...robotsSitemaps.map(x =>this.tryFetchSitemapLinks(@@ -531,7 +532,7 @@ export class WebCrawler {_urlsHandler,abort,mock,- SITEMAP_MAX_AGE,+ maxAge,),),]).then(results => results.reduce((a, x) => a + x, 0)),