Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Remove old cache mechanisms (redis cache, PDF cache, crawl maps, etc.) (FIR-2266) (#1667)
Gergő Móricz3f0b8b8e2742
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 1336a87bd..3a2f0e796 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -161,6 +161,29 @@ describe("Scrape tests", () => {expect(response4.metadata.cacheState).toBe("miss");}, 150000 + 2 * 17000);+ it.concurrent("caches PDFs properly", async () => {+ const id = crypto.randomUUID();+ const url = "https://www.orimi.com/pdf-test.pdf?testId=" + id;++ const response1 = await scrape({+ url,+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response1.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, 17000));++ const response2 = await scrape({+ url,+ timeout: 60000,+ maxAge: 120000,+ });++ expect(response2.metadata.cacheState).toBe("hit");+ }, 150000 + 2 * 17000);+it.concurrent("respects screenshot", async () => {const id = crypto.randomUUID();const url = "https://firecrawl.dev/?testId=" + id;diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 111b8c818..06423e4c6 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -23,8 +23,6 @@ import { logJob } from "../../services/logging/log_job";import { performCosineSimilarity } from "../../lib/map-cosine";import { logger } from "../../lib/logger";import Redis from "ioredis";-import { querySitemapIndex } from "../../scraper/WebScraper/sitemap-index";-import { getIndexQueue } from "../../services/queue-service";import { generateURLSplits, queryIndexAtDomainSplitLevel, queryIndexAtSplitLevel } from "../../services/index";configDotenv();@@ -44,7 +42,7 @@ interface MapResult {mapResults: MapDocument[];}-async function queryIndex(url: string, limit: number, useIndex: boolean): Promise<string[]> {+async function queryIndex(url: string, limit: number, useIndex: boolean, includeSubdomains: boolean): Promise<string[]> {if (!useIndex) {return [];}@@ -54,9 +52,9 @@ async function queryIndex(url: string, limit: number, useIndex: boolean): Promisconst urlObj = new URL(url);const hostname = urlObj.hostname;- // TEMP: this can be removed in 2 days- const domainLinks = await queryIndexAtDomainSplitLevel(hostname, limit);- const splitLinks = await queryIndexAtSplitLevel(url, limit);+ // TEMP: this should be altered on June 15th 2025 7AM PT - mogery+ const domainLinks = includeSubdomains ? await queryIndexAtDomainSplitLevel(hostname, limit, 14 * 24 * 60 * 60 * 1000) : [];+ const splitLinks = await queryIndexAtSplitLevel(url, limit, 14 * 24 * 60 * 60 * 1000);return Array.from(new Set([...domainLinks, ...splitLinks]));} else {@@ -188,9 +186,8 @@ export async function getMapResults({}// Parallelize sitemap index query with search results- const [sitemapIndexResult, indexResults, ...searchResults] = await Promise.all([- querySitemapIndex(url, abort),- queryIndex(url, limit, useIndex),+ const [indexResults, ...searchResults] = await Promise.all([+ queryIndex(url, limit, useIndex, includeSubdomains),...(cachedResult ? [] : pagePromises),]);@@ -198,14 +195,10 @@ export async function getMapResults({links.push(...indexResults);}- const twoDaysAgo = new Date();- twoDaysAgo.setDate(twoDaysAgo.getDate() - 2);-- // If sitemap is not ignored and either we have few URLs (<100) or the data is stale (>2 days old), fetch fresh sitemap+ // If sitemap is not ignored, fetch sitemap+ // This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 daysif (- !ignoreSitemap &&- (sitemapIndexResult.urls.length < 100 ||- new Date(sitemapIndexResult.lastUpdated) < twoDaysAgo)+ !ignoreSitemap) {try {await crawler.tryGetSitemap(@@ -250,9 +243,6 @@ export async function getMapResults({}}- // Add sitemap-index URLs- links.push(...sitemapIndexResult.urls);-// Perform cosine similarity between the search query and the list of linksif (search) {const searchQuery = search.toLowerCase();@@ -308,19 +298,6 @@ export async function getMapResults({? links: links.slice(0, limit);- //-- await getIndexQueue().add(- id,- {- originUrl: url,- visitedUrls: linksToReturn.filter(link => link !== url),- },- {- priority: 10,- },- );-return {success: true,links: linksToReturn,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 29d78a420..993f3d4f6 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -51,12 +51,16 @@ export async function scrapeController(url: req.body.url,mode: "single_urls",team_id: req.auth.team_id,- scrapeOptions: req.body,+ scrapeOptions: {+ ...req.body,+ ...(req.body.__experimental_cache ? {+ maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours+ } : {}),+ },internalOptions: {teamId: req.auth.team_id,saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,unnormalizedSourceURL: preNormalizedBody.url,- useCache: req.body.__experimental_cache ? true : false,bypassBilling: isDirectToBullMQ,},origin,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex b716d5a9a..65b45836d 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -101,9 +101,9 @@ async function scrapeSearchResult(team_id: options.teamId,scrapeOptions: {...options.scrapeOptions,- maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache+ maxAge: 4 * 60 * 60 * 1000,},- internalOptions: { teamId: options.teamId, useCache: true, bypassBilling: true },+ internalOptions: { teamId: options.teamId, bypassBilling: true },origin: options.origin,is_scrape: true,startTime: Date.now(),diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex ca35ab5bc..f1e8ed34f 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -328,6 +328,7 @@ const baseScrapeOptions = zproxy: z.enum(["basic", "stealth", "auto"]).optional(),maxAge: z.number().int().gte(0).safe().default(0),storeInCache: z.boolean().default(true),+ // @deprecated__experimental_cache: z.boolean().default(false).optional(),__searchPreviewToken: z.string().optional(),})diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 05fe005ee..7e79a49a4 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -7,7 +7,6 @@ import cors from "cors";import {getExtractQueue,getScrapeQueue,- getIndexQueue,getGenerateLlmsTxtQueue,getDeepResearchQueue,getBillingQueue,@@ -57,7 +56,6 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({queues: [new BullAdapter(getScrapeQueue()),new BullAdapter(getExtractQueue()),- new BullAdapter(getIndexQueue()),new BullAdapter(getGenerateLlmsTxtQueue()),new BullAdapter(getDeepResearchQueue()),new BullAdapter(getBillingQueue()),diff --git a/apps/api/src/lib/cache.ts b/apps/api/src/lib/cache.tsdeleted file mode 100644index a60664373..000000000--- a/apps/api/src/lib/cache.ts+++ /dev/null@@ -1,70 +0,0 @@-import IORedis from "ioredis";-import { ScrapeOptions } from "../controllers/v1/types";-import { InternalOptions } from "../scraper/scrapeURL";-import { logger as _logger } from "./logger";-const logger = _logger.child({ module: "cache" });--export const cacheRedis = process.env.CACHE_REDIS_URL- ? new IORedis(process.env.CACHE_REDIS_URL, {- maxRetriesPerRequest: null,- })- : null;--export function cacheKey(- url: string,- scrapeOptions: ScrapeOptions,- internalOptions: InternalOptions,-): string | null {- if (!cacheRedis) return null;-- // these options disqualify a cache- if (- internalOptions.v0CrawlOnlyUrls ||- internalOptions.forceEngine?.includes("cache") ||- scrapeOptions.fastMode ||- internalOptions.atsv ||- (scrapeOptions.actions && scrapeOptions.actions.length > 0)- ) {- return null;- }-- return "cache:" + url + ":waitFor:" + scrapeOptions.waitFor;-}--export type CacheEntry = {- url: string;- html: string;- statusCode: number;- error?: string;-};--export async function saveEntryToCache(key: string, entry: CacheEntry) {- if (!cacheRedis) return;-- if (!entry.html || entry.html.length < 100) {- logger.warn("Skipping cache save for short HTML", {- key,- htmlLength: entry.html?.length,- });- return;- }-- try {- await cacheRedis.set(key, JSON.stringify(entry), "EX", 14400); // 4 hours in seconds- } catch (error) {- logger.warn("Failed to save to cache", { key, error });- }-}--export async function getEntryFromCache(- key: string,-): Promise<CacheEntry | null> {- if (!cacheRedis) return null;-- try {- return JSON.parse((await cacheRedis.get(key)) ?? "null");- } catch (error) {- logger.warn("Failed to get from cache", { key, error });- return null;- }-}diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.tsindex 8d2c6b6db..1a8238f0b 100644--- a/apps/api/src/lib/deep-research/deep-research-service.ts+++ b/apps/api/src/lib/deep-research/deep-research-service.ts@@ -131,9 +131,8 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {removeBase64Images: false,fastMode: false,blockAds: false,- maxAge: 0,+ maxAge: 4 * 60 * 60 * 1000,storeInCache: true,- __experimental_cache: true,},}, logger, costTracking, acuc?.flags ?? null);return response.length > 0 ? response : [];diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex ec603e325..f6ae72c2a 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -42,10 +42,9 @@ export async function scrapeDocument(team_id: options.teamId,scrapeOptions: scrapeOptions.parse({...internalScrapeOptions,- maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache+ maxAge: 4 * 60 * 60 * 1000,}),internalOptions: {- useCache: true,teamId: options.teamId,saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,bypassBilling: true,diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex a502a7367..f00038b6c 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -41,10 +41,9 @@ export async function scrapeDocument_F0(team_id: options.teamId,scrapeOptions: scrapeOptions.parse({...internalScrapeOptions,- maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache+ maxAge: 4 * 60 * 60 * 1000,}),internalOptions: {- useCache: true,teamId: options.teamId,bypassBilling: true,},diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex 65e88fb8d..d7446f600 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -10,105 +10,105 @@ const PDF_CACHE_PREFIX = "pdf-cache-v2/";* Directly hashes the content without any conversion*/export function createPdfCacheKey(pdfContent: string | Buffer): string {- return crypto- .createHash('sha256')- .update(pdfContent)- .digest('hex');+ return crypto+ .createHash('sha256')+ .update(pdfContent)+ .digest('hex');}/*** Save RunPod markdown results to GCS cache*/export async function savePdfResultToCache(- pdfContent: string,- result: { markdown: string; html: string }+ pdfContent: string,+ result: { markdown: string; html: string }): Promise<string | null> {- try {- if (!process.env.GCS_BUCKET_NAME) {- return null;- }+ try {+ if (!process.env.GCS_BUCKET_NAME) {+ return null;+ }- const cacheKey = createPdfCacheKey(pdfContent);- const storage = new Storage({ credentials });- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);- const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);+ const cacheKey = createPdfCacheKey(pdfContent);+ const storage = new Storage({ credentials });+ const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);++ for (let i = 0; i < 3; i++) {+ try {+ await blob.save(JSON.stringify(result), {+ contentType: "application/json",+ metadata: {+ source: "runpod_pdf_conversion",+ cache_type: "pdf_markdown",+ created_at: new Date().toISOString(),+ }+ });++ logger.info(`Saved PDF RunPod result to GCS cache`, {+ cacheKey,+ });++ return cacheKey;+ } catch (error) {+ if (i === 2) {+ throw error;+ } else {+ logger.error(`Error saving PDF RunPod result to GCS cache, retrying`, {+ error,+ cacheKey,+ i,+ });+ }+ }+ }- for (let i = 0; i < 3; i++) {- try {- await blob.save(JSON.stringify(result), {- contentType: "application/json",- metadata: {- source: "runpod_pdf_conversion",- cache_type: "pdf_markdown",- created_at: new Date().toISOString(),- }- });-- logger.info(`Saved PDF RunPod result to GCS cache`, {- cacheKey,- });-return cacheKey;- } catch (error) {- if (i === 2) {- throw error;- } else {- logger.error(`Error saving PDF RunPod result to GCS cache, retrying`, {+ } catch (error) {+ logger.error(`Error saving PDF RunPod result to GCS cache`, {error,- cacheKey,- i,- });- }- }+ });+ return null;}-- return cacheKey;- } catch (error) {- logger.error(`Error saving PDF RunPod result to GCS cache`, {- error,- });- return null;- }}/*** Get cached RunPod markdown results from GCS*/export async function getPdfResultFromCache(- pdfContent: string+ pdfContent: string): Promise<{ markdown: string; html: string } | null> {- try {- if (!process.env.GCS_BUCKET_NAME) {- return null;- }+ try {+ if (!process.env.GCS_BUCKET_NAME) {+ return null;+ }++ const cacheKey = createPdfCacheKey(pdfContent);+ const storage = new Storage({ credentials });+ const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);+ const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);++ const [exists] = await blob.exists();+ if (!exists) {+ logger.debug(`PDF RunPod result not found in GCS cache`, {+ cacheKey,+ });+ return null;+ }++ const [content] = await blob.download();+ const result = JSON.parse(content.toString());- const cacheKey = createPdfCacheKey(pdfContent);- const storage = new Storage({ credentials });- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);- const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);-- const [exists] = await blob.exists();- if (!exists) {- logger.debug(`PDF RunPod result not found in GCS cache`, {- cacheKey,- });- return null;+ logger.info(`Retrieved PDF RunPod result from GCS cache`, {+ cacheKey,+ });++ return {+ ...result,+ };+ } catch (error) {+ logger.error(`Error retrieving PDF RunPod result from GCS cache`, {+ error,+ });+ return null;}-- const [content] = await blob.download();- const result = JSON.parse(content.toString());-- logger.info(`Retrieved PDF RunPod result from GCS cache`, {- cacheKey,- });-- return {- ...result,- };- } catch (error) {- logger.error(`Error retrieving PDF RunPod result from GCS cache`, {- error,- });- return null;- }-}+}\ No newline at end of filediff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 91a54b315..90fd9afe5 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -349,13 +349,17 @@ export class WebCrawler {setTimeout(() => reject(new Error("Sitemap fetch timeout")), timeout);});+ // Allow sitemaps to be cached for 48 hours if they are requested from /map+ // - mogery+ const maxAge = (fromMap && !onlySitemap) ? 48 * 60 * 60 * 1000 : 0;+try {let count = (await Promise.race([Promise.all([- this.tryFetchSitemapLinks(this.initialUrl, _urlsHandler, abort, mock),+ this.tryFetchSitemapLinks(this.initialUrl, _urlsHandler, abort, mock, maxAge),...this.robots.getSitemaps()- .map((x) => this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock)),+ .map((x) => this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock, maxAge)),]).then((results) => results.reduce((a, x) => a + x, 0)),timeoutPromise,])) as number;@@ -669,6 +673,7 @@ export class WebCrawler {urlsHandler: (urls: string[]) => unknown,abort?: AbortSignal,mock?: string,+ maxAge?: number,): Promise<number> {const sitemapUrl = url.endsWith(".xml")? url@@ -679,7 +684,7 @@ export class WebCrawler {// Try to get sitemap from the provided URL firsttry {sitemapCount = await getLinksFromSitemap(- { sitemapUrl, urlsHandler, mode: "fire-engine" },+ { sitemapUrl, urlsHandler, mode: "fire-engine", maxAge },this.logger,this.jobId,this.sitemapsHit,@@ -727,6 +732,7 @@ export class WebCrawler {);},mode: "fire-engine",+ maxAge,},this.logger,this.jobId,@@ -762,7 +768,7 @@ export class WebCrawler {const baseUrlSitemap = `${this.baseUrl}/sitemap.xml`;try {sitemapCount += await getLinksFromSitemap(- { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },+ { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge },this.logger,this.jobId,this.sitemapsHit,@@ -782,7 +788,7 @@ export class WebCrawler {// ignore 404} else {sitemapCount += await getLinksFromSitemap(- { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },+ { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge },this.logger,this.jobId,this.sitemapsHit,diff --git a/apps/api/src/scraper/WebScraper/sitemap-index.ts b/apps/api/src/scraper/WebScraper/sitemap-index.tsdeleted file mode 100644index f8826bf98..000000000--- a/apps/api/src/scraper/WebScraper/sitemap-index.ts+++ /dev/null@@ -1,62 +0,0 @@-import { logger } from "../../lib/logger";-import {- normalizeUrl,- normalizeUrlOnlyHostname,-} from "../../lib/canonical-url";-import { supabase_service } from "../../services/supabase";--/**- * Query the sitemap index for a given URL- * @param url The URL to query- * @returns A list of URLs found in the sitemap index aggregated from all sitemaps- */-import { withAuth } from "../../lib/withAuth";--async function querySitemapIndexFunction(url: string, abort?: AbortSignal) {- const originUrl = normalizeUrlOnlyHostname(url);-- for (let attempt = 1; attempt <= 3; attempt++) {- abort?.throwIfAborted();- try {- const { data, error } = await supabase_service- .from("crawl_maps")- .select("urls, updated_at")- .eq("origin_url", originUrl)- .order("updated_at", { ascending: false });-- if (error) {- throw error;- }-- if (!data || data.length === 0) {- return { urls: [], lastUpdated: new Date(0) };- }-- const allUrls = [- ...new Set(- data- .map((entry) => entry.urls)- .flat()- .map((url) => normalizeUrl(url)),- ),- ];- return { urls: allUrls, lastUpdated: data[0].updated_at };- } catch (error) {- logger.error("(sitemap-index) Error querying the index", {- error,- attempt,- });-- if (attempt === 3) {- return { urls: [], lastUpdated: new Date(0) };- }- }- }-- return { urls: [], lastUpdated: new Date(0) };-}--export const querySitemapIndex = withAuth(querySitemapIndexFunction, {- urls: [],- lastUpdated: new Date(0),-});diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 67d3a2ff3..a89c1e0fc 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -12,10 +12,12 @@ export async function getLinksFromSitemap(sitemapUrl,urlsHandler,mode = "axios",+ maxAge = 0,}: {sitemapUrl: string;urlsHandler(urls: string[]): unknown;mode?: "axios" | "fire-engine";+ maxAge?: number;},logger: Logger,crawlId: string,@@ -40,9 +42,10 @@ export async function getLinksFromSitemap(const response = await scrapeURL("sitemap;" + crawlId,sitemapUrl,- scrapeOptions.parse({ formats: ["rawHtml"], useMock: mock }),+ scrapeOptions.parse({ formats: ["rawHtml"], useMock: mock, maxAge }),{forceEngine: [+ ...(maxAge > 0 ? ["index" as const] : []),"fetch",...((mode === "fire-engine" && useFireEngine) ? ["fire-engine;tlsclient" as const] : []),],diff --git a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts b/apps/api/src/scraper/scrapeURL/engines/cache/index.tsdeleted file mode 100644index 49acbcd7b..000000000--- a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts+++ /dev/null@@ -1,29 +0,0 @@-import { cacheKey, getEntryFromCache } from "../../../../lib/cache";-import { EngineScrapeResult } from "..";-import { Meta } from "../..";-import { EngineError, IndexMissError } from "../../error";--export async function scrapeCache(meta: Meta): Promise<EngineScrapeResult> {- const key = cacheKey(meta.rewrittenUrl ?? meta.url, meta.options, meta.internalOptions);- if (key === null) throw new EngineError("Scrape not eligible for caching");-- const entry = await getEntryFromCache(key);- if (entry === null) throw new IndexMissError();-- if (!entry.html) {- throw new EngineError("Cache hit but HTML is missing");- }- if (entry.html.length < 100) {- throw new EngineError("Cache hit but HTML is too short to be useful");- }-- // Set fromCache flag to indicate this document was retrieved from cache- meta.internalOptions.fromCache = true;-- return {- url: entry.url,- html: entry.html,- statusCode: entry.statusCode,- error: entry.error,- };-}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex a2306f289..713e1959c 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -9,7 +9,6 @@ import {import { scrapePDF } from "./pdf";import { scrapeURLWithFetch } from "./fetch";import { scrapeURLWithPlaywright } from "./playwright";-import { scrapeCache } from "./cache";import { scrapeURLWithIndex } from "./index/index";import { useIndex } from "../../../services";@@ -26,7 +25,6 @@ export type Engine =| "fetch"| "pdf"| "docx"- | "cache"| "index"| "index;documents";@@ -36,12 +34,8 @@ const useFireEngine =const usePlaywright =process.env.PLAYWRIGHT_MICROSERVICE_URL !== "" &&process.env.PLAYWRIGHT_MICROSERVICE_URL !== undefined;-const useCache =- process.env.CACHE_REDIS_URL !== "" &&- process.env.CACHE_REDIS_URL !== undefined;export const engines: Engine[] = [- ...(useCache ? ["cache" as const] : []),...(useIndex ? ["index" as const, "index;documents" as const] : []),...(useFireEngine? [@@ -132,7 +126,6 @@ const engineHandlers: {timeToRun: number | undefined,) => Promise<EngineScrapeResult>;} = {- cache: scrapeCache,index: scrapeURLWithIndex,"index;documents": scrapeURLWithIndex,"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,@@ -159,24 +152,6 @@ export const engineOptions: {quality: number;};} = {- cache: {- features: {- actions: false,- waitFor: true,- screenshot: false,- "screenshot@fullScreen": false,- pdf: false, // TODO: figure this out- docx: false, // TODO: figure this out- atsv: false,- location: false,- mobile: false,- skipTlsVerification: false,- useFastMode: false,- stealthProxy: false,- disableAdblock: false,- },- quality: 1000, // cache should always be tried first- },index: {features: {actions: false,@@ -193,7 +168,7 @@ export const engineOptions: {stealthProxy: false,disableAdblock: true,},- quality: 999, // index should always be tried second ? - MG+ quality: 1000, // index should always be tried first},"fire-engine;chrome-cdp": {features: {@@ -442,13 +417,6 @@ export function buildFallbackList(meta: Meta): {...((!useFireEngine && meta.mock !== null) ? ["fire-engine;chrome-cdp", "fire-engine(retry);chrome-cdp", "fire-engine;chrome-cdp;stealth", "fire-engine(retry);chrome-cdp;stealth", "fire-engine;playwright", "fire-engine;tlsclient", "fire-engine;playwright;stealth", "fire-engine;tlsclient;stealth"] as Engine[] : [])];- if (meta.internalOptions.useCache !== true) {- const cacheIndex = _engines.indexOf("cache");- if (cacheIndex !== -1) {- _engines.splice(cacheIndex, 1);- }- }-const shouldUseIndex =useIndex&& process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true"diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex bf6d1fdb0..fb0c765de 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -7,7 +7,6 @@ import crypto from "crypto";export async function sendDocumentToIndex(meta: Meta, document: Document) {const shouldCache = meta.options.storeInCache- && meta.winnerEngine !== "cache"&& meta.winnerEngine !== "index"&& meta.winnerEngine !== "index;documents"&& (diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 3dd97995e..8bc968130 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -16,11 +16,8 @@ import {import { readFile, unlink } from "node:fs/promises";import path from "node:path";import type { Response } from "undici";-import {- getPdfResultFromCache,- savePdfResultToCache,-} from "../../../../lib/gcs-pdf-cache";import { getPageCount } from "../../../../lib/pdf-parser";+import { getPdfResultFromCache, savePdfResultToCache } from "../../../../lib/gcs-pdf-cache";type PDFProcessorResult = { html: string; markdown?: string };diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 73f9ebb2c..a2a822d6f 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -192,10 +192,8 @@ export type InternalOptions = {v0CrawlOnlyUrls?: boolean;v0DisableJsDom?: boolean;- useCache?: boolean;disableSmartWaitCache?: boolean; // Passed along to fire-engineisBackgroundIndex?: boolean;- fromCache?: boolean; // Indicates if the document was retrieved from cacheabort?: AbortSignal;urlInvisibleInCurrentCrawl?: boolean;unnormalizedSourceURL?: string;@@ -410,7 +408,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {numPages: result.result.numPages,contentType: result.result.contentType,proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",- ...(results["index"] ? (+ ...((results["index"] || results["index;documents"]) ? (result.result.cacheInfo ? {cacheState: "hit",cachedAt: result.result.cacheInfo.created_at.toISOString(),diff --git a/apps/api/src/scraper/scrapeURL/transformers/cache.ts b/apps/api/src/scraper/scrapeURL/transformers/cache.tsdeleted file mode 100644index 96c3d4378..000000000--- a/apps/api/src/scraper/scrapeURL/transformers/cache.ts+++ /dev/null@@ -1,41 +0,0 @@-import { Document } from "../../../controllers/v1/types";-import { Meta } from "..";-import { CacheEntry, cacheKey, saveEntryToCache } from "../../../lib/cache";--export function saveToCache(meta: Meta, document: Document): Document {- if (meta.internalOptions.useCache !== true) {- return document;- }-- if (- document.metadata.statusCode! < 200 ||- document.metadata.statusCode! >= 300- )- return document;-- if (document.rawHtml === undefined) {- throw new Error(- "rawHtml is undefined -- this transformer is being called out of order",- );- }-- // If the document was retrieved from cache, we don't need to save it- if (meta.internalOptions.fromCache) {- return document;- }-- const key = cacheKey(meta.url, meta.options, meta.internalOptions);-- if (key !== null) {- const entry: CacheEntry = {- html: document.rawHtml!,- statusCode: document.metadata.statusCode!,- url: document.metadata.url ?? document.metadata.sourceURL!,- error: document.metadata.error ?? undefined,- };-- saveEntryToCache(key, entry);- }-- return document;-}diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex f229793b3..a9e2742c0 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -7,7 +7,6 @@ import { extractMetadata } from "../lib/extractMetadata";import { performLLMExtract } from "./llmExtract";import { uploadScreenshot } from "./uploadScreenshot";import { removeBase64Images } from "./removeBase64Images";-import { saveToCache } from "./cache";import { performAgent } from "./agent";import { deriveDiff } from "./diff";@@ -202,7 +201,6 @@ export function coerceFieldsToFormats(// TODO: allow some of these to run in parallelexport const transformerStack: Transformer[] = [- saveToCache,deriveHTMLFromRawHTML,deriveMarkdownFromHTML,deriveLinksFromHTML,diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 08a901775..882fed151 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -237,7 +237,7 @@ export async function getIndexInsertQueueLength(): Promise<number> {return await redisEvictConnection.llen(INDEX_INSERT_QUEUE_KEY) ?? 0;}-export async function queryIndexAtSplitLevel(url: string, limit: number): Promise<string[]> {+export async function queryIndexAtSplitLevel(url: string, limit: number, maxAge = 2 * 24 * 60 * 60 * 1000): Promise<string[]> {if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {return [];}@@ -258,7 +258,7 @@ export async function queryIndexAtSplitLevel(url: string, limit: number): Promis.rpc("query_index_at_split_level", {i_level: level,i_url_hash: urlSplitsHash[level],- i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),+ i_newer_than: new Date(Date.now() - maxAge).toISOString(),}).range(iteration * 1000, (iteration + 1) * 1000)@@ -286,7 +286,7 @@ export async function queryIndexAtSplitLevel(url: string, limit: number): Promis}}-export async function queryIndexAtDomainSplitLevel(hostname: string, limit: number): Promise<string[]> {+export async function queryIndexAtDomainSplitLevel(hostname: string, limit: number, maxAge = 2 * 24 * 60 * 60 * 1000): Promise<string[]> {if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {return [];}@@ -307,7 +307,7 @@ export async function queryIndexAtDomainSplitLevel(hostname: string, limit: numb.rpc("query_index_at_domain_split_level", {i_level: level,i_domain_hash: domainSplitsHash[level],- i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),+ i_newer_than: new Date(Date.now() - maxAge).toISOString(),}).range(iteration * 1000, (iteration + 1) * 1000)diff --git a/apps/api/src/services/indexing/crawl-maps-index.ts b/apps/api/src/services/indexing/crawl-maps-index.tsdeleted file mode 100644index 957f3effe..000000000--- a/apps/api/src/services/indexing/crawl-maps-index.ts+++ /dev/null@@ -1,278 +0,0 @@-import { logger } from "../../lib/logger";-import {- normalizeUrl,- normalizeUrlOnlyHostname,-} from "../../lib/canonical-url";-import { supabase_service } from "../supabase";-import { redisConnection } from "../queue-service";--const BATCH_KEY = "crawl_maps_batch";-const BATCH_LOCK_KEY = "crawl_maps_batch_lock";-const BATCH_SIZE = 20;-const BATCH_TIMEOUT = 20000; // 10 seconds-const LOCK_TIMEOUT = 30000; // 30 seconds--interface CrawlMapOperation {- originUrl: string;- standardizedUrls: string[];- timestamp: string;-}--interface CrawlMapRecord {- id?: string;- origin_url: string;- urls: string[];- num_urls: number;- updated_at: string;- created_at?: string;-}--async function acquireLock(): Promise<boolean> {- const redis = redisConnection;- // Set lock with NX (only if it doesn't exist) and PX (millisecond expiry)- const result = await redis.set(BATCH_LOCK_KEY, "1", "PX", LOCK_TIMEOUT, "NX");- const acquired = result === "OK";- if (acquired) {- logger.info("🔒 Acquired batch processing lock");- }- return acquired;-}--async function releaseLock() {- const redis = redisConnection;- await redis.del(BATCH_LOCK_KEY);- logger.info("🔓 Released batch processing lock");-}--async function processBatch() {- const redis = redisConnection;-- // Try to acquire lock- if (!(await acquireLock())) {- return;- }-- try {- // Get all operations from Redis list- const operations: CrawlMapOperation[] = [];- while (operations.length < BATCH_SIZE) {- const op = await redis.lpop(BATCH_KEY);- if (!op) break;- operations.push(JSON.parse(op));- }-- if (operations.length === 0) {- logger.info("No operations to process in batch");- return;- }-- logger.info(`📦 Processing batch of ${operations.length} operations`, {- origins: operations.map((op) => op.originUrl),- });-- // Get existing maps for all origins in batch- const origins = operations.map((op) => op.originUrl);- const { data: existingMaps } = await supabase_service- .from("crawl_maps")- .select("id, origin_url, urls, updated_at")- .in("origin_url", origins)- .order("updated_at", { ascending: false });-- // Group maps by origin and handle duplicates- const mapsByOrigin = new Map<string, any[]>();- existingMaps?.forEach((map) => {- const maps = mapsByOrigin.get(map.origin_url) || [];- maps.push(map);- mapsByOrigin.set(map.origin_url, maps);- });-- // Handle duplicates and prepare updates- const updates: CrawlMapRecord[] = [];- const inserts: CrawlMapRecord[] = [];- const duplicatesToDelete: string[] = [];-- // Track processed origins to avoid duplicates in the same batch- const processedOrigins = new Set<string>();-- for (const op of operations) {- // Skip if we've already processed this origin in this batch- if (processedOrigins.has(op.originUrl)) {- continue;- }- processedOrigins.add(op.originUrl);-- const existingForOrigin = mapsByOrigin.get(op.originUrl) || [];-- if (existingForOrigin.length > 0) {- // Keep most recent entry and mark others for deletion- const [mostRecent, ...duplicates] = existingForOrigin;- if (duplicates.length > 0) {- duplicatesToDelete.push(...duplicates.map((d) => d.id));- }-- // Merge and deduplicate URLs- const mergedUrls = [- ...new Set([- ...mostRecent.urls,- ...op.standardizedUrls.map((url) => normalizeUrl(url)),- ]),- ];-- updates.push({- id: mostRecent.id,- origin_url: op.originUrl,- urls: mergedUrls,- num_urls: mergedUrls.length,- updated_at: op.timestamp,- });- } else {- // Prepare insert with deduplicated URLs- const deduplicatedUrls = [- ...new Set(op.standardizedUrls.map((url) => normalizeUrl(url))),- ];- inserts.push({- origin_url: op.originUrl,- urls: deduplicatedUrls,- num_urls: deduplicatedUrls.length,- created_at: op.timestamp,- updated_at: op.timestamp,- });- }- }-- // Delete duplicate entries- if (duplicatesToDelete.length > 0) {- logger.info(- `🗑️ Deleting ${duplicatesToDelete.length} duplicate crawl maps in batches of 100`,- );-- // Delete in batches of 100- for (let i = 0; i < duplicatesToDelete.length; i += 100) {- const batch = duplicatesToDelete.slice(i, i + 100);- const { error: deleteError } = await supabase_service- .from("crawl_maps")- .delete()- .in("id", batch);-- if (deleteError) {- logger.error(- `Failed to delete batch ${i / 100 + 1} of duplicate crawl maps`,- {- error: deleteError,- batchSize: batch.length,- startIndex: i,- },- );- }- }- }-- // Execute batch operations- if (updates.length > 0) {- logger.info(`🔄 Updating ${updates.length} existing crawl maps`, {- origins: updates.map((u) => u.origin_url),- });-- // Process updates one at a time to avoid conflicts- for (const update of updates) {- const { error: updateError } = await supabase_service- .from("crawl_maps")- .upsert(update);-- if (updateError) {- logger.error("Failed to update crawl map", {- error: updateError,- origin: update.origin_url,- });- }- }- }-- if (inserts.length > 0) {- logger.info(`➕ Inserting ${inserts.length} new crawl maps`, {- origins: inserts.map((i) => i.origin_url),- });- const { error: insertError } = await supabase_service- .from("crawl_maps")- .insert(inserts);-- if (insertError) {- logger.error("Failed to batch insert crawl maps", {- error: insertError,- });- }- }-- logger.info("✅ Batch processing completed successfully");- } catch (error) {- logger.error("Error processing crawl map batch", { error });- } finally {- await releaseLock();- }-}--// Start periodic batch processing-let batchInterval: NodeJS.Timeout | null = null;--function startBatchProcessing() {- if (batchInterval) return;-- logger.info("🔄 Starting periodic batch processing");- batchInterval = setInterval(async () => {- const queueLength = await redisConnection.llen(BATCH_KEY);- logger.info(`Checking batch queue (${queueLength} items pending)`);- await processBatch();- }, BATCH_TIMEOUT);-- // Unref to not keep process alive- batchInterval.unref();-}--export async function saveCrawlMap(originUrl: string, visitedUrls: string[]) {- logger.info("Queueing crawl map", { originUrl });- originUrl = normalizeUrlOnlyHostname(originUrl);-- try {- // Standardize URLs to canonical form- const standardizedUrls = [- ...new Set(visitedUrls.map((url) => normalizeUrl(url))),- ];-- const operation: CrawlMapOperation = {- originUrl,- standardizedUrls,- timestamp: new Date().toISOString(),- };-- // Add operation to Redis list- const redis = redisConnection;- await redis.rpush(BATCH_KEY, JSON.stringify(operation));- const queueLength = await redis.llen(BATCH_KEY);- logger.info(`📥 Added operation to queue (${queueLength} total pending)`, {- originUrl,- });-- // Start batch processing if not already started- startBatchProcessing();-- // If we have enough items, trigger immediate processing- if (queueLength >= BATCH_SIZE) {- logger.info(- "🔄 Queue reached batch size, triggering immediate processing",- );- await processBatch();- }- } catch (error) {- logger.error("Error queueing crawl map", { error });- }-}--// Cleanup on exit-process.on("beforeExit", async () => {- if (batchInterval) {- clearInterval(batchInterval);- batchInterval = null;- logger.info("Stopped periodic batch processing");- }- await processBatch();-});diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 06ad21f5d..7ff0b9f4c 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -5,12 +5,8 @@ import { Job, Queue, Worker } from "bullmq";import { logger as _logger, logger } from "../../lib/logger";import {redisConnection,- indexQueueName,- getIndexQueue,- billingQueueName,getBillingQueue,} from "../queue-service";-import { saveCrawlMap } from "./crawl-maps-index";import { processBillingBatch, queueBillingOperation, startBillingBatchProcessing } from "../billing/batch_billing";import systemMonitor from "../system-monitor";import { v4 as uuidv4 } from "uuid";@@ -33,39 +29,6 @@ const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;const runningJobs: Set<string> = new Set();-const processJobInternal = async (token: string, job: Job) => {- if (!job.id) {- throw new Error("Job has no ID");- }-- const logger = _logger.child({- module: "index-worker",- method: "processJobInternal",- jobId: job.id,- });-- const extendLockInterval = setInterval(async () => {- logger.info(`🔄 Worker extending lock on job ${job.id}`);- await job.extendLock(token, jobLockExtensionTime);- }, jobLockExtendInterval);-- let err = null;- try {- const { originUrl, visitedUrls } = job.data;- await saveCrawlMap(originUrl, visitedUrls);- await job.moveToCompleted({ success: true }, token, false);- } catch (error) {- logger.error("Error processing index job", { error });- Sentry.captureException(error);- err = error;- await job.moveToFailed(error, token, false);- } finally {- clearInterval(extendLockInterval);- }-- return err;-};-// Create a processor for billing jobsconst processBillingJobInternal = async (token: string, job: Job) => {if (!job.id) {@@ -233,9 +196,6 @@ const WEBHOOK_INSERT_INTERVAL = 15000;// Start the workers(async () => {- // Start index worker- const indexWorkerPromise = workerFun(getIndexQueue(), processJobInternal);-// Start billing worker and batch processingstartBillingBatchProcessing();const billingWorkerPromise = workerFun(getBillingQueue(), processBillingJobInternal);@@ -255,8 +215,8 @@ const WEBHOOK_INSERT_INTERVAL = 15000;await processWebhookInsertJobs();}, WEBHOOK_INSERT_INTERVAL);- // Wait for both workers to complete (which should only happen on shutdown)- await Promise.all([indexWorkerPromise, billingWorkerPromise]);+ // Wait for all workers to complete (which should only happen on shutdown)+ await Promise.all([billingWorkerPromise]);clearInterval(indexInserterInterval);clearInterval(webhookInserterInterval);diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex b4938dce9..77f62c73a 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -60,24 +60,6 @@ export function getExtractQueue() {return extractQueue;}-export function getIndexQueue() {- if (!indexQueue) {- indexQueue = new Queue(indexQueueName, {- connection: redisConnection,- defaultJobOptions: {- removeOnComplete: {- age: 3600, // 1 hour- },- removeOnFail: {- age: 3600, // 1 hour- },- },- });- logger.info("Index queue created");- }- return indexQueue;-}-export function getGenerateLlmsTxtQueue() {if (!generateLlmsTxtQueue) {generateLlmsTxtQueue = new Queue(generateLlmsTxtQueueName, {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 2d56cc45f..54a9e6ff4 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -6,7 +6,6 @@ import {getScrapeQueue,getExtractQueue,getDeepResearchQueue,- getIndexQueue,redisConnection,getGenerateLlmsTxtQueue,getBillingQueue,@@ -64,7 +63,6 @@ import { supabase_service } from "../services/supabase";import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";import { saveExtract, updateExtract } from "../lib/extract/extract-redis";import { billTeam } from "./billing/credit_billing";-import { saveCrawlMap } from "./indexing/crawl-maps-index";import { updateDeepResearch } from "../lib/deep-research/deep-research-redis";import { performDeepResearch } from "../lib/deep-research/deep-research-service";import { performGenerateLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-service";@@ -253,35 +251,6 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {logger.info("Finishing crawl");await finishCrawl(job.data.crawl_id);- (async () => {- const originUrl = sc.originUrl- ? normalizeUrlOnlyHostname(sc.originUrl)- : undefined;- // Get all visited unique URLs from Redis- const visitedUrls = await redisEvictConnection.smembers(- "crawl:" + job.data.crawl_id + ":visited_unique",- );- // Upload to Supabase if we have URLs and this is a crawl (not a batch scrape)- if (- visitedUrls.length > 0 &&- job.data.crawlerOptions !== null &&- originUrl &&- process.env.USE_DB_AUTHENTICATION === "true"- ) {- // Queue the indexing job instead of doing it directly- await getIndexQueue().add(- job.data.crawl_id,- {- originUrl,- visitedUrls,- },- {- priority: 10,- },- );- }- })();-if (!job.data.v1) {const jobIDs = await getCrawlJobs(job.data.crawl_id);