Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Remove old cache mechanisms (redis cache, PDF cache, crawl maps, etc.) (FIR-2266) (#1667)

Gergő Móricz3f0b8b8e2742
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index 1336a87bd..3a2f0e796 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -161,6 +161,29 @@ describe("Scrape tests", () => {
expect(response4.metadata.cacheState).toBe("miss");
}, 150000 + 2 * 17000);
+ it.concurrent("caches PDFs properly", async () => {
+ const id = crypto.randomUUID();
+ const url = "https://www.orimi.com/pdf-test.pdf?testId=" + id;
+
+ const response1 = await scrape({
+ url,
+ timeout: 60000,
+ maxAge: 120000,
+ });
+
+ expect(response1.metadata.cacheState).toBe("miss");
+
+ await new Promise(resolve => setTimeout(resolve, 17000));
+
+ const response2 = await scrape({
+ url,
+ timeout: 60000,
+ maxAge: 120000,
+ });
+
+ expect(response2.metadata.cacheState).toBe("hit");
+ }, 150000 + 2 * 17000);
+
it.concurrent("respects screenshot", async () => {
const id = crypto.randomUUID();
const url = "https://firecrawl.dev/?testId=" + id;
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 111b8c818..06423e4c6 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -23,8 +23,6 @@ import { logJob } from "../../services/logging/log_job";
import { performCosineSimilarity } from "../../lib/map-cosine";
import { logger } from "../../lib/logger";
import Redis from "ioredis";
-import { querySitemapIndex } from "../../scraper/WebScraper/sitemap-index";
-import { getIndexQueue } from "../../services/queue-service";
import { generateURLSplits, queryIndexAtDomainSplitLevel, queryIndexAtSplitLevel } from "../../services/index";
configDotenv();
@@ -44,7 +42,7 @@ interface MapResult {
mapResults: MapDocument[];
}
-async function queryIndex(url: string, limit: number, useIndex: boolean): Promise<string[]> {
+async function queryIndex(url: string, limit: number, useIndex: boolean, includeSubdomains: boolean): Promise<string[]> {
if (!useIndex) {
return [];
}
@@ -54,9 +52,9 @@ async function queryIndex(url: string, limit: number, useIndex: boolean): Promis
const urlObj = new URL(url);
const hostname = urlObj.hostname;
- // TEMP: this can be removed in 2 days
- const domainLinks = await queryIndexAtDomainSplitLevel(hostname, limit);
- const splitLinks = await queryIndexAtSplitLevel(url, limit);
+ // TEMP: this should be altered on June 15th 2025 7AM PT - mogery
+ const domainLinks = includeSubdomains ? await queryIndexAtDomainSplitLevel(hostname, limit, 14 * 24 * 60 * 60 * 1000) : [];
+ const splitLinks = await queryIndexAtSplitLevel(url, limit, 14 * 24 * 60 * 60 * 1000);
return Array.from(new Set([...domainLinks, ...splitLinks]));
} else {
@@ -188,9 +186,8 @@ export async function getMapResults({
}
// Parallelize sitemap index query with search results
- const [sitemapIndexResult, indexResults, ...searchResults] = await Promise.all([
- querySitemapIndex(url, abort),
- queryIndex(url, limit, useIndex),
+ const [indexResults, ...searchResults] = await Promise.all([
+ queryIndex(url, limit, useIndex, includeSubdomains),
...(cachedResult ? [] : pagePromises),
]);
@@ -198,14 +195,10 @@ export async function getMapResults({
links.push(...indexResults);
}
- const twoDaysAgo = new Date();
- twoDaysAgo.setDate(twoDaysAgo.getDate() - 2);
-
- // If sitemap is not ignored and either we have few URLs (<100) or the data is stale (>2 days old), fetch fresh sitemap
+ // If sitemap is not ignored, fetch sitemap
+ // This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 days
if (
- !ignoreSitemap &&
- (sitemapIndexResult.urls.length < 100 ||
- new Date(sitemapIndexResult.lastUpdated) < twoDaysAgo)
+ !ignoreSitemap
) {
try {
await crawler.tryGetSitemap(
@@ -250,9 +243,6 @@ export async function getMapResults({
}
}
- // Add sitemap-index URLs
- links.push(...sitemapIndexResult.urls);
-
// Perform cosine similarity between the search query and the list of links
if (search) {
const searchQuery = search.toLowerCase();
@@ -308,19 +298,6 @@ export async function getMapResults({
? links
: links.slice(0, limit);
- //
-
- await getIndexQueue().add(
- id,
- {
- originUrl: url,
- visitedUrls: linksToReturn.filter(link => link !== url),
- },
- {
- priority: 10,
- },
- );
-
return {
success: true,
links: linksToReturn,
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 29d78a420..993f3d4f6 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -51,12 +51,16 @@ export async function scrapeController(
url: req.body.url,
mode: "single_urls",
team_id: req.auth.team_id,
- scrapeOptions: req.body,
+ scrapeOptions: {
+ ...req.body,
+ ...(req.body.__experimental_cache ? {
+ maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours
+ } : {}),
+ },
internalOptions: {
teamId: req.auth.team_id,
saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
unnormalizedSourceURL: preNormalizedBody.url,
- useCache: req.body.__experimental_cache ? true : false,
bypassBilling: isDirectToBullMQ,
},
origin,
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index b716d5a9a..65b45836d 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -101,9 +101,9 @@ async function scrapeSearchResult(
team_id: options.teamId,
scrapeOptions: {
...options.scrapeOptions,
- maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache
+ maxAge: 4 * 60 * 60 * 1000,
},
- internalOptions: { teamId: options.teamId, useCache: true, bypassBilling: true },
+ internalOptions: { teamId: options.teamId, bypassBilling: true },
origin: options.origin,
is_scrape: true,
startTime: Date.now(),
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index ca35ab5bc..f1e8ed34f 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -328,6 +328,7 @@ const baseScrapeOptions = z
proxy: z.enum(["basic", "stealth", "auto"]).optional(),
maxAge: z.number().int().gte(0).safe().default(0),
storeInCache: z.boolean().default(true),
+ // @deprecated
__experimental_cache: z.boolean().default(false).optional(),
__searchPreviewToken: z.string().optional(),
})
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 05fe005ee..7e79a49a4 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -7,7 +7,6 @@ import cors from "cors";
import {
getExtractQueue,
getScrapeQueue,
- getIndexQueue,
getGenerateLlmsTxtQueue,
getDeepResearchQueue,
getBillingQueue,
@@ -57,7 +56,6 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [
new BullAdapter(getScrapeQueue()),
new BullAdapter(getExtractQueue()),
- new BullAdapter(getIndexQueue()),
new BullAdapter(getGenerateLlmsTxtQueue()),
new BullAdapter(getDeepResearchQueue()),
new BullAdapter(getBillingQueue()),
diff --git a/apps/api/src/lib/cache.ts b/apps/api/src/lib/cache.ts
deleted file mode 100644
index a60664373..000000000
--- a/apps/api/src/lib/cache.ts
+++ /dev/null
@@ -1,70 +0,0 @@
-import IORedis from "ioredis";
-import { ScrapeOptions } from "../controllers/v1/types";
-import { InternalOptions } from "../scraper/scrapeURL";
-import { logger as _logger } from "./logger";
-const logger = _logger.child({ module: "cache" });
-
-export const cacheRedis = process.env.CACHE_REDIS_URL
- ? new IORedis(process.env.CACHE_REDIS_URL, {
- maxRetriesPerRequest: null,
- })
- : null;
-
-export function cacheKey(
- url: string,
- scrapeOptions: ScrapeOptions,
- internalOptions: InternalOptions,
-): string | null {
- if (!cacheRedis) return null;
-
- // these options disqualify a cache
- if (
- internalOptions.v0CrawlOnlyUrls ||
- internalOptions.forceEngine?.includes("cache") ||
- scrapeOptions.fastMode ||
- internalOptions.atsv ||
- (scrapeOptions.actions && scrapeOptions.actions.length > 0)
- ) {
- return null;
- }
-
- return "cache:" + url + ":waitFor:" + scrapeOptions.waitFor;
-}
-
-export type CacheEntry = {
- url: string;
- html: string;
- statusCode: number;
- error?: string;
-};
-
-export async function saveEntryToCache(key: string, entry: CacheEntry) {
- if (!cacheRedis) return;
-
- if (!entry.html || entry.html.length < 100) {
- logger.warn("Skipping cache save for short HTML", {
- key,
- htmlLength: entry.html?.length,
- });
- return;
- }
-
- try {
- await cacheRedis.set(key, JSON.stringify(entry), "EX", 14400); // 4 hours in seconds
- } catch (error) {
- logger.warn("Failed to save to cache", { key, error });
- }
-}
-
-export async function getEntryFromCache(
- key: string,
-): Promise<CacheEntry | null> {
- if (!cacheRedis) return null;
-
- try {
- return JSON.parse((await cacheRedis.get(key)) ?? "null");
- } catch (error) {
- logger.warn("Failed to get from cache", { key, error });
- return null;
- }
-}
diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.ts
index 8d2c6b6db..1a8238f0b 100644
--- a/apps/api/src/lib/deep-research/deep-research-service.ts
+++ b/apps/api/src/lib/deep-research/deep-research-service.ts
@@ -131,9 +131,8 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
removeBase64Images: false,
fastMode: false,
blockAds: false,
- maxAge: 0,
+ maxAge: 4 * 60 * 60 * 1000,
storeInCache: true,
- __experimental_cache: true,
},
}, logger, costTracking, acuc?.flags ?? null);
return response.length > 0 ? response : [];
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index ec603e325..f6ae72c2a 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -42,10 +42,9 @@ export async function scrapeDocument(
team_id: options.teamId,
scrapeOptions: scrapeOptions.parse({
...internalScrapeOptions,
- maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache
+ maxAge: 4 * 60 * 60 * 1000,
}),
internalOptions: {
- useCache: true,
teamId: options.teamId,
saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
bypassBilling: true,
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index a502a7367..f00038b6c 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -41,10 +41,9 @@ export async function scrapeDocument_F0(
team_id: options.teamId,
scrapeOptions: scrapeOptions.parse({
...internalScrapeOptions,
- maxAge: 4 * 60 * 60 * 1000, // 4 hours, same as useCache
+ maxAge: 4 * 60 * 60 * 1000,
}),
internalOptions: {
- useCache: true,
teamId: options.teamId,
bypassBilling: true,
},
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index 65e88fb8d..d7446f600 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -10,105 +10,105 @@ const PDF_CACHE_PREFIX = "pdf-cache-v2/";
* Directly hashes the content without any conversion
*/
export function createPdfCacheKey(pdfContent: string | Buffer): string {
- return crypto
- .createHash('sha256')
- .update(pdfContent)
- .digest('hex');
+ return crypto
+ .createHash('sha256')
+ .update(pdfContent)
+ .digest('hex');
}
/**
* Save RunPod markdown results to GCS cache
*/
export async function savePdfResultToCache(
- pdfContent: string,
- result: { markdown: string; html: string }
+ pdfContent: string,
+ result: { markdown: string; html: string }
): Promise<string | null> {
- try {
- if (!process.env.GCS_BUCKET_NAME) {
- return null;
- }
+ try {
+ if (!process.env.GCS_BUCKET_NAME) {
+ return null;
+ }
- const cacheKey = createPdfCacheKey(pdfContent);
- const storage = new Storage({ credentials });
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
- const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
+ const cacheKey = createPdfCacheKey(pdfContent);
+ const storage = new Storage({ credentials });
+ const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
+
+ for (let i = 0; i < 3; i++) {
+ try {
+ await blob.save(JSON.stringify(result), {
+ contentType: "application/json",
+ metadata: {
+ source: "runpod_pdf_conversion",
+ cache_type: "pdf_markdown",
+ created_at: new Date().toISOString(),
+ }
+ });
+
+ logger.info(`Saved PDF RunPod result to GCS cache`, {
+ cacheKey,
+ });
+
+ return cacheKey;
+ } catch (error) {
+ if (i === 2) {
+ throw error;
+ } else {
+ logger.error(`Error saving PDF RunPod result to GCS cache, retrying`, {
+ error,
+ cacheKey,
+ i,
+ });
+ }
+ }
+ }
- for (let i = 0; i < 3; i++) {
- try {
- await blob.save(JSON.stringify(result), {
- contentType: "application/json",
- metadata: {
- source: "runpod_pdf_conversion",
- cache_type: "pdf_markdown",
- created_at: new Date().toISOString(),
- }
- });
-
- logger.info(`Saved PDF RunPod result to GCS cache`, {
- cacheKey,
- });
-
return cacheKey;
- } catch (error) {
- if (i === 2) {
- throw error;
- } else {
- logger.error(`Error saving PDF RunPod result to GCS cache, retrying`, {
+ } catch (error) {
+ logger.error(`Error saving PDF RunPod result to GCS cache`, {
error,
- cacheKey,
- i,
- });
- }
- }
+ });
+ return null;
}
-
- return cacheKey;
- } catch (error) {
- logger.error(`Error saving PDF RunPod result to GCS cache`, {
- error,
- });
- return null;
- }
}
/**
* Get cached RunPod markdown results from GCS
*/
export async function getPdfResultFromCache(
- pdfContent: string
+ pdfContent: string
): Promise<{ markdown: string; html: string } | null> {
- try {
- if (!process.env.GCS_BUCKET_NAME) {
- return null;
- }
+ try {
+ if (!process.env.GCS_BUCKET_NAME) {
+ return null;
+ }
+
+ const cacheKey = createPdfCacheKey(pdfContent);
+ const storage = new Storage({ credentials });
+ const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
+ const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
+
+ const [exists] = await blob.exists();
+ if (!exists) {
+ logger.debug(`PDF RunPod result not found in GCS cache`, {
+ cacheKey,
+ });
+ return null;
+ }
+
+ const [content] = await blob.download();
+ const result = JSON.parse(content.toString());
- const cacheKey = createPdfCacheKey(pdfContent);
- const storage = new Storage({ credentials });
- const bucket = storage.bucket(process.env.GCS_BUCKET_NAME);
- const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
-
- const [exists] = await blob.exists();
- if (!exists) {
- logger.debug(`PDF RunPod result not found in GCS cache`, {
- cacheKey,
- });
- return null;
+ logger.info(`Retrieved PDF RunPod result from GCS cache`, {
+ cacheKey,
+ });
+
+ return {
+ ...result,
+ };
+ } catch (error) {
+ logger.error(`Error retrieving PDF RunPod result from GCS cache`, {
+ error,
+ });
+ return null;
}
-
- const [content] = await blob.download();
- const result = JSON.parse(content.toString());
-
- logger.info(`Retrieved PDF RunPod result from GCS cache`, {
- cacheKey,
- });
-
- return {
- ...result,
- };
- } catch (error) {
- logger.error(`Error retrieving PDF RunPod result from GCS cache`, {
- error,
- });
- return null;
- }
-}
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 91a54b315..90fd9afe5 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -349,13 +349,17 @@ export class WebCrawler {
setTimeout(() => reject(new Error("Sitemap fetch timeout")), timeout);
});
+ // Allow sitemaps to be cached for 48 hours if they are requested from /map
+ // - mogery
+ const maxAge = (fromMap && !onlySitemap) ? 48 * 60 * 60 * 1000 : 0;
+
try {
let count = (await Promise.race([
Promise.all([
- this.tryFetchSitemapLinks(this.initialUrl, _urlsHandler, abort, mock),
+ this.tryFetchSitemapLinks(this.initialUrl, _urlsHandler, abort, mock, maxAge),
...this.robots
.getSitemaps()
- .map((x) => this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock)),
+ .map((x) => this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock, maxAge)),
]).then((results) => results.reduce((a, x) => a + x, 0)),
timeoutPromise,
])) as number;
@@ -669,6 +673,7 @@ export class WebCrawler {
urlsHandler: (urls: string[]) => unknown,
abort?: AbortSignal,
mock?: string,
+ maxAge?: number,
): Promise<number> {
const sitemapUrl = url.endsWith(".xml")
? url
@@ -679,7 +684,7 @@ export class WebCrawler {
// Try to get sitemap from the provided URL first
try {
sitemapCount = await getLinksFromSitemap(
- { sitemapUrl, urlsHandler, mode: "fire-engine" },
+ { sitemapUrl, urlsHandler, mode: "fire-engine", maxAge },
this.logger,
this.jobId,
this.sitemapsHit,
@@ -727,6 +732,7 @@ export class WebCrawler {
);
},
mode: "fire-engine",
+ maxAge,
},
this.logger,
this.jobId,
@@ -762,7 +768,7 @@ export class WebCrawler {
const baseUrlSitemap = `${this.baseUrl}/sitemap.xml`;
try {
sitemapCount += await getLinksFromSitemap(
- { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },
+ { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge },
this.logger,
this.jobId,
this.sitemapsHit,
@@ -782,7 +788,7 @@ export class WebCrawler {
// ignore 404
} else {
sitemapCount += await getLinksFromSitemap(
- { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },
+ { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge },
this.logger,
this.jobId,
this.sitemapsHit,
diff --git a/apps/api/src/scraper/WebScraper/sitemap-index.ts b/apps/api/src/scraper/WebScraper/sitemap-index.ts
deleted file mode 100644
index f8826bf98..000000000
--- a/apps/api/src/scraper/WebScraper/sitemap-index.ts
+++ /dev/null
@@ -1,62 +0,0 @@
-import { logger } from "../../lib/logger";
-import {
- normalizeUrl,
- normalizeUrlOnlyHostname,
-} from "../../lib/canonical-url";
-import { supabase_service } from "../../services/supabase";
-
-/**
- * Query the sitemap index for a given URL
- * @param url The URL to query
- * @returns A list of URLs found in the sitemap index aggregated from all sitemaps
- */
-import { withAuth } from "../../lib/withAuth";
-
-async function querySitemapIndexFunction(url: string, abort?: AbortSignal) {
- const originUrl = normalizeUrlOnlyHostname(url);
-
- for (let attempt = 1; attempt <= 3; attempt++) {
- abort?.throwIfAborted();
- try {
- const { data, error } = await supabase_service
- .from("crawl_maps")
- .select("urls, updated_at")
- .eq("origin_url", originUrl)
- .order("updated_at", { ascending: false });
-
- if (error) {
- throw error;
- }
-
- if (!data || data.length === 0) {
- return { urls: [], lastUpdated: new Date(0) };
- }
-
- const allUrls = [
- ...new Set(
- data
- .map((entry) => entry.urls)
- .flat()
- .map((url) => normalizeUrl(url)),
- ),
- ];
- return { urls: allUrls, lastUpdated: data[0].updated_at };
- } catch (error) {
- logger.error("(sitemap-index) Error querying the index", {
- error,
- attempt,
- });
-
- if (attempt === 3) {
- return { urls: [], lastUpdated: new Date(0) };
- }
- }
- }
-
- return { urls: [], lastUpdated: new Date(0) };
-}
-
-export const querySitemapIndex = withAuth(querySitemapIndexFunction, {
- urls: [],
- lastUpdated: new Date(0),
-});
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 67d3a2ff3..a89c1e0fc 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -12,10 +12,12 @@ export async function getLinksFromSitemap(
sitemapUrl,
urlsHandler,
mode = "axios",
+ maxAge = 0,
}: {
sitemapUrl: string;
urlsHandler(urls: string[]): unknown;
mode?: "axios" | "fire-engine";
+ maxAge?: number;
},
logger: Logger,
crawlId: string,
@@ -40,9 +42,10 @@ export async function getLinksFromSitemap(
const response = await scrapeURL(
"sitemap;" + crawlId,
sitemapUrl,
- scrapeOptions.parse({ formats: ["rawHtml"], useMock: mock }),
+ scrapeOptions.parse({ formats: ["rawHtml"], useMock: mock, maxAge }),
{
forceEngine: [
+ ...(maxAge > 0 ? ["index" as const] : []),
"fetch",
...((mode === "fire-engine" && useFireEngine) ? ["fire-engine;tlsclient" as const] : []),
],
diff --git a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts b/apps/api/src/scraper/scrapeURL/engines/cache/index.ts
deleted file mode 100644
index 49acbcd7b..000000000
--- a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-import { cacheKey, getEntryFromCache } from "../../../../lib/cache";
-import { EngineScrapeResult } from "..";
-import { Meta } from "../..";
-import { EngineError, IndexMissError } from "../../error";
-
-export async function scrapeCache(meta: Meta): Promise<EngineScrapeResult> {
- const key = cacheKey(meta.rewrittenUrl ?? meta.url, meta.options, meta.internalOptions);
- if (key === null) throw new EngineError("Scrape not eligible for caching");
-
- const entry = await getEntryFromCache(key);
- if (entry === null) throw new IndexMissError();
-
- if (!entry.html) {
- throw new EngineError("Cache hit but HTML is missing");
- }
- if (entry.html.length < 100) {
- throw new EngineError("Cache hit but HTML is too short to be useful");
- }
-
- // Set fromCache flag to indicate this document was retrieved from cache
- meta.internalOptions.fromCache = true;
-
- return {
- url: entry.url,
- html: entry.html,
- statusCode: entry.statusCode,
- error: entry.error,
- };
-}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index a2306f289..713e1959c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -9,7 +9,6 @@ import {
import { scrapePDF } from "./pdf";
import { scrapeURLWithFetch } from "./fetch";
import { scrapeURLWithPlaywright } from "./playwright";
-import { scrapeCache } from "./cache";
import { scrapeURLWithIndex } from "./index/index";
import { useIndex } from "../../../services";
@@ -26,7 +25,6 @@ export type Engine =
| "fetch"
| "pdf"
| "docx"
- | "cache"
| "index"
| "index;documents";
@@ -36,12 +34,8 @@ const useFireEngine =
const usePlaywright =
process.env.PLAYWRIGHT_MICROSERVICE_URL !== "" &&
process.env.PLAYWRIGHT_MICROSERVICE_URL !== undefined;
-const useCache =
- process.env.CACHE_REDIS_URL !== "" &&
- process.env.CACHE_REDIS_URL !== undefined;
export const engines: Engine[] = [
- ...(useCache ? ["cache" as const] : []),
...(useIndex ? ["index" as const, "index;documents" as const] : []),
...(useFireEngine
? [
@@ -132,7 +126,6 @@ const engineHandlers: {
timeToRun: number | undefined,
) => Promise<EngineScrapeResult>;
} = {
- cache: scrapeCache,
index: scrapeURLWithIndex,
"index;documents": scrapeURLWithIndex,
"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,
@@ -159,24 +152,6 @@ export const engineOptions: {
quality: number;
};
} = {
- cache: {
- features: {
- actions: false,
- waitFor: true,
- screenshot: false,
- "screenshot@fullScreen": false,
- pdf: false, // TODO: figure this out
- docx: false, // TODO: figure this out
- atsv: false,
- location: false,
- mobile: false,
- skipTlsVerification: false,
- useFastMode: false,
- stealthProxy: false,
- disableAdblock: false,
- },
- quality: 1000, // cache should always be tried first
- },
index: {
features: {
actions: false,
@@ -193,7 +168,7 @@ export const engineOptions: {
stealthProxy: false,
disableAdblock: true,
},
- quality: 999, // index should always be tried second ? - MG
+ quality: 1000, // index should always be tried first
},
"fire-engine;chrome-cdp": {
features: {
@@ -442,13 +417,6 @@ export function buildFallbackList(meta: Meta): {
...((!useFireEngine && meta.mock !== null) ? ["fire-engine;chrome-cdp", "fire-engine(retry);chrome-cdp", "fire-engine;chrome-cdp;stealth", "fire-engine(retry);chrome-cdp;stealth", "fire-engine;playwright", "fire-engine;tlsclient", "fire-engine;playwright;stealth", "fire-engine;tlsclient;stealth"] as Engine[] : [])
];
- if (meta.internalOptions.useCache !== true) {
- const cacheIndex = _engines.indexOf("cache");
- if (cacheIndex !== -1) {
- _engines.splice(cacheIndex, 1);
- }
- }
-
const shouldUseIndex =
useIndex
&& process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true"
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index bf6d1fdb0..fb0c765de 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -7,7 +7,6 @@ import crypto from "crypto";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
const shouldCache = meta.options.storeInCache
- && meta.winnerEngine !== "cache"
&& meta.winnerEngine !== "index"
&& meta.winnerEngine !== "index;documents"
&& (
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 3dd97995e..8bc968130 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -16,11 +16,8 @@ import {
import { readFile, unlink } from "node:fs/promises";
import path from "node:path";
import type { Response } from "undici";
-import {
- getPdfResultFromCache,
- savePdfResultToCache,
-} from "../../../../lib/gcs-pdf-cache";
import { getPageCount } from "../../../../lib/pdf-parser";
+import { getPdfResultFromCache, savePdfResultToCache } from "../../../../lib/gcs-pdf-cache";
type PDFProcessorResult = { html: string; markdown?: string };
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 73f9ebb2c..a2a822d6f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -192,10 +192,8 @@ export type InternalOptions = {
v0CrawlOnlyUrls?: boolean;
v0DisableJsDom?: boolean;
- useCache?: boolean;
disableSmartWaitCache?: boolean; // Passed along to fire-engine
isBackgroundIndex?: boolean;
- fromCache?: boolean; // Indicates if the document was retrieved from cache
abort?: AbortSignal;
urlInvisibleInCurrentCrawl?: boolean;
unnormalizedSourceURL?: string;
@@ -410,7 +408,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
numPages: result.result.numPages,
contentType: result.result.contentType,
proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",
- ...(results["index"] ? (
+ ...((results["index"] || results["index;documents"]) ? (
result.result.cacheInfo ? {
cacheState: "hit",
cachedAt: result.result.cacheInfo.created_at.toISOString(),
diff --git a/apps/api/src/scraper/scrapeURL/transformers/cache.ts b/apps/api/src/scraper/scrapeURL/transformers/cache.ts
deleted file mode 100644
index 96c3d4378..000000000
--- a/apps/api/src/scraper/scrapeURL/transformers/cache.ts
+++ /dev/null
@@ -1,41 +0,0 @@
-import { Document } from "../../../controllers/v1/types";
-import { Meta } from "..";
-import { CacheEntry, cacheKey, saveEntryToCache } from "../../../lib/cache";
-
-export function saveToCache(meta: Meta, document: Document): Document {
- if (meta.internalOptions.useCache !== true) {
- return document;
- }
-
- if (
- document.metadata.statusCode! < 200 ||
- document.metadata.statusCode! >= 300
- )
- return document;
-
- if (document.rawHtml === undefined) {
- throw new Error(
- "rawHtml is undefined -- this transformer is being called out of order",
- );
- }
-
- // If the document was retrieved from cache, we don't need to save it
- if (meta.internalOptions.fromCache) {
- return document;
- }
-
- const key = cacheKey(meta.url, meta.options, meta.internalOptions);
-
- if (key !== null) {
- const entry: CacheEntry = {
- html: document.rawHtml!,
- statusCode: document.metadata.statusCode!,
- url: document.metadata.url ?? document.metadata.sourceURL!,
- error: document.metadata.error ?? undefined,
- };
-
- saveEntryToCache(key, entry);
- }
-
- return document;
-}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index f229793b3..a9e2742c0 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -7,7 +7,6 @@ import { extractMetadata } from "../lib/extractMetadata";
import { performLLMExtract } from "./llmExtract";
import { uploadScreenshot } from "./uploadScreenshot";
import { removeBase64Images } from "./removeBase64Images";
-import { saveToCache } from "./cache";
import { performAgent } from "./agent";
import { deriveDiff } from "./diff";
@@ -202,7 +201,6 @@ export function coerceFieldsToFormats(
// TODO: allow some of these to run in parallel
export const transformerStack: Transformer[] = [
- saveToCache,
deriveHTMLFromRawHTML,
deriveMarkdownFromHTML,
deriveLinksFromHTML,
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 08a901775..882fed151 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -237,7 +237,7 @@ export async function getIndexInsertQueueLength(): Promise<number> {
return await redisEvictConnection.llen(INDEX_INSERT_QUEUE_KEY) ?? 0;
}
-export async function queryIndexAtSplitLevel(url: string, limit: number): Promise<string[]> {
+export async function queryIndexAtSplitLevel(url: string, limit: number, maxAge = 2 * 24 * 60 * 60 * 1000): Promise<string[]> {
if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
return [];
}
@@ -258,7 +258,7 @@ export async function queryIndexAtSplitLevel(url: string, limit: number): Promis
.rpc("query_index_at_split_level", {
i_level: level,
i_url_hash: urlSplitsHash[level],
- i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
+ i_newer_than: new Date(Date.now() - maxAge).toISOString(),
})
.range(iteration * 1000, (iteration + 1) * 1000)
@@ -286,7 +286,7 @@ export async function queryIndexAtSplitLevel(url: string, limit: number): Promis
}
}
-export async function queryIndexAtDomainSplitLevel(hostname: string, limit: number): Promise<string[]> {
+export async function queryIndexAtDomainSplitLevel(hostname: string, limit: number, maxAge = 2 * 24 * 60 * 60 * 1000): Promise<string[]> {
if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
return [];
}
@@ -307,7 +307,7 @@ export async function queryIndexAtDomainSplitLevel(hostname: string, limit: numb
.rpc("query_index_at_domain_split_level", {
i_level: level,
i_domain_hash: domainSplitsHash[level],
- i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
+ i_newer_than: new Date(Date.now() - maxAge).toISOString(),
})
.range(iteration * 1000, (iteration + 1) * 1000)
diff --git a/apps/api/src/services/indexing/crawl-maps-index.ts b/apps/api/src/services/indexing/crawl-maps-index.ts
deleted file mode 100644
index 957f3effe..000000000
--- a/apps/api/src/services/indexing/crawl-maps-index.ts
+++ /dev/null
@@ -1,278 +0,0 @@
-import { logger } from "../../lib/logger";
-import {
- normalizeUrl,
- normalizeUrlOnlyHostname,
-} from "../../lib/canonical-url";
-import { supabase_service } from "../supabase";
-import { redisConnection } from "../queue-service";
-
-const BATCH_KEY = "crawl_maps_batch";
-const BATCH_LOCK_KEY = "crawl_maps_batch_lock";
-const BATCH_SIZE = 20;
-const BATCH_TIMEOUT = 20000; // 10 seconds
-const LOCK_TIMEOUT = 30000; // 30 seconds
-
-interface CrawlMapOperation {
- originUrl: string;
- standardizedUrls: string[];
- timestamp: string;
-}
-
-interface CrawlMapRecord {
- id?: string;
- origin_url: string;
- urls: string[];
- num_urls: number;
- updated_at: string;
- created_at?: string;
-}
-
-async function acquireLock(): Promise<boolean> {
- const redis = redisConnection;
- // Set lock with NX (only if it doesn't exist) and PX (millisecond expiry)
- const result = await redis.set(BATCH_LOCK_KEY, "1", "PX", LOCK_TIMEOUT, "NX");
- const acquired = result === "OK";
- if (acquired) {
- logger.info("🔒 Acquired batch processing lock");
- }
- return acquired;
-}
-
-async function releaseLock() {
- const redis = redisConnection;
- await redis.del(BATCH_LOCK_KEY);
- logger.info("🔓 Released batch processing lock");
-}
-
-async function processBatch() {
- const redis = redisConnection;
-
- // Try to acquire lock
- if (!(await acquireLock())) {
- return;
- }
-
- try {
- // Get all operations from Redis list
- const operations: CrawlMapOperation[] = [];
- while (operations.length < BATCH_SIZE) {
- const op = await redis.lpop(BATCH_KEY);
- if (!op) break;
- operations.push(JSON.parse(op));
- }
-
- if (operations.length === 0) {
- logger.info("No operations to process in batch");
- return;
- }
-
- logger.info(`📦 Processing batch of ${operations.length} operations`, {
- origins: operations.map((op) => op.originUrl),
- });
-
- // Get existing maps for all origins in batch
- const origins = operations.map((op) => op.originUrl);
- const { data: existingMaps } = await supabase_service
- .from("crawl_maps")
- .select("id, origin_url, urls, updated_at")
- .in("origin_url", origins)
- .order("updated_at", { ascending: false });
-
- // Group maps by origin and handle duplicates
- const mapsByOrigin = new Map<string, any[]>();
- existingMaps?.forEach((map) => {
- const maps = mapsByOrigin.get(map.origin_url) || [];
- maps.push(map);
- mapsByOrigin.set(map.origin_url, maps);
- });
-
- // Handle duplicates and prepare updates
- const updates: CrawlMapRecord[] = [];
- const inserts: CrawlMapRecord[] = [];
- const duplicatesToDelete: string[] = [];
-
- // Track processed origins to avoid duplicates in the same batch
- const processedOrigins = new Set<string>();
-
- for (const op of operations) {
- // Skip if we've already processed this origin in this batch
- if (processedOrigins.has(op.originUrl)) {
- continue;
- }
- processedOrigins.add(op.originUrl);
-
- const existingForOrigin = mapsByOrigin.get(op.originUrl) || [];
-
- if (existingForOrigin.length > 0) {
- // Keep most recent entry and mark others for deletion
- const [mostRecent, ...duplicates] = existingForOrigin;
- if (duplicates.length > 0) {
- duplicatesToDelete.push(...duplicates.map((d) => d.id));
- }
-
- // Merge and deduplicate URLs
- const mergedUrls = [
- ...new Set([
- ...mostRecent.urls,
- ...op.standardizedUrls.map((url) => normalizeUrl(url)),
- ]),
- ];
-
- updates.push({
- id: mostRecent.id,
- origin_url: op.originUrl,
- urls: mergedUrls,
- num_urls: mergedUrls.length,
- updated_at: op.timestamp,
- });
- } else {
- // Prepare insert with deduplicated URLs
- const deduplicatedUrls = [
- ...new Set(op.standardizedUrls.map((url) => normalizeUrl(url))),
- ];
- inserts.push({
- origin_url: op.originUrl,
- urls: deduplicatedUrls,
- num_urls: deduplicatedUrls.length,
- created_at: op.timestamp,
- updated_at: op.timestamp,
- });
- }
- }
-
- // Delete duplicate entries
- if (duplicatesToDelete.length > 0) {
- logger.info(
- `🗑️ Deleting ${duplicatesToDelete.length} duplicate crawl maps in batches of 100`,
- );
-
- // Delete in batches of 100
- for (let i = 0; i < duplicatesToDelete.length; i += 100) {
- const batch = duplicatesToDelete.slice(i, i + 100);
- const { error: deleteError } = await supabase_service
- .from("crawl_maps")
- .delete()
- .in("id", batch);
-
- if (deleteError) {
- logger.error(
- `Failed to delete batch ${i / 100 + 1} of duplicate crawl maps`,
- {
- error: deleteError,
- batchSize: batch.length,
- startIndex: i,
- },
- );
- }
- }
- }
-
- // Execute batch operations
- if (updates.length > 0) {
- logger.info(`🔄 Updating ${updates.length} existing crawl maps`, {
- origins: updates.map((u) => u.origin_url),
- });
-
- // Process updates one at a time to avoid conflicts
- for (const update of updates) {
- const { error: updateError } = await supabase_service
- .from("crawl_maps")
- .upsert(update);
-
- if (updateError) {
- logger.error("Failed to update crawl map", {
- error: updateError,
- origin: update.origin_url,
- });
- }
- }
- }
-
- if (inserts.length > 0) {
- logger.info(`➕ Inserting ${inserts.length} new crawl maps`, {
- origins: inserts.map((i) => i.origin_url),
- });
- const { error: insertError } = await supabase_service
- .from("crawl_maps")
- .insert(inserts);
-
- if (insertError) {
- logger.error("Failed to batch insert crawl maps", {
- error: insertError,
- });
- }
- }
-
- logger.info("✅ Batch processing completed successfully");
- } catch (error) {
- logger.error("Error processing crawl map batch", { error });
- } finally {
- await releaseLock();
- }
-}
-
-// Start periodic batch processing
-let batchInterval: NodeJS.Timeout | null = null;
-
-function startBatchProcessing() {
- if (batchInterval) return;
-
- logger.info("🔄 Starting periodic batch processing");
- batchInterval = setInterval(async () => {
- const queueLength = await redisConnection.llen(BATCH_KEY);
- logger.info(`Checking batch queue (${queueLength} items pending)`);
- await processBatch();
- }, BATCH_TIMEOUT);
-
- // Unref to not keep process alive
- batchInterval.unref();
-}
-
-export async function saveCrawlMap(originUrl: string, visitedUrls: string[]) {
- logger.info("Queueing crawl map", { originUrl });
- originUrl = normalizeUrlOnlyHostname(originUrl);
-
- try {
- // Standardize URLs to canonical form
- const standardizedUrls = [
- ...new Set(visitedUrls.map((url) => normalizeUrl(url))),
- ];
-
- const operation: CrawlMapOperation = {
- originUrl,
- standardizedUrls,
- timestamp: new Date().toISOString(),
- };
-
- // Add operation to Redis list
- const redis = redisConnection;
- await redis.rpush(BATCH_KEY, JSON.stringify(operation));
- const queueLength = await redis.llen(BATCH_KEY);
- logger.info(`📥 Added operation to queue (${queueLength} total pending)`, {
- originUrl,
- });
-
- // Start batch processing if not already started
- startBatchProcessing();
-
- // If we have enough items, trigger immediate processing
- if (queueLength >= BATCH_SIZE) {
- logger.info(
- "🔄 Queue reached batch size, triggering immediate processing",
- );
- await processBatch();
- }
- } catch (error) {
- logger.error("Error queueing crawl map", { error });
- }
-}
-
-// Cleanup on exit
-process.on("beforeExit", async () => {
- if (batchInterval) {
- clearInterval(batchInterval);
- batchInterval = null;
- logger.info("Stopped periodic batch processing");
- }
- await processBatch();
-});
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 06ad21f5d..7ff0b9f4c 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -5,12 +5,8 @@ import { Job, Queue, Worker } from "bullmq";
import { logger as _logger, logger } from "../../lib/logger";
import {
redisConnection,
- indexQueueName,
- getIndexQueue,
- billingQueueName,
getBillingQueue,
} from "../queue-service";
-import { saveCrawlMap } from "./crawl-maps-index";
import { processBillingBatch, queueBillingOperation, startBillingBatchProcessing } from "../billing/batch_billing";
import systemMonitor from "../system-monitor";
import { v4 as uuidv4 } from "uuid";
@@ -33,39 +29,6 @@ const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;
const runningJobs: Set<string> = new Set();
-const processJobInternal = async (token: string, job: Job) => {
- if (!job.id) {
- throw new Error("Job has no ID");
- }
-
- const logger = _logger.child({
- module: "index-worker",
- method: "processJobInternal",
- jobId: job.id,
- });
-
- const extendLockInterval = setInterval(async () => {
- logger.info(`🔄 Worker extending lock on job ${job.id}`);
- await job.extendLock(token, jobLockExtensionTime);
- }, jobLockExtendInterval);
-
- let err = null;
- try {
- const { originUrl, visitedUrls } = job.data;
- await saveCrawlMap(originUrl, visitedUrls);
- await job.moveToCompleted({ success: true }, token, false);
- } catch (error) {
- logger.error("Error processing index job", { error });
- Sentry.captureException(error);
- err = error;
- await job.moveToFailed(error, token, false);
- } finally {
- clearInterval(extendLockInterval);
- }
-
- return err;
-};
-
// Create a processor for billing jobs
const processBillingJobInternal = async (token: string, job: Job) => {
if (!job.id) {
@@ -233,9 +196,6 @@ const WEBHOOK_INSERT_INTERVAL = 15000;
// Start the workers
(async () => {
- // Start index worker
- const indexWorkerPromise = workerFun(getIndexQueue(), processJobInternal);
-
// Start billing worker and batch processing
startBillingBatchProcessing();
const billingWorkerPromise = workerFun(getBillingQueue(), processBillingJobInternal);
@@ -255,8 +215,8 @@ const WEBHOOK_INSERT_INTERVAL = 15000;
await processWebhookInsertJobs();
}, WEBHOOK_INSERT_INTERVAL);
- // Wait for both workers to complete (which should only happen on shutdown)
- await Promise.all([indexWorkerPromise, billingWorkerPromise]);
+ // Wait for all workers to complete (which should only happen on shutdown)
+ await Promise.all([billingWorkerPromise]);
clearInterval(indexInserterInterval);
clearInterval(webhookInserterInterval);
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index b4938dce9..77f62c73a 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -60,24 +60,6 @@ export function getExtractQueue() {
return extractQueue;
}
-export function getIndexQueue() {
- if (!indexQueue) {
- indexQueue = new Queue(indexQueueName, {
- connection: redisConnection,
- defaultJobOptions: {
- removeOnComplete: {
- age: 3600, // 1 hour
- },
- removeOnFail: {
- age: 3600, // 1 hour
- },
- },
- });
- logger.info("Index queue created");
- }
- return indexQueue;
-}
-
export function getGenerateLlmsTxtQueue() {
if (!generateLlmsTxtQueue) {
generateLlmsTxtQueue = new Queue(generateLlmsTxtQueueName, {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 2d56cc45f..54a9e6ff4 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -6,7 +6,6 @@ import {
getScrapeQueue,
getExtractQueue,
getDeepResearchQueue,
- getIndexQueue,
redisConnection,
getGenerateLlmsTxtQueue,
getBillingQueue,
@@ -64,7 +63,6 @@ import { supabase_service } from "../services/supabase";
import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";
import { saveExtract, updateExtract } from "../lib/extract/extract-redis";
import { billTeam } from "./billing/credit_billing";
-import { saveCrawlMap } from "./indexing/crawl-maps-index";
import { updateDeepResearch } from "../lib/deep-research/deep-research-redis";
import { performDeepResearch } from "../lib/deep-research/deep-research-service";
import { performGenerateLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-service";
@@ -253,35 +251,6 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
logger.info("Finishing crawl");
await finishCrawl(job.data.crawl_id);
- (async () => {
- const originUrl = sc.originUrl
- ? normalizeUrlOnlyHostname(sc.originUrl)
- : undefined;
- // Get all visited unique URLs from Redis
- const visitedUrls = await redisEvictConnection.smembers(
- "crawl:" + job.data.crawl_id + ":visited_unique",
- );
- // Upload to Supabase if we have URLs and this is a crawl (not a batch scrape)
- if (
- visitedUrls.length > 0 &&
- job.data.crawlerOptions !== null &&
- originUrl &&
- process.env.USE_DB_AUTHENTICATION === "true"
- ) {
- // Queue the indexing job instead of doing it directly
- await getIndexQueue().add(
- job.data.crawl_id,
- {
- originUrl,
- visitedUrls,
- },
- {
- priority: 10,
- },
- );
- }
- })();
-
if (!job.data.v1) {
const jobIDs = await getCrawlJobs(job.data.crawl_id);