Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix map and some tests
Gergő Móriczfa36ad664254
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.tsindex 304c9dd39..83e71366d 100644--- a/apps/api/src/__tests__/snips/v2/map.test.ts+++ b/apps/api/src/__tests__/snips/v2/map.test.ts@@ -39,6 +39,6 @@ describe("Map tests", () => {expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);- expect(response.body.links.some(x => x.match(/^https:\/\/www\.hfea\.gov\.uk\/choose-a-clinic\/clinic-search\/results\/?\?options=\d+$/))).toBe(true);+ expect(response.body.links.some(x => x.url.match(/^https:\/\/www\.hfea\.gov\.uk\/choose-a-clinic\/clinic-search\/results\/?\?options=\d+$/))).toBe(true);}, 60000);});diff --git a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.tsindex 339a6f9d1..658e5dcb4 100644--- a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts@@ -15,6 +15,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {const data = await scrape({url: "https://expired.badssl.com/",timeout: scrapeTimeout,+ maxAge: 0,}, identity);expect(data).toBeDefined();@@ -26,6 +27,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {url: "https://expired.badssl.com/",skipTlsVerification: false,timeout: scrapeTimeout,+ maxAge: 0,}, identity);if (response.status !== 500) {@@ -40,6 +42,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {const data = await scrape({url: "https://firecrawl.dev",timeout: scrapeTimeout,+ maxAge: 0,}, identity);expect(data).toBeDefined();@@ -52,6 +55,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {url: "https://firecrawl.dev",formats: [{ type: "screenshot", fullPage: false }],timeout: scrapeTimeout,+ maxAge: 0,}, identity);expect(data).toBeDefined();@@ -64,6 +68,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {url: "https://firecrawl.dev",formats: [{ type: "screenshot", fullPage: true }],timeout: scrapeTimeout,+ maxAge: 0,}, identity);expect(data).toBeDefined();diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex c4de182cc..8be52acc9 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -1,14 +1,14 @@import { Response } from "express";import { v4 as uuidv4 } from "uuid";import {+ mapRequestSchema,RequestWithAuth,+ scrapeOptions,TeamFlags,TimeoutSignal,- scrapeOptions,- mapRequestSchema,MapRequest,- MapV2Response,MapDocument,+ MapResponse,} from "./types";import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";import { configDotenv } from "dotenv";@@ -16,16 +16,13 @@ import {checkAndUpdateURLForMap,isSameDomain,isSameSubdomain,- removeDuplicateUrls,} from "../../lib/validateUrl";import { fireEngineMap } from "../../search/fireEngine";import { billTeam } from "../../services/billing/credit_billing";import { logJob } from "../../services/logging/log_job";-import { performCosineSimilarity } from "../../lib/map-cosine";import { logger } from "../../lib/logger";import Redis from "ioredis";-import { generateURLSplits, queryIndexAtDomainSplitLevel, queryIndexAtSplitLevel, queryIndexDocument } from "../../services/index";-import { WebSearchResult } from "../../lib/entities";+import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../../services/index";configDotenv();const redis = new Redis(process.env.REDIS_URL!);@@ -35,18 +32,14 @@ const MAX_MAP_LIMIT = 30000;// Max Links that "Smart /map" can returnconst MAX_FIRE_ENGINE_RESULTS = 500;-// MapRequest and MapV2Response are imported from ./types-interface MapResult {success: boolean;- links: string[];- scrape_id?: string;job_id: string;time_taken: number;- mapResults: WebSearchResult[];+ mapResults: MapDocument[];}-async function queryIndex(url: string, limit: number, useIndex: boolean, includeSubdomains: boolean): Promise<string[]> {+async function queryIndex(url: string, limit: number, useIndex: boolean): Promise<MapDocument[]> {if (!useIndex) {return [];}@@ -56,109 +49,49 @@ async function queryIndex(url: string, limit: number, useIndex: boolean, includeconst urlObj = new URL(url);const hostname = urlObj.hostname;- // TEMP: this should be altered on June 15th 2025 7AM PT - mogery- const domainLinks = includeSubdomains ? await queryIndexAtDomainSplitLevel(hostname, limit, 14 * 24 * 60 * 60 * 1000) : [];- const splitLinks = await queryIndexAtSplitLevel(url, limit, 14 * 24 * 60 * 60 * 1000);-- return Array.from(new Set([...domainLinks, ...splitLinks]));+ return (await queryIndexAtDomainSplitLevelWithMeta(hostname, limit));} else {- return await queryIndexAtSplitLevel(url, limit);+ return (await queryIndexAtSplitLevelWithMeta(url, limit));}}-async function enrichUrlsWithMetadata(urls: string[], useIndex: boolean, teamId: string, flags: TeamFlags): Promise<WebSearchResult[]> {- if (!useIndex || urls.length === 0) {- return urls.map(url => ({- url,- title: "",- description: "",- }));- }-- // Query metadata for all URLs in parallel- const metadataPromises = urls.map(async (url) => {- try {- const indexDocument = await queryIndexDocument(url);- if (indexDocument) {- return {- url,- title: indexDocument.title || "",- description: indexDocument.description || "",- metadata: {- title: indexDocument.title,- description: indexDocument.description,- language: indexDocument.language,- keywords: indexDocument.keywords,- robots: indexDocument.robots,- ogTitle: indexDocument.ogTitle,- ogDescription: indexDocument.ogDescription,- ogUrl: indexDocument.ogUrl,- ogImage: indexDocument.ogImage,- ogLocale: indexDocument.ogLocale,- ogSiteName: indexDocument.ogSiteName,- createdAt: indexDocument.createdAt,- lastScrapedAt: indexDocument.lastScrapedAt,- }- };- }- } catch (error) {- logger.warn(`Failed to fetch metadata for URL: ${url}`, { error });- }-- // Fallback if no metadata found- return {- url,- title: "",- description: "",- };- });-- return await Promise.all(metadataPromises);-}--export async function getMapV2Results({+export async function getMapResults({url,search,limit = MAX_MAP_LIMIT,- sitemap = "include",includeSubdomains = true,+ crawlerOptions = {},teamId,- origin,allowExternalLinks,- abort = new AbortController().signal,- mock,+ abort = new AbortController().signal, // noopfilterByPath = true,flags,useIndex = true,- includeMetadata = true,- timeout,}: {url: string;search?: string;limit?: number;- sitemap?: "only" | "include" | "skip";+ ignoreSitemap?: boolean;includeSubdomains?: boolean;+ crawlerOptions?: any;teamId: string;origin?: string;+ includeMetadata?: boolean;allowExternalLinks?: boolean;abort?: AbortSignal;mock?: string;filterByPath?: boolean;flags: TeamFlags;useIndex?: boolean;- includeMetadata?: boolean;- timeout?: number;}): Promise<MapResult> {const id = uuidv4();- let links: string[] = [url];- let mapResults: WebSearchResult[] = [];-- const zeroDataRetention = flags?.forceZDR ?? false;+ let mapResults: MapDocument[] = [];const sc: StoredCrawl = {originUrl: url,crawlerOptions: {- limit: sitemap === "only" ? 10000000 : limit,+ ...crawlerOptions,+ limit: crawlerOptions.sitemapOnly ? 10000000 : limit,scrapeOptions: undefined,},scrapeOptions: scrapeOptions.parse({}),@@ -174,267 +107,156 @@ export async function getMapV2Results({crawler.importRobotsTxt(sc.robots);} catch (_) {}- // If sitemap is "only", only get links from sitemap- if (sitemap === "only") {- const sitemapCount = await crawler.tryGetSitemap(- (urls) => {- urls.forEach((x) => {- links.push(x);- });- },- true,- true,- timeout ?? 30000,- abort,- mock,- );- if (sitemapCount > 0) {- links = links- .slice(1)- .map((x) => {- try {- return checkAndUpdateURLForMap(x).url.trim();- } catch (_) {- return null;- }- })- .filter((x) => x !== null) as string[];- }- } else if (sitemap !== "skip") {- // sitemap is "include" - fetch both search results and sitemap- let urlWithoutWww = url.replace("www.", "");-- let mapUrl =- search && allowExternalLinks- ? `${search} ${urlWithoutWww}`- : search- ? `${search} site:${urlWithoutWww}`- : `site:${url}`;-- const resultsPerPage = 100;- const maxPages = Math.ceil(- Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,- );+ let urlWithoutWww = url.replace("www.", "");- const cacheKey = `fireEngineMap:${mapUrl}`;- const cachedResult = await redis.get(cacheKey);+ let mapUrl =+ search && allowExternalLinks+ ? `${search} ${urlWithoutWww}`+ : search+ ? `${search} site:${urlWithoutWww}`+ : `site:${url}`;- let allResults: any[] = [];- let pagePromises: Promise<any>[] = [];+ const resultsPerPage = 100;+ const maxPages = Math.ceil(+ Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,+ );- if (cachedResult) {- allResults = JSON.parse(cachedResult);- } else {- const fetchPage = async (page: number) => {- return fireEngineMap(mapUrl, {- numResults: resultsPerPage,- page: page,- }, abort);- };-- pagePromises = Array.from({ length: maxPages }, (_, i) =>- fetchPage(i + 1),- );- allResults = await Promise.all(pagePromises);-- if (!zeroDataRetention) {- await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours- }- }+ const cacheKey = `fireEngineMap:${mapUrl}`;+ const cachedResult = await redis.get(cacheKey);- // Parallelize sitemap index query with search results- const [indexResults, ...searchResults] = await Promise.all([- queryIndex(url, limit, useIndex, includeSubdomains),- ...(cachedResult ? [] : pagePromises),- ]);+ let allResults: any[] = [];+ let pagePromises: Promise<any>[] = [];- if (indexResults.length > 0) {- links.push(...indexResults);- }+ if (cachedResult) {+ allResults = JSON.parse(cachedResult);+ } else {+ const fetchPage = async (page: number) => {+ return fireEngineMap(mapUrl, {+ numResults: resultsPerPage,+ page: page,+ }, abort);+ };- // Fetch sitemap if sitemap is "include"- try {- await crawler.tryGetSitemap(- (urls) => {- links.push(...urls);- },- true,- false,- timeout ?? 30000,- abort,- );- } catch (e) {- logger.warn("tryGetSitemap threw an error", { error: e });- }+ pagePromises = Array.from({ length: maxPages }, (_, i) =>+ fetchPage(i + 1),+ );+ allResults = await Promise.all(pagePromises);- if (!cachedResult) {- allResults = searchResults;- }+ await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours+ }- // Convert map results to WebSearchResult format- const fireEngineResults = allResults- .flat()- .filter((result) => result !== null && result !== undefined)- .map((result) => ({- url: result.url,- title: result.title || "",- description: result.description || "",- }));-- const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);- if (fireEngineResults.length > minumumCutoff) {- mapResults = fireEngineResults.slice(0, minumumCutoff);- } else {- mapResults = fireEngineResults;- }+ // Parallelize sitemap index query with search results+ const [indexResults, ...searchResults] = await Promise.all([+ queryIndex(url, limit, useIndex),+ ...(cachedResult ? [] : pagePromises),+ ]);- if (mapResults.length > 0) {- if (search) {- // Ensure all map results are first, maintaining their order- links = [- mapResults[0].url,- ...mapResults.slice(1).map((x) => x.url),- ...links,- ];- } else {- mapResults.map((x) => {- links.push(x.url);- });- }- }+ if (indexResults.length > 0) {+ mapResults.push(...indexResults);+ }- // Perform cosine similarity between the search query and the list of links- if (search) {- const searchQuery = search.toLowerCase();- links = performCosineSimilarity(links, searchQuery);- }+ console.log("searchResults", searchResults);- links = links- .map((x) => {- try {- return checkAndUpdateURLForMap(x).url.trim();- } catch (_) {- return null;- }- })- .filter((x) => x !== null) as string[];-- // allows for subdomains to be included- links = links.filter((x) => isSameDomain(x, url));-- // if includeSubdomains is false, filter out subdomains- if (!includeSubdomains) {- links = links.filter((x) => isSameSubdomain(x, url));- }+ mapResults = mapResults.concat(searchResults.map(x => ({+ url: x.url,+ title: x.title,+ description: x.description,+ })).flat());++ const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);+ if (mapResults.length > minumumCutoff) {+ mapResults = mapResults.slice(0, minumumCutoff);+ }- // Filter by path if enabled- if (filterByPath && !allowExternalLinks) {+ mapResults = mapResults+ .map((x) => {try {- const urlObj = new URL(url);- const urlPath = urlObj.pathname;- // Only apply path filtering if the URL has a significant path (not just '/' or empty)- // This means we only filter by path if the user has not selected a root domain- if (urlPath && urlPath !== '/' && urlPath.length > 1) {- links = links.filter(link => {- try {- const linkObj = new URL(link);- return linkObj.pathname.startsWith(urlPath);- } catch (e) {- return false;- }- });- }- } catch (e) {- // If URL parsing fails, continue without path filtering- logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });+ return {+ ...x,+ url: checkAndUpdateURLForMap(x.url).url.trim(),+ };+ } catch (_) {+ return null;}- }+ })+ .filter((x) => x !== null) as MapDocument[];- // remove duplicates that could be due to http/https or www- links = removeDuplicateUrls(links);- }+ // allows for subdomains to be included+ mapResults = mapResults.filter((x) => isSameDomain(x.url, url));- const linksToReturn = sitemap === "only"- ? links- : links.slice(0, limit);+ // if includeSubdomains is false, filter out subdomains+ if (!includeSubdomains) {+ mapResults = mapResults.filter((x) => isSameSubdomain(x.url, url));+ }- // Enrich URLs with metadata if requested- let enrichedResults: WebSearchResult[] = [];- if (includeMetadata) {- enrichedResults = await enrichUrlsWithMetadata(linksToReturn, useIndex, teamId, flags);- } else {- enrichedResults = linksToReturn.map(url => ({- url,- title: "",- description: "",- }));+ // Filter by path if enabled+ if (filterByPath && !allowExternalLinks) {+ try {+ const urlObj = new URL(url);+ const urlPath = urlObj.pathname;+ // Only apply path filtering if the URL has a significant path (not just '/' or empty)+ // This means we only filter by path if the user has not selected a root domain+ if (urlPath && urlPath !== '/' && urlPath.length > 1) {+ mapResults = mapResults.filter(x => {+ try {+ const linkObj = new URL(x.url);+ return linkObj.pathname.startsWith(urlPath);+ } catch (e) {+ return false;+ }+ });+ }+ } catch (e) {+ // If URL parsing fails, continue without path filtering+ logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });+ }}return {success: true,- links: linksToReturn,- mapResults: enrichedResults,- scrape_id: origin?.includes("website") ? id : undefined,+ mapResults: mapResults,job_id: id,time_taken: (new Date().getTime() - Date.now()) / 1000,};}-export async function mapV2Controller(- req: RequestWithAuth<{}, MapV2Response, MapRequest>,- res: Response<MapV2Response>,+export async function mapController(+ req: RequestWithAuth<{}, MapResponse, MapRequest>,+ res: Response<MapResponse>,) {const originalRequest = req.body;-- // Parse and validate the request using the v2 schema- const parsedRequest = mapRequestSchema.parse(req.body);-- // Transform ignoreSitemap and sitemapOnly to the new unified sitemap parameter- let sitemap: "only" | "include" | "skip" = "include";- if (parsedRequest.sitemapOnly) {- sitemap = "only";- } else if (parsedRequest.ignoreSitemap) {- sitemap = "skip";- }-- if (req.acuc?.flags?.forceZDR) {- return res.status(400).json({- success: false,- error: "Your team has zero data retention enabled. This is not supported on map. Please contact [email protected] to unblock this feature."- });- }+ req.body = mapRequestSchema.parse(req.body);- logger.info("Map V2 request", {- request: parsedRequest,+ logger.info("Map request", {+ request: req.body,originalRequest,teamId: req.auth.team_id,});- let result: Awaited<ReturnType<typeof getMapV2Results>>;+ let result: Awaited<ReturnType<typeof getMapResults>>;const abort = new AbortController();try {result = await Promise.race([- getMapV2Results({- url: parsedRequest.url,- search: parsedRequest.search,- limit: parsedRequest.limit,- sitemap,- includeSubdomains: parsedRequest.includeSubdomains,- origin: parsedRequest.origin,+ getMapResults({+ url: req.body.url,+ search: req.body.search,+ limit: req.body.limit,+ ignoreSitemap: req.body.ignoreSitemap,+ includeSubdomains: req.body.includeSubdomains,+ crawlerOptions: req.body,+ origin: req.body.origin,teamId: req.auth.team_id,abort: abort.signal,- mock: parsedRequest.useMock,- filterByPath: parsedRequest.filterByPath,+ mock: req.body.useMock,+ filterByPath: req.body.filterByPath !== false,flags: req.acuc?.flags ?? null,- useIndex: parsedRequest.useIndex,- includeMetadata: true, // Always include metadata in v2- timeout: parsedRequest.timeout,+ useIndex: req.body.useIndex,}),- ...(parsedRequest.timeout !== undefined ? [+ ...(req.body.timeout !== undefined ? [new Promise((resolve, reject) => setTimeout(() => {abort.abort(new TimeoutSignal());reject(new TimeoutSignal());- }, parsedRequest.timeout!))+ }, req.body.timeout))] : []),]) as any;} catch (error) {@@ -458,33 +280,27 @@ export async function mapV2Controller(// Log the joblogJob({job_id: result.job_id,- success: result.links.length > 0,- message: "Map V2 completed",- num_docs: result.links.length,- docs: result.links,+ success: result.mapResults.length > 0,+ message: "Map completed",+ num_docs: result.mapResults.length,+ docs: result.mapResults,time_taken: result.time_taken,team_id: req.auth.team_id,mode: "map",- url: parsedRequest.url,+ url: req.body.url,crawlerOptions: {},scrapeOptions: {},- origin: parsedRequest.origin ?? "api",- integration: parsedRequest.integration,+ origin: req.body.origin ?? "api",+ integration: req.body.integration,num_tokens: 0,credits_billed: 1,zeroDataRetention: false, // not supported});- const response: MapV2Response = {+ const response = {success: true as const,- links: result.links, // For backwards compatibility- web: result.mapResults,- metadata: {- totalCount: result.links.length,- hasMore: result.links.length >= parsedRequest.limit,- searchQuery: parsedRequest.search,- },+ links: result.mapResults,};return res.status(200).json(response);-}\ No newline at end of file+}\ No newline at end of filediff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 2136b8c20..47b2c0aa1 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -804,14 +804,6 @@ export type BatchScrapeResponse =invalidURLs?: string[];};-export type MapResponse =- | ErrorResponse- | {- success: true;- links: string[];- scrape_id?: string;- };-// Map document interface (transitioned from v1)export interface MapDocument {url: string;@@ -820,12 +812,11 @@ export interface MapDocument {}// V2 Map Response with dictionary format-export type MapV2Response =+export type MapResponse =| ErrorResponse| {success: true;- links?: string[]; // For backwards compatibility- web?: WebSearchResult[]; // New structured format+ links?: MapDocument[];metadata?: {totalCount: number;hasMore: boolean;diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex b819d420b..a9c992ebb 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -7,7 +7,7 @@ import { batchScrapeController } from "../controllers/v2/batch-scrape";import { crawlController } from "../controllers/v2/crawl";import { crawlParamsPreviewController } from "../controllers/v2/crawl-params-preview";import { crawlStatusController } from "../controllers/v2/crawl-status";-import { mapV2Controller } from "../controllers/v2/map";+import { mapController } from "../controllers/v2/map";import { crawlErrorsController } from "../controllers/v2/crawl-errors";import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";import { scrapeStatusController } from "../controllers/v2/scrape-status";@@ -54,7 +54,7 @@ v2Router.post("/map",authMiddleware(RateLimiterMode.Map),checkCreditsMiddleware(),- wrap(mapV2Controller),+ wrap(mapController),);v2Router.post(diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 34560b017..2f18d91be 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -6,6 +6,7 @@ import crypto from "crypto";import { redisEvictConnection } from "./redis";import type { Logger } from "winston";import psl from "psl";+import { MapDocument } from "../controllers/v2/types";configDotenv();// SupabaseService class initializes the Supabase client conditionally based on environment variables.@@ -624,69 +625,108 @@ export async function queryOMCESignatures(hostname: string, maxAge = 2 * 24 * 60return data?.[0]?.signatures ?? [];}-export async function queryIndexDocument(url: string): Promise<{- title?: string;- description?: string;- language?: string;- keywords?: string;- robots?: string;- ogTitle?: string;- ogDescription?: string;- ogUrl?: string;- ogImage?: string;- ogLocale?: string;- ogSiteName?: string;- createdAt?: string;- lastScrapedAt?: string;-} | null> {+export async function queryIndexAtSplitLevelWithMeta(url: string, limit: number): Promise<MapDocument[]> {if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {- return null;+ return [];}- try {- const normalizedUrl = normalizeURLForIndex(url);- const urlHash = hashURL(normalizedUrl);-- const { data, error } = await index_supabase_service- .from("index")- .select("id, title, description, created_at")- .eq("url_hash", urlHash)- .order("created_at", { ascending: false })- .limit(1)- .single();-- if (error || !data) {- return null;+ const urlObj = new URL(url);+ urlObj.search = "";++ const urlSplitsHash = generateURLSplits(urlObj.href).map(x => hashURL(x));++ const level = urlSplitsHash.length - 1;++ let links: MapDocument[] = [];+ let iteration = 0;++ while (true) {+ // Query the index for the next set of links+ const { data: _data, error } = await index_supabase_service+ .rpc("query_index_at_split_level_with_meta", {+ i_level: level,+ i_url_hash: urlSplitsHash[level],+ i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),+ })+ .range(iteration * 1000, (iteration + 1) * 1000)++ // If there's an error, return the links we have+ if (error) {+ _logger.warn("Error querying index", { error, url, limit });+ return links.slice(0, limit);}- // Try to get more metadata from GCS if available- let gcsData: any = null;- try {- gcsData = await getIndexFromGCS(data.id + ".json");- } catch (e) {- // Ignore GCS errors, we'll use what we have from the database+ // Add the links to the set+ const data = _data ?? [];+ data.forEach((x) => links.push({+ url: x.resolved_url,+ title: x.title ?? undefined,+ description: x.description ?? undefined,+ }));++ // If we have enough links, return them+ if (links.length >= limit) {+ return links.slice(0, limit);}- return {- title: data.title,- description: data.description,- createdAt: data.created_at,- lastScrapedAt: data.created_at,- // Add more fields from GCS data if available- ...(gcsData && {- language: gcsData.language,- keywords: gcsData.keywords,- robots: gcsData.robots,- ogTitle: gcsData.ogTitle,- ogDescription: gcsData.ogDescription,- ogUrl: gcsData.ogUrl,- ogImage: gcsData.ogImage,- ogLocale: gcsData.ogLocale,- ogSiteName: gcsData.ogSiteName,+ // If we get less than 1000 links from the query, we're done+ if (data.length < 1000) {+ return links.slice(0, limit);+ }++ iteration++;+ }+}++export async function queryIndexAtDomainSplitLevelWithMeta(hostname: string, limit: number): Promise<MapDocument[]> {+ if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {+ return [];+ }++ const domainSplitsHash = generateDomainSplits(hostname).map(x => hashURL(x));++ const level = domainSplitsHash.length - 1;+ if (domainSplitsHash.length === 0) {+ return [];+ }++ let links: MapDocument[] = [];+ let iteration = 0;++ while (true) {+ // Query the index for the next set of links+ const { data: _data, error } = await index_supabase_service+ .rpc("query_index_at_domain_split_level_with_meta", {+ i_level: level,+ i_domain_hash: domainSplitsHash[level],+ i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),})- };- } catch (error) {- _logger.warn("Error querying index document", { error, url });- return null;+ .range(iteration * 1000, (iteration + 1) * 1000)++ // If there's an error, return the links we have+ if (error) {+ _logger.warn("Error querying index", { error, hostname, limit });+ return links.slice(0, limit);+ }++ // Add the links to the set+ const data = _data ?? [];+ data.forEach((x) => links.push({+ url: x.resolved_url,+ title: x.title ?? undefined,+ description: x.description ?? undefined,+ }));++ // If we have enough links, return them+ if (links.length >= limit) {+ return links.slice(0, limit);+ }++ // If we get less than 1000 links from the query, we're done+ if (data.length < 1000) {+ return links.slice(0, limit);+ }++ iteration++;}}