Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix map and some tests

Gergő Móriczfa36ad664254
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.ts
index 304c9dd39..83e71366d 100644
--- a/apps/api/src/__tests__/snips/v2/map.test.ts
+++ b/apps/api/src/__tests__/snips/v2/map.test.ts
@@ -39,6 +39,6 @@ describe("Map tests", () => {
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
- expect(response.body.links.some(x => x.match(/^https:\/\/www\.hfea\.gov\.uk\/choose-a-clinic\/clinic-search\/results\/?\?options=\d+$/))).toBe(true);
+ expect(response.body.links.some(x => x.url.match(/^https:\/\/www\.hfea\.gov\.uk\/choose-a-clinic\/clinic-search\/results\/?\?options=\d+$/))).toBe(true);
}, 60000);
});
diff --git a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts
index 339a6f9d1..658e5dcb4 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts
@@ -15,6 +15,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {
const data = await scrape({
url: "https://expired.badssl.com/",
timeout: scrapeTimeout,
+ maxAge: 0,
}, identity);
expect(data).toBeDefined();
@@ -26,6 +27,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {
url: "https://expired.badssl.com/",
skipTlsVerification: false,
timeout: scrapeTimeout,
+ maxAge: 0,
}, identity);
if (response.status !== 500) {
@@ -40,6 +42,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {
const data = await scrape({
url: "https://firecrawl.dev",
timeout: scrapeTimeout,
+ maxAge: 0,
}, identity);
expect(data).toBeDefined();
@@ -52,6 +55,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {
url: "https://firecrawl.dev",
formats: [{ type: "screenshot", fullPage: false }],
timeout: scrapeTimeout,
+ maxAge: 0,
}, identity);
expect(data).toBeDefined();
@@ -64,6 +68,7 @@ describe("V2 Scrape skipTlsVerification Default", () => {
url: "https://firecrawl.dev",
formats: [{ type: "screenshot", fullPage: true }],
timeout: scrapeTimeout,
+ maxAge: 0,
}, identity);
expect(data).toBeDefined();
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index c4de182cc..8be52acc9 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -1,14 +1,14 @@
import { Response } from "express";
import { v4 as uuidv4 } from "uuid";
import {
+ mapRequestSchema,
RequestWithAuth,
+ scrapeOptions,
TeamFlags,
TimeoutSignal,
- scrapeOptions,
- mapRequestSchema,
MapRequest,
- MapV2Response,
MapDocument,
+ MapResponse,
} from "./types";
import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
import { configDotenv } from "dotenv";
@@ -16,16 +16,13 @@ import {
checkAndUpdateURLForMap,
isSameDomain,
isSameSubdomain,
- removeDuplicateUrls,
} from "../../lib/validateUrl";
import { fireEngineMap } from "../../search/fireEngine";
import { billTeam } from "../../services/billing/credit_billing";
import { logJob } from "../../services/logging/log_job";
-import { performCosineSimilarity } from "../../lib/map-cosine";
import { logger } from "../../lib/logger";
import Redis from "ioredis";
-import { generateURLSplits, queryIndexAtDomainSplitLevel, queryIndexAtSplitLevel, queryIndexDocument } from "../../services/index";
-import { WebSearchResult } from "../../lib/entities";
+import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../../services/index";
configDotenv();
const redis = new Redis(process.env.REDIS_URL!);
@@ -35,18 +32,14 @@ const MAX_MAP_LIMIT = 30000;
// Max Links that "Smart /map" can return
const MAX_FIRE_ENGINE_RESULTS = 500;
-// MapRequest and MapV2Response are imported from ./types
-
interface MapResult {
success: boolean;
- links: string[];
- scrape_id?: string;
job_id: string;
time_taken: number;
- mapResults: WebSearchResult[];
+ mapResults: MapDocument[];
}
-async function queryIndex(url: string, limit: number, useIndex: boolean, includeSubdomains: boolean): Promise<string[]> {
+async function queryIndex(url: string, limit: number, useIndex: boolean): Promise<MapDocument[]> {
if (!useIndex) {
return [];
}
@@ -56,109 +49,49 @@ async function queryIndex(url: string, limit: number, useIndex: boolean, include
const urlObj = new URL(url);
const hostname = urlObj.hostname;
- // TEMP: this should be altered on June 15th 2025 7AM PT - mogery
- const domainLinks = includeSubdomains ? await queryIndexAtDomainSplitLevel(hostname, limit, 14 * 24 * 60 * 60 * 1000) : [];
- const splitLinks = await queryIndexAtSplitLevel(url, limit, 14 * 24 * 60 * 60 * 1000);
-
- return Array.from(new Set([...domainLinks, ...splitLinks]));
+ return (await queryIndexAtDomainSplitLevelWithMeta(hostname, limit));
} else {
- return await queryIndexAtSplitLevel(url, limit);
+ return (await queryIndexAtSplitLevelWithMeta(url, limit));
}
}
-async function enrichUrlsWithMetadata(urls: string[], useIndex: boolean, teamId: string, flags: TeamFlags): Promise<WebSearchResult[]> {
- if (!useIndex || urls.length === 0) {
- return urls.map(url => ({
- url,
- title: "",
- description: "",
- }));
- }
-
- // Query metadata for all URLs in parallel
- const metadataPromises = urls.map(async (url) => {
- try {
- const indexDocument = await queryIndexDocument(url);
- if (indexDocument) {
- return {
- url,
- title: indexDocument.title || "",
- description: indexDocument.description || "",
- metadata: {
- title: indexDocument.title,
- description: indexDocument.description,
- language: indexDocument.language,
- keywords: indexDocument.keywords,
- robots: indexDocument.robots,
- ogTitle: indexDocument.ogTitle,
- ogDescription: indexDocument.ogDescription,
- ogUrl: indexDocument.ogUrl,
- ogImage: indexDocument.ogImage,
- ogLocale: indexDocument.ogLocale,
- ogSiteName: indexDocument.ogSiteName,
- createdAt: indexDocument.createdAt,
- lastScrapedAt: indexDocument.lastScrapedAt,
- }
- };
- }
- } catch (error) {
- logger.warn(`Failed to fetch metadata for URL: ${url}`, { error });
- }
-
- // Fallback if no metadata found
- return {
- url,
- title: "",
- description: "",
- };
- });
-
- return await Promise.all(metadataPromises);
-}
-
-export async function getMapV2Results({
+export async function getMapResults({
url,
search,
limit = MAX_MAP_LIMIT,
- sitemap = "include",
includeSubdomains = true,
+ crawlerOptions = {},
teamId,
- origin,
allowExternalLinks,
- abort = new AbortController().signal,
- mock,
+ abort = new AbortController().signal, // noop
filterByPath = true,
flags,
useIndex = true,
- includeMetadata = true,
- timeout,
}: {
url: string;
search?: string;
limit?: number;
- sitemap?: "only" | "include" | "skip";
+ ignoreSitemap?: boolean;
includeSubdomains?: boolean;
+ crawlerOptions?: any;
teamId: string;
origin?: string;
+ includeMetadata?: boolean;
allowExternalLinks?: boolean;
abort?: AbortSignal;
mock?: string;
filterByPath?: boolean;
flags: TeamFlags;
useIndex?: boolean;
- includeMetadata?: boolean;
- timeout?: number;
}): Promise<MapResult> {
const id = uuidv4();
- let links: string[] = [url];
- let mapResults: WebSearchResult[] = [];
-
- const zeroDataRetention = flags?.forceZDR ?? false;
+ let mapResults: MapDocument[] = [];
const sc: StoredCrawl = {
originUrl: url,
crawlerOptions: {
- limit: sitemap === "only" ? 10000000 : limit,
+ ...crawlerOptions,
+ limit: crawlerOptions.sitemapOnly ? 10000000 : limit,
scrapeOptions: undefined,
},
scrapeOptions: scrapeOptions.parse({}),
@@ -174,267 +107,156 @@ export async function getMapV2Results({
crawler.importRobotsTxt(sc.robots);
} catch (_) {}
- // If sitemap is "only", only get links from sitemap
- if (sitemap === "only") {
- const sitemapCount = await crawler.tryGetSitemap(
- (urls) => {
- urls.forEach((x) => {
- links.push(x);
- });
- },
- true,
- true,
- timeout ?? 30000,
- abort,
- mock,
- );
- if (sitemapCount > 0) {
- links = links
- .slice(1)
- .map((x) => {
- try {
- return checkAndUpdateURLForMap(x).url.trim();
- } catch (_) {
- return null;
- }
- })
- .filter((x) => x !== null) as string[];
- }
- } else if (sitemap !== "skip") {
- // sitemap is "include" - fetch both search results and sitemap
- let urlWithoutWww = url.replace("www.", "");
-
- let mapUrl =
- search && allowExternalLinks
- ? `${search} ${urlWithoutWww}`
- : search
- ? `${search} site:${urlWithoutWww}`
- : `site:${url}`;
-
- const resultsPerPage = 100;
- const maxPages = Math.ceil(
- Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,
- );
+ let urlWithoutWww = url.replace("www.", "");
- const cacheKey = `fireEngineMap:${mapUrl}`;
- const cachedResult = await redis.get(cacheKey);
+ let mapUrl =
+ search && allowExternalLinks
+ ? `${search} ${urlWithoutWww}`
+ : search
+ ? `${search} site:${urlWithoutWww}`
+ : `site:${url}`;
- let allResults: any[] = [];
- let pagePromises: Promise<any>[] = [];
+ const resultsPerPage = 100;
+ const maxPages = Math.ceil(
+ Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,
+ );
- if (cachedResult) {
- allResults = JSON.parse(cachedResult);
- } else {
- const fetchPage = async (page: number) => {
- return fireEngineMap(mapUrl, {
- numResults: resultsPerPage,
- page: page,
- }, abort);
- };
-
- pagePromises = Array.from({ length: maxPages }, (_, i) =>
- fetchPage(i + 1),
- );
- allResults = await Promise.all(pagePromises);
-
- if (!zeroDataRetention) {
- await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours
- }
- }
+ const cacheKey = `fireEngineMap:${mapUrl}`;
+ const cachedResult = await redis.get(cacheKey);
- // Parallelize sitemap index query with search results
- const [indexResults, ...searchResults] = await Promise.all([
- queryIndex(url, limit, useIndex, includeSubdomains),
- ...(cachedResult ? [] : pagePromises),
- ]);
+ let allResults: any[] = [];
+ let pagePromises: Promise<any>[] = [];
- if (indexResults.length > 0) {
- links.push(...indexResults);
- }
+ if (cachedResult) {
+ allResults = JSON.parse(cachedResult);
+ } else {
+ const fetchPage = async (page: number) => {
+ return fireEngineMap(mapUrl, {
+ numResults: resultsPerPage,
+ page: page,
+ }, abort);
+ };
- // Fetch sitemap if sitemap is "include"
- try {
- await crawler.tryGetSitemap(
- (urls) => {
- links.push(...urls);
- },
- true,
- false,
- timeout ?? 30000,
- abort,
- );
- } catch (e) {
- logger.warn("tryGetSitemap threw an error", { error: e });
- }
+ pagePromises = Array.from({ length: maxPages }, (_, i) =>
+ fetchPage(i + 1),
+ );
+ allResults = await Promise.all(pagePromises);
- if (!cachedResult) {
- allResults = searchResults;
- }
+ await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours
+ }
- // Convert map results to WebSearchResult format
- const fireEngineResults = allResults
- .flat()
- .filter((result) => result !== null && result !== undefined)
- .map((result) => ({
- url: result.url,
- title: result.title || "",
- description: result.description || "",
- }));
-
- const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);
- if (fireEngineResults.length > minumumCutoff) {
- mapResults = fireEngineResults.slice(0, minumumCutoff);
- } else {
- mapResults = fireEngineResults;
- }
+ // Parallelize sitemap index query with search results
+ const [indexResults, ...searchResults] = await Promise.all([
+ queryIndex(url, limit, useIndex),
+ ...(cachedResult ? [] : pagePromises),
+ ]);
- if (mapResults.length > 0) {
- if (search) {
- // Ensure all map results are first, maintaining their order
- links = [
- mapResults[0].url,
- ...mapResults.slice(1).map((x) => x.url),
- ...links,
- ];
- } else {
- mapResults.map((x) => {
- links.push(x.url);
- });
- }
- }
+ if (indexResults.length > 0) {
+ mapResults.push(...indexResults);
+ }
- // Perform cosine similarity between the search query and the list of links
- if (search) {
- const searchQuery = search.toLowerCase();
- links = performCosineSimilarity(links, searchQuery);
- }
+ console.log("searchResults", searchResults);
- links = links
- .map((x) => {
- try {
- return checkAndUpdateURLForMap(x).url.trim();
- } catch (_) {
- return null;
- }
- })
- .filter((x) => x !== null) as string[];
-
- // allows for subdomains to be included
- links = links.filter((x) => isSameDomain(x, url));
-
- // if includeSubdomains is false, filter out subdomains
- if (!includeSubdomains) {
- links = links.filter((x) => isSameSubdomain(x, url));
- }
+ mapResults = mapResults.concat(searchResults.map(x => ({
+ url: x.url,
+ title: x.title,
+ description: x.description,
+ })).flat());
+
+ const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);
+ if (mapResults.length > minumumCutoff) {
+ mapResults = mapResults.slice(0, minumumCutoff);
+ }
- // Filter by path if enabled
- if (filterByPath && !allowExternalLinks) {
+ mapResults = mapResults
+ .map((x) => {
try {
- const urlObj = new URL(url);
- const urlPath = urlObj.pathname;
- // Only apply path filtering if the URL has a significant path (not just '/' or empty)
- // This means we only filter by path if the user has not selected a root domain
- if (urlPath && urlPath !== '/' && urlPath.length > 1) {
- links = links.filter(link => {
- try {
- const linkObj = new URL(link);
- return linkObj.pathname.startsWith(urlPath);
- } catch (e) {
- return false;
- }
- });
- }
- } catch (e) {
- // If URL parsing fails, continue without path filtering
- logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });
+ return {
+ ...x,
+ url: checkAndUpdateURLForMap(x.url).url.trim(),
+ };
+ } catch (_) {
+ return null;
}
- }
+ })
+ .filter((x) => x !== null) as MapDocument[];
- // remove duplicates that could be due to http/https or www
- links = removeDuplicateUrls(links);
- }
+ // allows for subdomains to be included
+ mapResults = mapResults.filter((x) => isSameDomain(x.url, url));
- const linksToReturn = sitemap === "only"
- ? links
- : links.slice(0, limit);
+ // if includeSubdomains is false, filter out subdomains
+ if (!includeSubdomains) {
+ mapResults = mapResults.filter((x) => isSameSubdomain(x.url, url));
+ }
- // Enrich URLs with metadata if requested
- let enrichedResults: WebSearchResult[] = [];
- if (includeMetadata) {
- enrichedResults = await enrichUrlsWithMetadata(linksToReturn, useIndex, teamId, flags);
- } else {
- enrichedResults = linksToReturn.map(url => ({
- url,
- title: "",
- description: "",
- }));
+ // Filter by path if enabled
+ if (filterByPath && !allowExternalLinks) {
+ try {
+ const urlObj = new URL(url);
+ const urlPath = urlObj.pathname;
+ // Only apply path filtering if the URL has a significant path (not just '/' or empty)
+ // This means we only filter by path if the user has not selected a root domain
+ if (urlPath && urlPath !== '/' && urlPath.length > 1) {
+ mapResults = mapResults.filter(x => {
+ try {
+ const linkObj = new URL(x.url);
+ return linkObj.pathname.startsWith(urlPath);
+ } catch (e) {
+ return false;
+ }
+ });
+ }
+ } catch (e) {
+ // If URL parsing fails, continue without path filtering
+ logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });
+ }
}
return {
success: true,
- links: linksToReturn,
- mapResults: enrichedResults,
- scrape_id: origin?.includes("website") ? id : undefined,
+ mapResults: mapResults,
job_id: id,
time_taken: (new Date().getTime() - Date.now()) / 1000,
};
}
-export async function mapV2Controller(
- req: RequestWithAuth<{}, MapV2Response, MapRequest>,
- res: Response<MapV2Response>,
+export async function mapController(
+ req: RequestWithAuth<{}, MapResponse, MapRequest>,
+ res: Response<MapResponse>,
) {
const originalRequest = req.body;
-
- // Parse and validate the request using the v2 schema
- const parsedRequest = mapRequestSchema.parse(req.body);
-
- // Transform ignoreSitemap and sitemapOnly to the new unified sitemap parameter
- let sitemap: "only" | "include" | "skip" = "include";
- if (parsedRequest.sitemapOnly) {
- sitemap = "only";
- } else if (parsedRequest.ignoreSitemap) {
- sitemap = "skip";
- }
-
- if (req.acuc?.flags?.forceZDR) {
- return res.status(400).json({
- success: false,
- error: "Your team has zero data retention enabled. This is not supported on map. Please contact [email protected] to unblock this feature."
- });
- }
+ req.body = mapRequestSchema.parse(req.body);
- logger.info("Map V2 request", {
- request: parsedRequest,
+ logger.info("Map request", {
+ request: req.body,
originalRequest,
teamId: req.auth.team_id,
});
- let result: Awaited<ReturnType<typeof getMapV2Results>>;
+ let result: Awaited<ReturnType<typeof getMapResults>>;
const abort = new AbortController();
try {
result = await Promise.race([
- getMapV2Results({
- url: parsedRequest.url,
- search: parsedRequest.search,
- limit: parsedRequest.limit,
- sitemap,
- includeSubdomains: parsedRequest.includeSubdomains,
- origin: parsedRequest.origin,
+ getMapResults({
+ url: req.body.url,
+ search: req.body.search,
+ limit: req.body.limit,
+ ignoreSitemap: req.body.ignoreSitemap,
+ includeSubdomains: req.body.includeSubdomains,
+ crawlerOptions: req.body,
+ origin: req.body.origin,
teamId: req.auth.team_id,
abort: abort.signal,
- mock: parsedRequest.useMock,
- filterByPath: parsedRequest.filterByPath,
+ mock: req.body.useMock,
+ filterByPath: req.body.filterByPath !== false,
flags: req.acuc?.flags ?? null,
- useIndex: parsedRequest.useIndex,
- includeMetadata: true, // Always include metadata in v2
- timeout: parsedRequest.timeout,
+ useIndex: req.body.useIndex,
}),
- ...(parsedRequest.timeout !== undefined ? [
+ ...(req.body.timeout !== undefined ? [
new Promise((resolve, reject) => setTimeout(() => {
abort.abort(new TimeoutSignal());
reject(new TimeoutSignal());
- }, parsedRequest.timeout!))
+ }, req.body.timeout))
] : []),
]) as any;
} catch (error) {
@@ -458,33 +280,27 @@ export async function mapV2Controller(
// Log the job
logJob({
job_id: result.job_id,
- success: result.links.length > 0,
- message: "Map V2 completed",
- num_docs: result.links.length,
- docs: result.links,
+ success: result.mapResults.length > 0,
+ message: "Map completed",
+ num_docs: result.mapResults.length,
+ docs: result.mapResults,
time_taken: result.time_taken,
team_id: req.auth.team_id,
mode: "map",
- url: parsedRequest.url,
+ url: req.body.url,
crawlerOptions: {},
scrapeOptions: {},
- origin: parsedRequest.origin ?? "api",
- integration: parsedRequest.integration,
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
num_tokens: 0,
credits_billed: 1,
zeroDataRetention: false, // not supported
});
- const response: MapV2Response = {
+ const response = {
success: true as const,
- links: result.links, // For backwards compatibility
- web: result.mapResults,
- metadata: {
- totalCount: result.links.length,
- hasMore: result.links.length >= parsedRequest.limit,
- searchQuery: parsedRequest.search,
- },
+ links: result.mapResults,
};
return res.status(200).json(response);
-}
\ No newline at end of file
+}
\ No newline at end of file
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 2136b8c20..47b2c0aa1 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -804,14 +804,6 @@ export type BatchScrapeResponse =
invalidURLs?: string[];
};
-export type MapResponse =
- | ErrorResponse
- | {
- success: true;
- links: string[];
- scrape_id?: string;
- };
-
// Map document interface (transitioned from v1)
export interface MapDocument {
url: string;
@@ -820,12 +812,11 @@ export interface MapDocument {
}
// V2 Map Response with dictionary format
-export type MapV2Response =
+export type MapResponse =
| ErrorResponse
| {
success: true;
- links?: string[]; // For backwards compatibility
- web?: WebSearchResult[]; // New structured format
+ links?: MapDocument[];
metadata?: {
totalCount: number;
hasMore: boolean;
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index b819d420b..a9c992ebb 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -7,7 +7,7 @@ import { batchScrapeController } from "../controllers/v2/batch-scrape";
import { crawlController } from "../controllers/v2/crawl";
import { crawlParamsPreviewController } from "../controllers/v2/crawl-params-preview";
import { crawlStatusController } from "../controllers/v2/crawl-status";
-import { mapV2Controller } from "../controllers/v2/map";
+import { mapController } from "../controllers/v2/map";
import { crawlErrorsController } from "../controllers/v2/crawl-errors";
import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";
import { scrapeStatusController } from "../controllers/v2/scrape-status";
@@ -54,7 +54,7 @@ v2Router.post(
"/map",
authMiddleware(RateLimiterMode.Map),
checkCreditsMiddleware(),
- wrap(mapV2Controller),
+ wrap(mapController),
);
v2Router.post(
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 34560b017..2f18d91be 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -6,6 +6,7 @@ import crypto from "crypto";
import { redisEvictConnection } from "./redis";
import type { Logger } from "winston";
import psl from "psl";
+import { MapDocument } from "../controllers/v2/types";
configDotenv();
// SupabaseService class initializes the Supabase client conditionally based on environment variables.
@@ -624,69 +625,108 @@ export async function queryOMCESignatures(hostname: string, maxAge = 2 * 24 * 60
return data?.[0]?.signatures ?? [];
}
-export async function queryIndexDocument(url: string): Promise<{
- title?: string;
- description?: string;
- language?: string;
- keywords?: string;
- robots?: string;
- ogTitle?: string;
- ogDescription?: string;
- ogUrl?: string;
- ogImage?: string;
- ogLocale?: string;
- ogSiteName?: string;
- createdAt?: string;
- lastScrapedAt?: string;
-} | null> {
+export async function queryIndexAtSplitLevelWithMeta(url: string, limit: number): Promise<MapDocument[]> {
if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
- return null;
+ return [];
}
- try {
- const normalizedUrl = normalizeURLForIndex(url);
- const urlHash = hashURL(normalizedUrl);
-
- const { data, error } = await index_supabase_service
- .from("index")
- .select("id, title, description, created_at")
- .eq("url_hash", urlHash)
- .order("created_at", { ascending: false })
- .limit(1)
- .single();
-
- if (error || !data) {
- return null;
+ const urlObj = new URL(url);
+ urlObj.search = "";
+
+ const urlSplitsHash = generateURLSplits(urlObj.href).map(x => hashURL(x));
+
+ const level = urlSplitsHash.length - 1;
+
+ let links: MapDocument[] = [];
+ let iteration = 0;
+
+ while (true) {
+ // Query the index for the next set of links
+ const { data: _data, error } = await index_supabase_service
+ .rpc("query_index_at_split_level_with_meta", {
+ i_level: level,
+ i_url_hash: urlSplitsHash[level],
+ i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
+ })
+ .range(iteration * 1000, (iteration + 1) * 1000)
+
+ // If there's an error, return the links we have
+ if (error) {
+ _logger.warn("Error querying index", { error, url, limit });
+ return links.slice(0, limit);
}
- // Try to get more metadata from GCS if available
- let gcsData: any = null;
- try {
- gcsData = await getIndexFromGCS(data.id + ".json");
- } catch (e) {
- // Ignore GCS errors, we'll use what we have from the database
+ // Add the links to the set
+ const data = _data ?? [];
+ data.forEach((x) => links.push({
+ url: x.resolved_url,
+ title: x.title ?? undefined,
+ description: x.description ?? undefined,
+ }));
+
+ // If we have enough links, return them
+ if (links.length >= limit) {
+ return links.slice(0, limit);
}
- return {
- title: data.title,
- description: data.description,
- createdAt: data.created_at,
- lastScrapedAt: data.created_at,
- // Add more fields from GCS data if available
- ...(gcsData && {
- language: gcsData.language,
- keywords: gcsData.keywords,
- robots: gcsData.robots,
- ogTitle: gcsData.ogTitle,
- ogDescription: gcsData.ogDescription,
- ogUrl: gcsData.ogUrl,
- ogImage: gcsData.ogImage,
- ogLocale: gcsData.ogLocale,
- ogSiteName: gcsData.ogSiteName,
+ // If we get less than 1000 links from the query, we're done
+ if (data.length < 1000) {
+ return links.slice(0, limit);
+ }
+
+ iteration++;
+ }
+}
+
+export async function queryIndexAtDomainSplitLevelWithMeta(hostname: string, limit: number): Promise<MapDocument[]> {
+ if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
+ return [];
+ }
+
+ const domainSplitsHash = generateDomainSplits(hostname).map(x => hashURL(x));
+
+ const level = domainSplitsHash.length - 1;
+ if (domainSplitsHash.length === 0) {
+ return [];
+ }
+
+ let links: MapDocument[] = [];
+ let iteration = 0;
+
+ while (true) {
+ // Query the index for the next set of links
+ const { data: _data, error } = await index_supabase_service
+ .rpc("query_index_at_domain_split_level_with_meta", {
+ i_level: level,
+ i_domain_hash: domainSplitsHash[level],
+ i_newer_than: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
})
- };
- } catch (error) {
- _logger.warn("Error querying index document", { error, url });
- return null;
+ .range(iteration * 1000, (iteration + 1) * 1000)
+
+ // If there's an error, return the links we have
+ if (error) {
+ _logger.warn("Error querying index", { error, hostname, limit });
+ return links.slice(0, limit);
+ }
+
+ // Add the links to the set
+ const data = _data ?? [];
+ data.forEach((x) => links.push({
+ url: x.resolved_url,
+ title: x.title ?? undefined,
+ description: x.description ?? undefined,
+ }));
+
+ // If we have enough links, return them
+ if (links.length >= limit) {
+ return links.slice(0, limit);
+ }
+
+ // If we get less than 1000 links from the query, we're done
+ if (data.length < 1000) {
+ return links.slice(0, limit);
+ }
+
+ iteration++;
}
}