Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): more metric logs
Gergő Móricz9c1bc2f52191
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex affea25d2..0b26bc5ca 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -443,6 +443,22 @@ const engineOptions: {},};+export function shouldUseIndex(meta: Meta) {+ return (+ useIndex &&+ process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true" &&+ meta.options.waitFor === 0 &&+ !hasFormatOfType(meta.options.formats, "changeTracking") &&+ // Skip index if a non-default PDF maxPages is specified+ getPDFMaxPages(meta.options.parsers) === undefined &&+ meta.options.maxAge !== 0 &&+ (meta.options.headers === undefined ||+ Object.keys(meta.options.headers).length === 0) &&+ (meta.options.actions === undefined || meta.options.actions.length === 0) &&+ meta.options.proxy !== "stealth"+ );+}+export function buildFallbackList(meta: Meta): {engine: Engine;unsupportedFeatures: Set<FeatureFlag>;@@ -465,20 +481,7 @@ export function buildFallbackList(meta: Meta): {: []),];- const shouldUseIndex =- useIndex &&- process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true" &&- meta.options.waitFor === 0 &&- !hasFormatOfType(meta.options.formats, "changeTracking") &&- // Skip index if a non-default PDF maxPages is specified- getPDFMaxPages(meta.options.parsers) === undefined &&- meta.options.maxAge !== 0 &&- (meta.options.headers === undefined ||- Object.keys(meta.options.headers).length === 0) &&- (meta.options.actions === undefined || meta.options.actions.length === 0) &&- meta.options.proxy !== "stealth";-- if (!shouldUseIndex) {+ if (!shouldUseIndex(meta)) {const indexIndex = _engines.indexOf("index");if (indexIndex !== -1) {_engines.splice(indexIndex, 1);diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 73819e577..a4099210f 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -16,6 +16,7 @@ import {FeatureFlag,getEngineMaxReasonableTime,scrapeURLWithEngine,+ shouldUseIndex,} from "./engines";import { parseMarkdown } from "../../lib/html-to-markdown";import { hasFormatOfType } from "../../lib/format-utils";@@ -709,6 +710,8 @@ export async function scrapeURL(costTracking,);+ const startTime = Date.now();+meta.logger.info("scrapeURL entered");if (meta.rewrittenUrl) {@@ -811,9 +814,11 @@ export async function scrapeURL(}try {+ let result: ScrapeUrlResponse;while (true) {try {- return await scrapeURLLoop(meta);+ result = await scrapeURLLoop(meta);+ break;} catch (error) {if (error instanceof AddFeatureError &&@@ -866,6 +871,17 @@ export async function scrapeURL(}}}++ meta.logger.debug("scrapeURL metrics", {+ module: "scrapeURL/metrics",+ timeTaken: Date.now() - startTime,+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,+ shouldUseIndex: shouldUseIndex(meta),+ success: result.success,+ indexHit: result.success && result.document.metadata.cacheState === "hit",+ });++ return result;} catch (error) {// if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {// throw new FEPageLoadFailed();