Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): more metric logs

Gergő Móricz9c1bc2f52191
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index affea25d2..0b26bc5ca 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -443,6 +443,22 @@ const engineOptions: {
},
};
+export function shouldUseIndex(meta: Meta) {
+ return (
+ useIndex &&
+ process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true" &&
+ meta.options.waitFor === 0 &&
+ !hasFormatOfType(meta.options.formats, "changeTracking") &&
+ // Skip index if a non-default PDF maxPages is specified
+ getPDFMaxPages(meta.options.parsers) === undefined &&
+ meta.options.maxAge !== 0 &&
+ (meta.options.headers === undefined ||
+ Object.keys(meta.options.headers).length === 0) &&
+ (meta.options.actions === undefined || meta.options.actions.length === 0) &&
+ meta.options.proxy !== "stealth"
+ );
+}
+
export function buildFallbackList(meta: Meta): {
engine: Engine;
unsupportedFeatures: Set<FeatureFlag>;
@@ -465,20 +481,7 @@ export function buildFallbackList(meta: Meta): {
: []),
];
- const shouldUseIndex =
- useIndex &&
- process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true" &&
- meta.options.waitFor === 0 &&
- !hasFormatOfType(meta.options.formats, "changeTracking") &&
- // Skip index if a non-default PDF maxPages is specified
- getPDFMaxPages(meta.options.parsers) === undefined &&
- meta.options.maxAge !== 0 &&
- (meta.options.headers === undefined ||
- Object.keys(meta.options.headers).length === 0) &&
- (meta.options.actions === undefined || meta.options.actions.length === 0) &&
- meta.options.proxy !== "stealth";
-
- if (!shouldUseIndex) {
+ if (!shouldUseIndex(meta)) {
const indexIndex = _engines.indexOf("index");
if (indexIndex !== -1) {
_engines.splice(indexIndex, 1);
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 73819e577..a4099210f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -16,6 +16,7 @@ import {
FeatureFlag,
getEngineMaxReasonableTime,
scrapeURLWithEngine,
+ shouldUseIndex,
} from "./engines";
import { parseMarkdown } from "../../lib/html-to-markdown";
import { hasFormatOfType } from "../../lib/format-utils";
@@ -709,6 +710,8 @@ export async function scrapeURL(
costTracking,
);
+ const startTime = Date.now();
+
meta.logger.info("scrapeURL entered");
if (meta.rewrittenUrl) {
@@ -811,9 +814,11 @@ export async function scrapeURL(
}
try {
+ let result: ScrapeUrlResponse;
while (true) {
try {
- return await scrapeURLLoop(meta);
+ result = await scrapeURLLoop(meta);
+ break;
} catch (error) {
if (
error instanceof AddFeatureError &&
@@ -866,6 +871,17 @@ export async function scrapeURL(
}
}
}
+
+ meta.logger.debug("scrapeURL metrics", {
+ module: "scrapeURL/metrics",
+ timeTaken: Date.now() - startTime,
+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,
+ shouldUseIndex: shouldUseIndex(meta),
+ success: result.success,
+ indexHit: result.success && result.document.metadata.cacheState === "hit",
+ });
+
+ return result;
} catch (error) {
// if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {
// throw new FEPageLoadFailed();