Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL/index): index metrics (#2160)

Gergő Móricz3c3d382c8dd6
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 538c33a1a..95eb1d0ae 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -12,11 +12,9 @@ import {
addIndexInsertJob,
generateDomainSplits,
addOMCEJob,
- addDomainFrequencyJob,
} from "../../../../services";
import { EngineError, IndexMissError } from "../../error";
import { shouldParsePDF } from "../../../../controllers/v2/types";
-import { storage } from "../../../../lib/gcs-jobs";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
const shouldCache =
@@ -300,6 +298,12 @@ export async function scrapeURLWithIndex(
}
if (selectedRow === null || selectedRow === undefined) {
+ meta.logger.debug("Index metrics", {
+ module: "index/metrics",
+ hit: false,
+ maxAge,
+ dynamicMaxAge: meta.options.maxAge === undefined,
+ });
throw new IndexMissError();
}
@@ -333,6 +337,14 @@ export async function scrapeURLWithIndex(
}
}
+ meta.logger.debug("Index metrics", {
+ module: "index/metrics",
+ hit: true,
+ age: Date.now() - new Date(selectedRow.created_at).getTime(),
+ maxAge,
+ dynamicMaxAge: meta.options.maxAge === undefined,
+ });
+
return {
url: doc.url,
html: doc.html,