Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL/index): index metrics (#2160)
Gergő Móricz3c3d382c8dd6
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 538c33a1a..95eb1d0ae 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -12,11 +12,9 @@ import {addIndexInsertJob,generateDomainSplits,addOMCEJob,- addDomainFrequencyJob,} from "../../../../services";import { EngineError, IndexMissError } from "../../error";import { shouldParsePDF } from "../../../../controllers/v2/types";-import { storage } from "../../../../lib/gcs-jobs";export async function sendDocumentToIndex(meta: Meta, document: Document) {const shouldCache =@@ -300,6 +298,12 @@ export async function scrapeURLWithIndex(}if (selectedRow === null || selectedRow === undefined) {+ meta.logger.debug("Index metrics", {+ module: "index/metrics",+ hit: false,+ maxAge,+ dynamicMaxAge: meta.options.maxAge === undefined,+ });throw new IndexMissError();}@@ -333,6 +337,14 @@ export async function scrapeURLWithIndex(}}+ meta.logger.debug("Index metrics", {+ module: "index/metrics",+ hit: true,+ age: Date.now() - new Date(selectedRow.created_at).getTime(),+ maxAge,+ dynamicMaxAge: meta.options.maxAge === undefined,+ });+return {url: doc.url,html: doc.html,