Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update sendToSearchIndex.ts
Nicolascf31c78d4f8c
diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.tsindex 07ac02613..d630a5f28 100644--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts@@ -1,9 +1,9 @@/*** Transformer: Send Document to Search Index- *+ ** Integrates with the existing scraper transformer stack.* Queues documents for real-time search indexing.- *+ ** Sampling: Controlled via SEARCH_INDEX_SAMPLE_RATE (0.0-1.0)* - 0.1 = 10% of documents indexed (recommended for initial rollout)* - 1.0 = 100% of documents indexed (full production)@@ -18,38 +18,35 @@ import { Meta } from "..";* Check if document should be indexed for search*/function shouldIndexForSearch(meta: Meta, document: Document): boolean {--if (meta.internalOptions.zeroDataRetention) {return false;}-+const statusCode = document.metadata.statusCode;if (statusCode < 200 || statusCode >= 300) {return false;}-+// Check if markdown content exists and is substantialconst markdown = document.markdown ?? "";if (markdown.length < 200) {return false;}-+// Don't index if has auth headers (private content)if (meta.options.headers &&- (meta.options.headers["Authorization"] ||- meta.options.headers["Cookie"])+ (meta.options.headers["Authorization"] || meta.options.headers["Cookie"])) {return false;}-+// Don't index PDFs without parsingconst isPdf = document.metadata.contentType?.includes("pdf");if (isPdf && !document.markdown) {return false;}-+return true;}@@ -60,7 +57,7 @@ function shouldSampleDocument(): boolean {// Get sample rate from environment (default 10% for safe rollout)const sampleRateStr = process.env.SEARCH_INDEX_SAMPLE_RATE || "0.1";const sampleRate = parseFloat(sampleRateStr);-+// Validate sample rateif (isNaN(sampleRate) || sampleRate < 0 || sampleRate > 1) {_logger.warn("Invalid SEARCH_INDEX_SAMPLE_RATE, using 0.1 (10%)", {@@ -68,7 +65,7 @@ function shouldSampleDocument(): boolean {});return Math.random() < 0.1;}-+// Sample based on ratereturn Math.random() < sampleRate;}@@ -84,16 +81,16 @@ export async function sendDocumentToSearchIndex(const searchIndexEnabled =process.env.ENABLE_SEARCH_INDEX === "true" &&process.env.SEARCH_SERVICE_URL;-+meta.logger.debug("Sending document to search index", {url: meta.url,searchIndexEnabled,});-+if (!searchIndexEnabled) {return document;}-+// Apply sampling (canary rollout)if (!shouldSampleDocument()) {meta.logger.debug("Document not sampled for search indexing", {@@ -102,7 +99,7 @@ export async function sendDocumentToSearchIndex(});return document;}-+// Check if document should be indexedif (!shouldIndexForSearch(meta, document)) {meta.logger.debug("Document not suitable for search index", {@@ -112,40 +109,43 @@ export async function sendDocumentToSearchIndex(});return document;}-+// Get the indexId from document metadata (set by sendDocumentToIndex transformer)// Format as GCS path: {indexId}.json- const gcsPath = document.metadata.indexId-+ const gcsPath = document.metadata.indexId;+// Remove indexId from metadata after extracting it (internal field, shouldn't be exposed to user)delete document.metadata.indexId;-+// Send to search service via HTTP (async, don't block scraper)(async () => {try {- await indexDocumentIfEnabled({- url: meta.url,- resolvedUrl:- document.metadata.url ??- document.metadata.sourceURL ??- meta.rewrittenUrl ??- meta.url,- title:- document.metadata.title ?? document.metadata.ogTitle ?? undefined,- description:- document.metadata.description ??- document.metadata.ogDescription ??- undefined,- markdown: document.markdown ?? "",- html: document.rawHtml ?? "",- statusCode: document.metadata.statusCode,- gcsPath: gcsPath,- screenshotUrl: document.screenshot ?? undefined,- language: document.metadata.language ?? "en",- country: meta.options.location?.country ?? undefined,- isMobile: meta.options.mobile ?? false,- }, meta.logger);-+ await indexDocumentIfEnabled(+ {+ url: meta.url,+ resolvedUrl:+ document.metadata.url ??+ document.metadata.sourceURL ??+ meta.rewrittenUrl ??+ meta.url,+ title:+ document.metadata.title ?? document.metadata.ogTitle ?? undefined,+ description:+ document.metadata.description ??+ document.metadata.ogDescription ??+ undefined,+ markdown: document.markdown ?? "",+ html: document.rawHtml ?? "",+ statusCode: document.metadata.statusCode,+ gcsPath: gcsPath,+ screenshotUrl: document.screenshot ?? undefined,+ language: document.metadata.language ?? "en",+ country: meta.options.location?.country ?? undefined,+ isMobile: meta.options.mobile ?? false,+ },+ meta.logger,+ );+meta.logger.debug("Sent document to search service", {url: meta.url,});@@ -156,7 +156,6 @@ export async function sendDocumentToSearchIndex(});}})();-+return document;}-