Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update sendToSearchIndex.ts

Nicolascf31c78d4f8c
diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
index 07ac02613..d630a5f28 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
@@ -1,9 +1,9 @@
/**
* Transformer: Send Document to Search Index
- *
+ *
* Integrates with the existing scraper transformer stack.
* Queues documents for real-time search indexing.
- *
+ *
* Sampling: Controlled via SEARCH_INDEX_SAMPLE_RATE (0.0-1.0)
* - 0.1 = 10% of documents indexed (recommended for initial rollout)
* - 1.0 = 100% of documents indexed (full production)
@@ -18,38 +18,35 @@ import { Meta } from "..";
* Check if document should be indexed for search
*/
function shouldIndexForSearch(meta: Meta, document: Document): boolean {
-
-
if (meta.internalOptions.zeroDataRetention) {
return false;
}
-
+
const statusCode = document.metadata.statusCode;
if (statusCode < 200 || statusCode >= 300) {
return false;
}
-
+
// Check if markdown content exists and is substantial
const markdown = document.markdown ?? "";
if (markdown.length < 200) {
return false;
}
-
+
// Don't index if has auth headers (private content)
if (
meta.options.headers &&
- (meta.options.headers["Authorization"] ||
- meta.options.headers["Cookie"])
+ (meta.options.headers["Authorization"] || meta.options.headers["Cookie"])
) {
return false;
}
-
+
// Don't index PDFs without parsing
const isPdf = document.metadata.contentType?.includes("pdf");
if (isPdf && !document.markdown) {
return false;
}
-
+
return true;
}
@@ -60,7 +57,7 @@ function shouldSampleDocument(): boolean {
// Get sample rate from environment (default 10% for safe rollout)
const sampleRateStr = process.env.SEARCH_INDEX_SAMPLE_RATE || "0.1";
const sampleRate = parseFloat(sampleRateStr);
-
+
// Validate sample rate
if (isNaN(sampleRate) || sampleRate < 0 || sampleRate > 1) {
_logger.warn("Invalid SEARCH_INDEX_SAMPLE_RATE, using 0.1 (10%)", {
@@ -68,7 +65,7 @@ function shouldSampleDocument(): boolean {
});
return Math.random() < 0.1;
}
-
+
// Sample based on rate
return Math.random() < sampleRate;
}
@@ -84,16 +81,16 @@ export async function sendDocumentToSearchIndex(
const searchIndexEnabled =
process.env.ENABLE_SEARCH_INDEX === "true" &&
process.env.SEARCH_SERVICE_URL;
-
+
meta.logger.debug("Sending document to search index", {
url: meta.url,
searchIndexEnabled,
});
-
+
if (!searchIndexEnabled) {
return document;
}
-
+
// Apply sampling (canary rollout)
if (!shouldSampleDocument()) {
meta.logger.debug("Document not sampled for search indexing", {
@@ -102,7 +99,7 @@ export async function sendDocumentToSearchIndex(
});
return document;
}
-
+
// Check if document should be indexed
if (!shouldIndexForSearch(meta, document)) {
meta.logger.debug("Document not suitable for search index", {
@@ -112,40 +109,43 @@ export async function sendDocumentToSearchIndex(
});
return document;
}
-
+
// Get the indexId from document metadata (set by sendDocumentToIndex transformer)
// Format as GCS path: {indexId}.json
- const gcsPath = document.metadata.indexId
-
+ const gcsPath = document.metadata.indexId;
+
// Remove indexId from metadata after extracting it (internal field, shouldn't be exposed to user)
delete document.metadata.indexId;
-
+
// Send to search service via HTTP (async, don't block scraper)
(async () => {
try {
- await indexDocumentIfEnabled({
- url: meta.url,
- resolvedUrl:
- document.metadata.url ??
- document.metadata.sourceURL ??
- meta.rewrittenUrl ??
- meta.url,
- title:
- document.metadata.title ?? document.metadata.ogTitle ?? undefined,
- description:
- document.metadata.description ??
- document.metadata.ogDescription ??
- undefined,
- markdown: document.markdown ?? "",
- html: document.rawHtml ?? "",
- statusCode: document.metadata.statusCode,
- gcsPath: gcsPath,
- screenshotUrl: document.screenshot ?? undefined,
- language: document.metadata.language ?? "en",
- country: meta.options.location?.country ?? undefined,
- isMobile: meta.options.mobile ?? false,
- }, meta.logger);
-
+ await indexDocumentIfEnabled(
+ {
+ url: meta.url,
+ resolvedUrl:
+ document.metadata.url ??
+ document.metadata.sourceURL ??
+ meta.rewrittenUrl ??
+ meta.url,
+ title:
+ document.metadata.title ?? document.metadata.ogTitle ?? undefined,
+ description:
+ document.metadata.description ??
+ document.metadata.ogDescription ??
+ undefined,
+ markdown: document.markdown ?? "",
+ html: document.rawHtml ?? "",
+ statusCode: document.metadata.statusCode,
+ gcsPath: gcsPath,
+ screenshotUrl: document.screenshot ?? undefined,
+ language: document.metadata.language ?? "en",
+ country: meta.options.location?.country ?? undefined,
+ isMobile: meta.options.mobile ?? false,
+ },
+ meta.logger,
+ );
+
meta.logger.debug("Sent document to search service", {
url: meta.url,
});
@@ -156,7 +156,6 @@ export async function sendDocumentToSearchIndex(
});
}
})();
-
+
return document;
}
-