Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: gcs index id fix

Nicolasa7669cccb98f
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 4455e3b64..fc7c15109 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1018,6 +1018,7 @@ export type Document = {
cachedAt?: string;
creditsUsed?: number;
postprocessorsUsed?: string[];
+ indexId?: string; // ID used to store the document in the index (GCS)
// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
};
serpResults?: {
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index c9adb71ed..deb59f7ec 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -924,6 +924,7 @@ export type Document = {
cachedAt?: string;
creditsUsed?: number;
postprocessorsUsed?: string[];
+ indexId?: string; // ID used to store the document in the index (GCS)
// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
};
serpResults?: {
diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.ts
index fe2b943e2..32e07e83a 100644
--- a/apps/api/src/lib/search-index/service.ts
+++ b/apps/api/src/lib/search-index/service.ts
@@ -306,6 +306,7 @@ export async function indexDocumentForSearch(
description: input.description?.slice(0, 500) ?? null,
language: input.language ?? "en",
gcs_path: input.gcsPath ?? null,
+ gcp_path_id: input.gcsPath ?? null,
screenshot_url: input.screenshotUrl ?? null,
content_ts: contentTsVector,
domain,
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index b104a8d91..7327a92f4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -84,6 +84,9 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
contentType: document.metadata.contentType,
postprocessorsUsed: document.metadata.postprocessorsUsed,
});
+
+ // Store the indexId in document metadata so it can be used by other transformers (e.g., search index)
+ document.metadata.indexId = indexId;
} catch (error) {
meta.logger.error("Failed to save document to index", {
error,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
index 4fa44075b..e71c7e868 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
@@ -112,6 +112,13 @@ export async function sendDocumentToSearchIndex(
return document;
}
+ // Get the indexId from document metadata (set by sendDocumentToIndex transformer)
+ // Format as GCS path: {indexId}.json
+ const gcsPath = document.metadata.indexId
+
+ // Remove indexId from metadata after extracting it (internal field, shouldn't be exposed to user)
+ delete document.metadata.indexId;
+
// Queue job for async processing (don't block scraper)
(async () => {
try {
@@ -131,7 +138,7 @@ export async function sendDocumentToSearchIndex(
markdown: document.markdown ?? "",
html: document.rawHtml ?? "",
statusCode: document.metadata.statusCode,
- gcsPath: undefined, // Can be populated from GCS if needed
+ gcsPath: gcsPath,
screenshotUrl: document.screenshot ?? undefined,
language: document.metadata.language ?? "en",
country: meta.options.location?.country ?? undefined,