Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: gcs index id fix
Nicolasa7669cccb98f
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 4455e3b64..fc7c15109 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1018,6 +1018,7 @@ export type Document = {cachedAt?: string;creditsUsed?: number;postprocessorsUsed?: string[];+ indexId?: string; // ID used to store the document in the index (GCS)// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;};serpResults?: {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex c9adb71ed..deb59f7ec 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -924,6 +924,7 @@ export type Document = {cachedAt?: string;creditsUsed?: number;postprocessorsUsed?: string[];+ indexId?: string; // ID used to store the document in the index (GCS)// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;};serpResults?: {diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.tsindex fe2b943e2..32e07e83a 100644--- a/apps/api/src/lib/search-index/service.ts+++ b/apps/api/src/lib/search-index/service.ts@@ -306,6 +306,7 @@ export async function indexDocumentForSearch(description: input.description?.slice(0, 500) ?? null,language: input.language ?? "en",gcs_path: input.gcsPath ?? null,+ gcp_path_id: input.gcsPath ?? null,screenshot_url: input.screenshotUrl ?? null,content_ts: contentTsVector,domain,diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex b104a8d91..7327a92f4 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -84,6 +84,9 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {contentType: document.metadata.contentType,postprocessorsUsed: document.metadata.postprocessorsUsed,});++ // Store the indexId in document metadata so it can be used by other transformers (e.g., search index)+ document.metadata.indexId = indexId;} catch (error) {meta.logger.error("Failed to save document to index", {error,diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.tsindex 4fa44075b..e71c7e868 100644--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts@@ -112,6 +112,13 @@ export async function sendDocumentToSearchIndex(return document;}+ // Get the indexId from document metadata (set by sendDocumentToIndex transformer)+ // Format as GCS path: {indexId}.json+ const gcsPath = document.metadata.indexId++ // Remove indexId from metadata after extracting it (internal field, shouldn't be exposed to user)+ delete document.metadata.indexId;+// Queue job for async processing (don't block scraper)(async () => {try {@@ -131,7 +138,7 @@ export async function sendDocumentToSearchIndex(markdown: document.markdown ?? "",html: document.rawHtml ?? "",statusCode: document.metadata.statusCode,- gcsPath: undefined, // Can be populated from GCS if needed+ gcsPath: gcsPath,screenshotUrl: document.screenshot ?? undefined,language: document.metadata.language ?? "en",country: meta.options.location?.country ?? undefined,