Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(index): save contentType to index
Gergő Móricz5e14f62933fe
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex b20250320..789e0b637 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -54,6 +54,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {error: document.metadata.error,screenshot: document.screenshot,numPages: document.metadata.numPages,+ contentType: document.metadata.contentType,});} catch (error) {meta.logger.error("Failed to save document to index", {@@ -232,7 +233,8 @@ export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefinerror: doc.error,screenshot: doc.screenshot,numPages: doc.numPages,-+ contentType: doc.contentType,+cacheInfo: {created_at: new Date(data[0].created_at),},diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 2f18d91be..4ede28992 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -98,6 +98,7 @@ export async function saveIndexToGCS(id: string, doc: {error?: string;screenshot?: string;numPages?: number;+ contentType?: string;}): Promise<void> {try {if (!process.env.GCS_INDEX_BUCKET_NAME) {