Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: url normalization + max metadata size
Nicolas7a31306be591
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex d47692839..20214d72a 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -17,7 +17,6 @@ import expressWs from "express-ws";import { ErrorResponse, ResponseWithSentry } from "./controllers/v1/types";import { ZodError } from "zod";import { v4 as uuidv4 } from "uuid";-import { searchSimilarPages } from "./lib/extract/index/pinecone";const { createBullBoard } = require("@bull-board/api");const { BullAdapter } = require("@bull-board/api/bullAdapter");@@ -255,3 +254,4 @@ logger.info(`Worker ${process.pid} started`);// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));+//diff --git a/apps/api/src/lib/extract/index/pinecone.ts b/apps/api/src/lib/extract/index/pinecone.tsindex 603cd38f1..14c3bea40 100644--- a/apps/api/src/lib/extract/index/pinecone.ts+++ b/apps/api/src/lib/extract/index/pinecone.ts@@ -13,6 +13,8 @@ const pinecone = new Pinecone({const INDEX_NAME = process.env.PINECONE_INDEX_NAME ?? "";+const MAX_METADATA_SIZE = 30 * 1024; // 30KB in bytes+export interface PageMetadata {url: string;originUrl: string;@@ -42,40 +44,54 @@ function normalizeUrl(url: string) {return urlO.href;}-export async function indexPage(- document: Document,- originUrl: string,- crawlId?: string,- teamId?: string+export async function indexPage({+ document,+ originUrl,+ crawlId,+ teamId+}: {+ document: Document;+ originUrl: string;+ crawlId?: string;+ teamId?: string;+}) {try {const index = pinecone.index(INDEX_NAME);+ // Trim markdown if it's too long+ let trimmedMarkdown = document.markdown;+ if (trimmedMarkdown && Buffer.byteLength(trimmedMarkdown, 'utf-8') > MAX_METADATA_SIZE) {+ trimmedMarkdown = trimmedMarkdown.slice(0, Math.floor(MAX_METADATA_SIZE / 2)); // Using half the size to be safe with UTF-8 encoding+ }+// Create text to embedconst textToEmbed = [document.metadata.title,document.metadata.description,- document.markdown+ trimmedMarkdown].filter(Boolean).join('\n\n');// Get embedding from OpenAIconst embedding = await getEmbedding(textToEmbed);+ const normalizedUrl = normalizeUrl(document.metadata.sourceURL || document.metadata.url!);+// Prepare metadataconst metadata: PageMetadata = {- url: normalizeUrl(document.metadata.sourceURL || document.metadata.url!),+ url: normalizedUrl,originUrl: normalizeUrl(originUrl),title: document.metadata.title,description: document.metadata.description,crawlId,teamId,- markdown: document.markdown,+ markdown: trimmedMarkdown,timestamp: Date.now()};// Upsert to Pineconeawait index.upsert([{- id: document.metadata.sourceURL || document.metadata.url!,+ id: normalizedUrl,values: embedding,metadata: {...metadata,@@ -114,10 +130,11 @@ export async function searchSimilarPages(includeMetadata: true};+ const normalizedOriginUrl = originUrl ? normalizeUrl(originUrl) : undefined;// Add filter if originUrl is provided- if (originUrl) {+ if (normalizedOriginUrl) {queryParams.filter = {- [originUrl]: { $contains: normalizeUrl(originUrl) }+ originUrl: { $eq: normalizedOriginUrl }};}