Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas7a4f33a0b314
diff --git a/apps/api/src/lib/search-index/chunker.ts b/apps/api/src/lib/search-index/chunker.tsindex 5e1749c90..4c8512c4b 100644--- a/apps/api/src/lib/search-index/chunker.ts+++ b/apps/api/src/lib/search-index/chunker.ts@@ -191,8 +191,10 @@ export async function chunkText(endOffset: sentenceChunkOffset + chunkLength,});- // Update offset for next chunk- sentenceChunkOffset += chunkLength;+ // Update offset for next chunk - account for overlap+ // The next chunk will start with overlapText, so its offset should point+ // to where the overlap begins in the source (not where previous chunk ended)+ sentenceChunkOffset += chunkLength - overlapText.length;// Start new chunk with overlap from previous chunk + new sentencesentenceChunk = overlapText + " " + sentence;@@ -279,11 +281,14 @@ export async function chunkText(endOffset: currentOffset + chunkLength,});- // Update offset for next chunk- currentOffset += chunkLength;-// Add overlap for contextconst lastSentences = currentChunk.split(/[.!?]+\s+/).slice(-2).join(". ");++ // Update offset for next chunk - account for overlap+ // The next chunk will start with lastSentences, so its offset should point+ // to where the overlap begins in the source (not where previous chunk ended)+ currentOffset += chunkLength - lastSentences.length;+currentChunk = lastSentences + " " + sentence;currentTokens = estimateTokenCount(currentChunk);} else {diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.tsindex de00de720..fe2b943e2 100644--- a/apps/api/src/lib/search-index/service.ts+++ b/apps/api/src/lib/search-index/service.ts@@ -469,13 +469,16 @@ export async function indexDocumentForSearch(/*** Normalize URL for search index (remove query params, fragments, etc.)+ * Preserves the original protocol (http/https) to avoid collisions and maintain reachability*/function normalizeSearchURL(url: string): string {try {const urlObj = new URL(url);urlObj.hash = "";urlObj.search = ""; // Remove query params for canonical URL- urlObj.protocol = "https:";+ // Note: Preserve original protocol (don't force https) to avoid:+ // 1. Making HTTP-only pages unreachable+ // 2. Collisions between distinct HTTP/HTTPS resourcesif (urlObj.hostname.startsWith("www.")) {urlObj.hostname = urlObj.hostname.slice(4);