Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolas7a4f33a0b314
diff --git a/apps/api/src/lib/search-index/chunker.ts b/apps/api/src/lib/search-index/chunker.ts
index 5e1749c90..4c8512c4b 100644
--- a/apps/api/src/lib/search-index/chunker.ts
+++ b/apps/api/src/lib/search-index/chunker.ts
@@ -191,8 +191,10 @@ export async function chunkText(
endOffset: sentenceChunkOffset + chunkLength,
});
- // Update offset for next chunk
- sentenceChunkOffset += chunkLength;
+ // Update offset for next chunk - account for overlap
+ // The next chunk will start with overlapText, so its offset should point
+ // to where the overlap begins in the source (not where previous chunk ended)
+ sentenceChunkOffset += chunkLength - overlapText.length;
// Start new chunk with overlap from previous chunk + new sentence
sentenceChunk = overlapText + " " + sentence;
@@ -279,11 +281,14 @@ export async function chunkText(
endOffset: currentOffset + chunkLength,
});
- // Update offset for next chunk
- currentOffset += chunkLength;
-
// Add overlap for context
const lastSentences = currentChunk.split(/[.!?]+\s+/).slice(-2).join(". ");
+
+ // Update offset for next chunk - account for overlap
+ // The next chunk will start with lastSentences, so its offset should point
+ // to where the overlap begins in the source (not where previous chunk ended)
+ currentOffset += chunkLength - lastSentences.length;
+
currentChunk = lastSentences + " " + sentence;
currentTokens = estimateTokenCount(currentChunk);
} else {
diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.ts
index de00de720..fe2b943e2 100644
--- a/apps/api/src/lib/search-index/service.ts
+++ b/apps/api/src/lib/search-index/service.ts
@@ -469,13 +469,16 @@ export async function indexDocumentForSearch(
/**
* Normalize URL for search index (remove query params, fragments, etc.)
+ * Preserves the original protocol (http/https) to avoid collisions and maintain reachability
*/
function normalizeSearchURL(url: string): string {
try {
const urlObj = new URL(url);
urlObj.hash = "";
urlObj.search = ""; // Remove query params for canonical URL
- urlObj.protocol = "https:";
+ // Note: Preserve original protocol (don't force https) to avoid:
+ // 1. Making HTTP-only pages unreachable
+ // 2. Collisions between distinct HTTP/HTTPS resources
if (urlObj.hostname.startsWith("www.")) {
urlObj.hostname = urlObj.hostname.slice(4);