Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixes
Nicolas69aa5e7dcd34
diff --git a/apps/api/src/lib/search-index/chunker.ts b/apps/api/src/lib/search-index/chunker.tsindex 71241a477..587569e22 100644--- a/apps/api/src/lib/search-index/chunker.ts+++ b/apps/api/src/lib/search-index/chunker.ts@@ -177,6 +177,9 @@ export async function chunkText(const sentenceTokenCount = estimateTokenCount(sentence);if (sentenceTokens + sentenceTokenCount > opts.maxTokens && sentenceChunk.length > 0) {+ // Save overlap from the previous chunk before flushing+ const overlapText = sentenceChunk.slice(-opts.overlapTokens * 4); // Rough char estimate+chunks.push({text: sentenceChunk.trim(),ordinal: chunks.length,@@ -186,8 +189,7 @@ export async function chunkText(endOffset: section.offset + sentenceChunk.length,});- // Add overlap- const overlapText = sentence.slice(-opts.overlapTokens * 4); // Rough char estimate+ // Start new chunk with overlap from previous chunk + new sentencesentenceChunk = overlapText + " " + sentence;sentenceTokens = estimateTokenCount(sentenceChunk);} else {diff --git a/apps/api/src/lib/search-index/pinecone-service.ts b/apps/api/src/lib/search-index/pinecone-service.tsindex 661ad3307..c2809855e 100644--- a/apps/api/src/lib/search-index/pinecone-service.ts+++ b/apps/api/src/lib/search-index/pinecone-service.ts@@ -46,12 +46,10 @@ export function getPineconeIndex() {/*** Check if Pinecone is enabled+ * Note: PINECONE_INDEX_NAME is optional and defaults to "firecrawl-search" in getPineconeIndex()*/export function isPineconeEnabled(): boolean {- return !!(- process.env.PINECONE_API_KEY &&- process.env.PINECONE_INDEX_NAME- );+ return !!process.env.PINECONE_API_KEY;}export interface PineconeRecord {diff --git a/apps/api/src/lib/search-index/query.ts b/apps/api/src/lib/search-index/query.tsindex 8c1f60d09..2c9bff4ee 100644--- a/apps/api/src/lib/search-index/query.ts+++ b/apps/api/src/lib/search-index/query.ts@@ -85,21 +85,24 @@ export async function search(};}- try {- let results: SearchResult[] = [];-- // Choose search strategy based on mode- if (mode === "keyword") {- results = await keywordSearch(supabase, query, limit, filters, log);- } else if (mode === "semantic") {- results = await semanticSearch(supabase, query, limit, filters, log);- } else {- // Hybrid: combine BM25 + vector with RRF- results = await hybridSearch(supabase, query, limit, filters, log);- }-- // Apply offset- const paginatedResults = results.slice(offset, offset + limit);+ try {+ let results: SearchResult[] = [];++ // Fetch enough results to support pagination+ const fetchLimit = limit + offset;++ // Choose search strategy based on mode+ if (mode === "keyword") {+ results = await keywordSearch(supabase, query, fetchLimit, filters, log);+ } else if (mode === "semantic") {+ results = await semanticSearch(supabase, query, fetchLimit, filters, log);+ } else {+ // Hybrid: combine BM25 + vector with RRF+ results = await hybridSearch(supabase, query, fetchLimit, filters, log);+ }++ // Apply offset and limit for pagination+ const paginatedResults = results.slice(offset, offset + limit);const took = Date.now() - startTime;