Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: fixes

Nicolas69aa5e7dcd34
diff --git a/apps/api/src/lib/search-index/chunker.ts b/apps/api/src/lib/search-index/chunker.ts
index 71241a477..587569e22 100644
--- a/apps/api/src/lib/search-index/chunker.ts
+++ b/apps/api/src/lib/search-index/chunker.ts
@@ -177,6 +177,9 @@ export async function chunkText(
const sentenceTokenCount = estimateTokenCount(sentence);
if (sentenceTokens + sentenceTokenCount > opts.maxTokens && sentenceChunk.length > 0) {
+ // Save overlap from the previous chunk before flushing
+ const overlapText = sentenceChunk.slice(-opts.overlapTokens * 4); // Rough char estimate
+
chunks.push({
text: sentenceChunk.trim(),
ordinal: chunks.length,
@@ -186,8 +189,7 @@ export async function chunkText(
endOffset: section.offset + sentenceChunk.length,
});
- // Add overlap
- const overlapText = sentence.slice(-opts.overlapTokens * 4); // Rough char estimate
+ // Start new chunk with overlap from previous chunk + new sentence
sentenceChunk = overlapText + " " + sentence;
sentenceTokens = estimateTokenCount(sentenceChunk);
} else {
diff --git a/apps/api/src/lib/search-index/pinecone-service.ts b/apps/api/src/lib/search-index/pinecone-service.ts
index 661ad3307..c2809855e 100644
--- a/apps/api/src/lib/search-index/pinecone-service.ts
+++ b/apps/api/src/lib/search-index/pinecone-service.ts
@@ -46,12 +46,10 @@ export function getPineconeIndex() {
/**
* Check if Pinecone is enabled
+ * Note: PINECONE_INDEX_NAME is optional and defaults to "firecrawl-search" in getPineconeIndex()
*/
export function isPineconeEnabled(): boolean {
- return !!(
- process.env.PINECONE_API_KEY &&
- process.env.PINECONE_INDEX_NAME
- );
+ return !!process.env.PINECONE_API_KEY;
}
export interface PineconeRecord {
diff --git a/apps/api/src/lib/search-index/query.ts b/apps/api/src/lib/search-index/query.ts
index 8c1f60d09..2c9bff4ee 100644
--- a/apps/api/src/lib/search-index/query.ts
+++ b/apps/api/src/lib/search-index/query.ts
@@ -85,21 +85,24 @@ export async function search(
};
}
- try {
- let results: SearchResult[] = [];
-
- // Choose search strategy based on mode
- if (mode === "keyword") {
- results = await keywordSearch(supabase, query, limit, filters, log);
- } else if (mode === "semantic") {
- results = await semanticSearch(supabase, query, limit, filters, log);
- } else {
- // Hybrid: combine BM25 + vector with RRF
- results = await hybridSearch(supabase, query, limit, filters, log);
- }
-
- // Apply offset
- const paginatedResults = results.slice(offset, offset + limit);
+ try {
+ let results: SearchResult[] = [];
+
+ // Fetch enough results to support pagination
+ const fetchLimit = limit + offset;
+
+ // Choose search strategy based on mode
+ if (mode === "keyword") {
+ results = await keywordSearch(supabase, query, fetchLimit, filters, log);
+ } else if (mode === "semantic") {
+ results = await semanticSearch(supabase, query, fetchLimit, filters, log);
+ } else {
+ // Hybrid: combine BM25 + vector with RRF
+ results = await hybridSearch(supabase, query, fetchLimit, filters, log);
+ }
+
+ // Apply offset and limit for pagination
+ const paginatedResults = results.slice(offset, offset + limit);
const took = Date.now() - startTime;