Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixes
Nicolasd10b0956cf96
diff --git a/apps/api/src/controllers/v2/f-search.ts b/apps/api/src/controllers/v2/f-search.tsindex 482390dbf..b791c02a0 100644--- a/apps/api/src/controllers/v2/f-search.ts+++ b/apps/api/src/controllers/v2/f-search.ts@@ -114,7 +114,8 @@ export async function realtimeSearchController(pagination: {limit,offset,- hasMore: result.total > offset + limit,+ // hasMore is true if we got a full page of results, suggesting there may be more+ hasMore: result.results.length >= limit,},},costTracking: costTracking.toJSON(),diff --git a/apps/api/src/lib/search-index/__tests__/chunker.test.ts b/apps/api/src/lib/search-index/__tests__/chunker.test.tsnew file mode 100644index 000000000..1b2f8c26b--- /dev/null+++ b/apps/api/src/lib/search-index/__tests__/chunker.test.ts@@ -0,0 +1,474 @@+import {+ chunkText,+ cleanMarkdownForIndexing,+ estimateTokenCount,+ type TextChunk,+ type ChunkingOptions,+} from "../chunker";++describe("Chunker", () => {+ describe("estimateTokenCount", () => {+ it("should estimate token count for simple text", () => {+ const text = "Hello world";+ const tokens = estimateTokenCount(text);+ expect(tokens).toBeGreaterThan(0);+ expect(tokens).toBeLessThan(10);+ });++ it("should estimate token count for longer text", () => {+ const text = "This is a longer piece of text that should have more tokens than the previous example.";+ const tokens = estimateTokenCount(text);+ expect(tokens).toBeGreaterThan(10);+ expect(tokens).toBeLessThan(50);+ });++ it("should handle empty string", () => {+ const tokens = estimateTokenCount("");+ expect(tokens).toBe(0);+ });++ it("should normalize whitespace before counting", () => {+ const text1 = "Hello world";+ const text2 = "Hello world";+ expect(estimateTokenCount(text1)).toBe(estimateTokenCount(text2));+ });+ });++ describe("cleanMarkdownForIndexing", () => {+ it("should remove code blocks", () => {+ const markdown = "Some text\n```javascript\nconst x = 1;\n```\nMore text";+ const cleaned = cleanMarkdownForIndexing(markdown);+ expect(cleaned).not.toContain("```");+ expect(cleaned).toContain("Some text");+ expect(cleaned).toContain("More text");+ });++ it("should remove inline code but keep content", () => {+ const markdown = "This is `inline code` in text";+ const cleaned = cleanMarkdownForIndexing(markdown);+ expect(cleaned).not.toContain("`");+ expect(cleaned).toContain("inline code");+ });++ it("should remove images but keep alt text", () => {+ const markdown = "Check this  out";+ const cleaned = cleanMarkdownForIndexing(markdown);+ expect(cleaned).toContain("alt text");+ expect(cleaned).not.toContain(";+ });++ it("should remove links but keep text", () => {+ const markdown = "Visit [Google](https://google.com) for search";+ const cleaned = cleanMarkdownForIndexing(markdown);+ expect(cleaned).toContain("Google");+ expect(cleaned).not.toContain("[");+ expect(cleaned).not.toContain("](");+ });++ it("should remove headers but keep text", () => {+ const markdown = "# Heading 1\n## Heading 2\n### Heading 3";+ const cleaned = cleanMarkdownForIndexing(markdown);+ expect(cleaned).toContain("Heading 1");+ expect(cleaned).toContain("Heading 2");+ expect(cleaned).toContain("Heading 3");+ expect(cleaned).not.toContain("#");+ });++ it("should remove bold and italic markers", () => {+ const markdown = "This is **bold** and *italic* text";+ const cleaned = cleanMarkdownForIndexing(markdown);+ expect(cleaned).toContain("bold");+ expect(cleaned).toContain("italic");+ expect(cleaned).not.toContain("**");+ expect(cleaned).not.toContain("*");+ });++ it("should remove list markers", () => {+ const markdown = "- Item 1\n- Item 2\n1. Ordered item";+ const cleaned = cleanMarkdownForIndexing(markdown);+ expect(cleaned).toContain("Item 1");+ expect(cleaned).toContain("Item 2");+ expect(cleaned).toContain("Ordered item");+ expect(cleaned).not.toMatch(/^[-*+]\s/m);+ expect(cleaned).not.toMatch(/^\d+\.\s/m);+ });++ it("should handle empty input", () => {+ expect(cleanMarkdownForIndexing("")).toBe("");+ expect(cleanMarkdownForIndexing(null as any)).toBe("");+ });++ it("should normalize excessive whitespace", () => {+ const markdown = "Text\n\n\n\nwith\n\n\n\nmany\n\n\n\nlines";+ const cleaned = cleanMarkdownForIndexing(markdown);+ expect(cleaned).not.toMatch(/\n{3,}/);+ });+ });++ describe("chunkText", () => {+ describe("basic functionality", () => {+ it("should return empty array for empty text", async () => {+ const chunks = await chunkText("");+ expect(chunks).toEqual([]);+ });++ it("should return single chunk for short text", async () => {+ const text = "This is a short piece of text.";+ const chunks = await chunkText(text);+ expect(chunks).toHaveLength(1);+ expect(chunks[0].text).toBe(text.trim());+ expect(chunks[0].ordinal).toBe(0);+ });++ it("should split long text into multiple chunks", async () => {+ // Create a text that exceeds max tokens+ const longText = Array(200)+ .fill("This is a sentence with multiple words.")+ .join(" ");+ const chunks = await chunkText(longText, { maxTokens: 500 });+ expect(chunks.length).toBeGreaterThan(1);+ });++ it("should respect maxTokens limit", async () => {+ const longText = Array(100)+ .fill("This is a sentence.")+ .join(" ");+ const maxTokens = 300;+ const chunks = await chunkText(longText, { maxTokens });++ chunks.forEach(chunk => {+ expect(chunk.tokenCount).toBeLessThanOrEqual(maxTokens);+ });+ });+ });++ describe("chunk metadata", () => {+ it("should set correct ordinals", async () => {+ const longText = Array(100)+ .fill("This is a sentence.")+ .join(" ");+ const chunks = await chunkText(longText, { maxTokens: 300 });++ chunks.forEach((chunk, index) => {+ expect(chunk.ordinal).toBe(index);+ });+ });++ it("should calculate token counts", async () => {+ const text = "This is test text.";+ const chunks = await chunkText(text);++ expect(chunks[0].tokenCount).toBeGreaterThan(0);+ expect(chunks[0].tokenCount).toBe(estimateTokenCount(text.trim()));+ });++ it("should calculate character counts", async () => {+ const text = "This is test text.";+ const chunks = await chunkText(text);++ expect(chunks[0].charCount).toBe(text.length);+ });++ it("should set start and end offsets", async () => {+ const text = "First sentence. Second sentence.";+ const chunks = await chunkText(text);++ expect(chunks[0].startOffset).toBe(0);+ expect(chunks[0].endOffset).toBeGreaterThan(0);+ expect(chunks[0].endOffset).toBeLessThanOrEqual(text.length);+ });+ });++ describe("offset tracking", () => {+ it("should track offsets correctly for multiple chunks", async () => {+ const longText = Array(100)+ .fill("This is a sentence with several words.")+ .join(" ");+ const chunks = await chunkText(longText, { maxTokens: 300 });++ // First chunk should start at 0+ expect(chunks[0].startOffset).toBe(0);++ // Each chunk's offsets should be sequential (allowing for overlap)+ for (let i = 1; i < chunks.length; i++) {+ expect(chunks[i].startOffset).toBeGreaterThanOrEqual(0);+ expect(chunks[i].endOffset).toBeGreaterThan(chunks[i].startOffset);+ }+ });++ it("should maintain consistent offsets in structure-preserving mode", async () => {+ const text = "# Heading\n\nParagraph one. " ++ Array(50).fill("More text.").join(" ") ++ "\n\n# Another Heading\n\nParagraph two.";++ const chunks = await chunkText(text, {+ maxTokens: 300,+ preserveStructure: true+ });++ chunks.forEach(chunk => {+ expect(chunk.startOffset).toBeGreaterThanOrEqual(0);+ expect(chunk.endOffset).toBeGreaterThan(chunk.startOffset);+ expect(chunk.endOffset).toBeLessThanOrEqual(text.length + 100); // Allow some margin for processing+ });+ });++ it("should maintain consistent offsets in sentence-based mode", async () => {+ const text = Array(80)+ .fill("This is a sentence.")+ .join(" ");++ const chunks = await chunkText(text, {+ maxTokens: 300,+ preserveStructure: false+ });++ chunks.forEach(chunk => {+ expect(chunk.startOffset).toBeGreaterThanOrEqual(0);+ expect(chunk.endOffset).toBeGreaterThan(chunk.startOffset);+ });+ });+ });++ describe("chunking strategies", () => {+ it("should preserve markdown structure when enabled", async () => {+ const text =+ "# Heading 1\n\nFirst paragraph.\n\n" ++ "## Heading 2\n\nSecond paragraph with more content.\n\n" ++ "- List item 1\n- List item 2\n\n" ++ "Final paragraph.";++ const chunks = await chunkText(text, {+ preserveStructure: true,+ maxTokens: 200+ });++ expect(chunks.length).toBeGreaterThan(0);+ // Structure preservation should try to keep sections together+ expect(chunks.some(c => c.text.includes("Heading"))).toBe(true);+ });++ it("should use sentence-based chunking when structure preservation is disabled", async () => {+ const text = Array(50)+ .fill("This is a sentence. Another sentence here.")+ .join(" ");++ const chunks = await chunkText(text, {+ preserveStructure: false,+ maxTokens: 300+ });++ expect(chunks.length).toBeGreaterThan(1);+ chunks.forEach(chunk => {+ expect(chunk.text.length).toBeGreaterThan(0);+ });+ });++ it("should handle large sections that need splitting", async () => {+ const largeSection = Array(100)+ .fill("Sentence in a large section.")+ .join(" ");+ const text = `# Large Section\n\n${largeSection}`;++ const chunks = await chunkText(text, {+ preserveStructure: true,+ maxTokens: 300+ });++ expect(chunks.length).toBeGreaterThan(1);+ });+ });++ describe("overlap handling", () => {+ it("should create overlapping chunks", async () => {+ const longText = Array(80)+ .fill("This is a unique sentence number.")+ .join(" ");++ const chunks = await chunkText(longText, {+ maxTokens: 300,+ overlapTokens: 50+ });++ if (chunks.length > 1) {+ // Chunks should have some overlap in content+ expect(chunks.length).toBeGreaterThan(1);+ // Each chunk should exist and have content+ chunks.forEach(chunk => {+ expect(chunk.text.length).toBeGreaterThan(0);+ });+ }+ });++ it("should apply overlap correctly with proper context", async () => {+ const sentences = Array(60)+ .fill(0)+ .map((_, i) => `Sentence number ${i}.`)+ .join(" ");++ const chunks = await chunkText(sentences, {+ maxTokens: 200,+ overlapTokens: 30+ });++ if (chunks.length > 1) {+ // Verify chunks are created with proper content+ chunks.forEach((chunk, i) => {+ expect(chunk.ordinal).toBe(i);+ expect(chunk.text.length).toBeGreaterThan(0);+ });+ }+ });+ });++ describe("edge cases", () => {+ it("should handle text with only whitespace", async () => {+ const chunks = await chunkText(" \n\n ");+ expect(chunks).toEqual([]);+ });++ it("should handle text with special characters", async () => {+ const text = "Text with émojis 🔥 and spëcial çharacters!";+ const chunks = await chunkText(text);++ expect(chunks).toHaveLength(1);+ expect(chunks[0].text).toContain("émojis");+ expect(chunks[0].text).toContain("🔥");+ });++ it("should handle very long single sentences", async () => {+ // A very long single sentence cannot be split at sentence boundaries+ // The chunker will keep it as one chunk even if it exceeds maxTokens+ const longSentence = "This is a very long sentence that " ++ Array(200).fill("keeps going and going").join(" ") ++ ".";++ const chunks = await chunkText(longSentence, { maxTokens: 500 });++ // Should create at least one chunk+ expect(chunks.length).toBeGreaterThan(0);+ // When a single sentence is too long, it stays as one chunk+ // This is expected behavior to maintain sentence integrity+ expect(chunks[0].text).toContain("very long sentence");+ });++ it("should handle text with multiple consecutive newlines", async () => {+ const text = "First paragraph.\n\n\n\nSecond paragraph.";+ const chunks = await chunkText(text);++ expect(chunks.length).toBeGreaterThan(0);+ expect(chunks[0].text).toContain("First paragraph");+ });++ it("should handle mixed content types", async () => {+ const text =+ "# Heading\n\n" ++ "Paragraph with text.\n\n" ++ "```code\nconst x = 1;\n```\n\n" ++ "- List item\n" ++ "- Another item\n\n" ++ "Final paragraph.";++ const chunks = await chunkText(text);+ expect(chunks.length).toBeGreaterThan(0);+ });+ });++ describe("options validation", () => {+ it("should use default options when none provided", async () => {+ const text = Array(50).fill("Text here.").join(" ");+ const chunks = await chunkText(text);++ expect(chunks).toBeDefined();+ expect(Array.isArray(chunks)).toBe(true);+ });++ it("should respect custom targetTokens", async () => {+ const text = Array(100).fill("Text here.").join(" ");+ const chunks = await chunkText(text, { targetTokens: 200 });++ expect(chunks.length).toBeGreaterThan(0);+ });++ it("should respect custom minTokens", async () => {+ const text = Array(100).fill("Text here.").join(" ");+ const chunks = await chunkText(text, { minTokens: 100 });++ expect(chunks.length).toBeGreaterThan(0);+ });++ it("should respect custom overlapTokens", async () => {+ const text = Array(100).fill("Text here.").join(" ");+ const chunks = await chunkText(text, {+ maxTokens: 300,+ overlapTokens: 75+ });++ expect(chunks.length).toBeGreaterThan(0);+ });+ });++ describe("chunk quality", () => {+ it("should not create empty chunks", async () => {+ const text = Array(100).fill("Content here.").join(" ");+ const chunks = await chunkText(text, { maxTokens: 300 });++ chunks.forEach(chunk => {+ expect(chunk.text.trim().length).toBeGreaterThan(0);+ expect(chunk.tokenCount).toBeGreaterThan(0);+ expect(chunk.charCount).toBeGreaterThan(0);+ });+ });++ it("should maintain text integrity", async () => {+ const text = "This is important content that must be preserved.";+ const chunks = await chunkText(text);++ const reconstructed = chunks.map(c => c.text).join(" ");+ expect(reconstructed).toContain("important content");+ });++ it("should create reasonably sized chunks", async () => {+ const text = Array(100).fill("Some text.").join(" ");+ const maxTokens = 400;+ const chunks = await chunkText(text, { maxTokens });++ chunks.forEach(chunk => {+ expect(chunk.tokenCount).toBeGreaterThan(0);+ expect(chunk.tokenCount).toBeLessThanOrEqual(maxTokens);+ });+ });+ });++ describe("performance", () => {+ it("should handle large documents efficiently", async () => {+ const largeText = Array(500)+ .fill("This is a sentence in a large document.")+ .join(" ");++ const startTime = Date.now();+ const chunks = await chunkText(largeText, { maxTokens: 500 });+ const duration = Date.now() - startTime;++ expect(chunks.length).toBeGreaterThan(0);+ expect(duration).toBeLessThan(5000); // Should complete in under 5 seconds+ });++ it("should handle documents with complex structure", async () => {+ const sections = Array(20).fill(0).map((_, i) =>+ `# Section ${i}\n\nContent for section ${i}.\n\n` ++ Array(10).fill("More text here.").join(" ")+ ).join("\n\n");++ const chunks = await chunkText(sections, {+ preserveStructure: true,+ maxTokens: 500+ });++ expect(chunks.length).toBeGreaterThan(0);+ });+ });+ });+});+diff --git a/apps/api/src/lib/search-index/chunker.ts b/apps/api/src/lib/search-index/chunker.tsindex 587569e22..5e1749c90 100644--- a/apps/api/src/lib/search-index/chunker.ts+++ b/apps/api/src/lib/search-index/chunker.ts@@ -172,6 +172,7 @@ export async function chunkText(const sentences = splitIntoSentences(section.text);let sentenceChunk = "";let sentenceTokens = 0;+ let sentenceChunkOffset = section.offset;for (const sentence of sentences) {const sentenceTokenCount = estimateTokenCount(sentence);@@ -179,16 +180,20 @@ export async function chunkText(if (sentenceTokens + sentenceTokenCount > opts.maxTokens && sentenceChunk.length > 0) {// Save overlap from the previous chunk before flushingconst overlapText = sentenceChunk.slice(-opts.overlapTokens * 4); // Rough char estimate+ const chunkLength = sentenceChunk.length;chunks.push({text: sentenceChunk.trim(),ordinal: chunks.length,tokenCount: sentenceTokens,- charCount: sentenceChunk.length,- startOffset: section.offset,- endOffset: section.offset + sentenceChunk.length,+ charCount: chunkLength,+ startOffset: sentenceChunkOffset,+ endOffset: sentenceChunkOffset + chunkLength,});+ // Update offset for next chunk+ sentenceChunkOffset += chunkLength;+// Start new chunk with overlap from previous chunk + new sentencesentenceChunk = overlapText + " " + sentence;sentenceTokens = estimateTokenCount(sentenceChunk);@@ -204,8 +209,8 @@ export async function chunkText(ordinal: chunks.length,tokenCount: sentenceTokens,charCount: sentenceChunk.length,- startOffset: section.offset,- endOffset: section.offset + sentenceChunk.length,+ startOffset: sentenceChunkOffset,+ endOffset: sentenceChunkOffset + sentenceChunk.length,});}@@ -264,15 +269,19 @@ export async function chunkText(const sentenceTokens = estimateTokenCount(sentence);if (currentTokens + sentenceTokens > opts.maxTokens && currentChunk.length > 0) {+ const chunkLength = currentChunk.length;chunks.push({text: currentChunk.trim(),ordinal: chunks.length,tokenCount: currentTokens,- charCount: currentChunk.length,+ charCount: chunkLength,startOffset: currentOffset,- endOffset: currentOffset + currentChunk.length,+ endOffset: currentOffset + chunkLength,});+ // Update offset for next chunk+ currentOffset += chunkLength;+// Add overlap for contextconst lastSentences = currentChunk.split(/[.!?]+\s+/).slice(-2).join(". ");currentChunk = lastSentences + " " + sentence;@@ -295,7 +304,7 @@ export async function chunkText(tokenCount: currentTokens,charCount: currentChunk.length,startOffset: currentOffset,- endOffset: text.length,+ endOffset: currentOffset + currentChunk.length,});}}diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.tsindex a8e24aa60..de00de720 100644--- a/apps/api/src/lib/search-index/service.ts+++ b/apps/api/src/lib/search-index/service.ts@@ -123,6 +123,7 @@ export async function indexDocumentForSearch(let chunkEmbeddings: number[][] = [];let totalTokens = 0;let pineconeRecords: PineconeRecord[] = [];+ let embeddingsActuallyGenerated = false;if (embeddingsEnabled) {try {@@ -185,6 +186,9 @@ export async function indexDocumentForSearch(}});+ // Mark as successfully generated+ embeddingsActuallyGenerated = true;+setSpanAttributes(span, {"search.embeddings_generated": true,"search.embedding_tokens": totalTokens,@@ -199,6 +203,8 @@ export async function indexDocumentForSearch("search.embeddings_error": true,"search.embeddings_error_message": (error as Error).message,});+ // Reset flag since embeddings failed+ embeddingsActuallyGenerated = false;}}@@ -415,7 +421,7 @@ export async function indexDocumentForSearch(await updateSyncState(supabase, {documentsIndexed: 1,chunksIndexed: Math.min(chunks.length, 2), // Only inserted first 2- embeddingsGenerated: embeddingsEnabled ? 1 + chunks.length : 0,+ embeddingsGenerated: embeddingsActuallyGenerated ? 1 + chunks.length : 0,});log.info("Document indexed successfully", {@@ -423,7 +429,7 @@ export async function indexDocumentForSearch(documentId,chunks: chunks.length,totalTokens,- embeddingsGenerated: embeddingsEnabled,+ embeddingsGenerated: embeddingsActuallyGenerated,isUpdate,});@@ -437,7 +443,7 @@ export async function indexDocumentForSearch(documentId,chunkCount: chunks.length,totalTokens,- embeddingsGenerated: embeddingsEnabled,+ embeddingsGenerated: embeddingsActuallyGenerated,};} catch (error) {log.error("Failed to index document", {diff --git a/apps/api/src/services/search-index-db.ts b/apps/api/src/services/search-index-db.tsindex 6db8a2444..8b199f323 100644--- a/apps/api/src/services/search-index-db.ts+++ b/apps/api/src/services/search-index-db.ts@@ -59,14 +59,12 @@ export const search_index_supabase_service: SupabaseClient = new Proxy(get: function (target, prop, receiver) {const client = target.getClient();- // If client is not initialized, provide meaningful error+ // If client is not initialized, throw immediately to prevent nested access errorsif (client === null) {- return () => {- throw new Error(- "Search index database is not configured. " +- "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN environment variables."- );- };+ throw new Error(+ "Search index database is not configured. " ++ "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN environment variables."+ );}// Direct access to service properties