Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolasaebc2f0c6374
diff --git a/apps/api/src/lib/search-index/__tests__/chunker.test.ts b/apps/api/src/lib/search-index/__tests__/chunker.test.ts
deleted file mode 100644
index 1b2f8c26b..000000000
--- a/apps/api/src/lib/search-index/__tests__/chunker.test.ts
+++ /dev/null
@@ -1,474 +0,0 @@
-import {
- chunkText,
- cleanMarkdownForIndexing,
- estimateTokenCount,
- type TextChunk,
- type ChunkingOptions,
-} from "../chunker";
-
-describe("Chunker", () => {
- describe("estimateTokenCount", () => {
- it("should estimate token count for simple text", () => {
- const text = "Hello world";
- const tokens = estimateTokenCount(text);
- expect(tokens).toBeGreaterThan(0);
- expect(tokens).toBeLessThan(10);
- });
-
- it("should estimate token count for longer text", () => {
- const text = "This is a longer piece of text that should have more tokens than the previous example.";
- const tokens = estimateTokenCount(text);
- expect(tokens).toBeGreaterThan(10);
- expect(tokens).toBeLessThan(50);
- });
-
- it("should handle empty string", () => {
- const tokens = estimateTokenCount("");
- expect(tokens).toBe(0);
- });
-
- it("should normalize whitespace before counting", () => {
- const text1 = "Hello world";
- const text2 = "Hello world";
- expect(estimateTokenCount(text1)).toBe(estimateTokenCount(text2));
- });
- });
-
- describe("cleanMarkdownForIndexing", () => {
- it("should remove code blocks", () => {
- const markdown = "Some text\n```javascript\nconst x = 1;\n```\nMore text";
- const cleaned = cleanMarkdownForIndexing(markdown);
- expect(cleaned).not.toContain("```");
- expect(cleaned).toContain("Some text");
- expect(cleaned).toContain("More text");
- });
-
- it("should remove inline code but keep content", () => {
- const markdown = "This is `inline code` in text";
- const cleaned = cleanMarkdownForIndexing(markdown);
- expect(cleaned).not.toContain("`");
- expect(cleaned).toContain("inline code");
- });
-
- it("should remove images but keep alt text", () => {
- const markdown = "Check this ![alt text](image.png) out";
- const cleaned = cleanMarkdownForIndexing(markdown);
- expect(cleaned).toContain("alt text");
- expect(cleaned).not.toContain("![");
- expect(cleaned).not.toContain("](");
- });
-
- it("should remove links but keep text", () => {
- const markdown = "Visit [Google](https://google.com) for search";
- const cleaned = cleanMarkdownForIndexing(markdown);
- expect(cleaned).toContain("Google");
- expect(cleaned).not.toContain("[");
- expect(cleaned).not.toContain("](");
- });
-
- it("should remove headers but keep text", () => {
- const markdown = "# Heading 1\n## Heading 2\n### Heading 3";
- const cleaned = cleanMarkdownForIndexing(markdown);
- expect(cleaned).toContain("Heading 1");
- expect(cleaned).toContain("Heading 2");
- expect(cleaned).toContain("Heading 3");
- expect(cleaned).not.toContain("#");
- });
-
- it("should remove bold and italic markers", () => {
- const markdown = "This is **bold** and *italic* text";
- const cleaned = cleanMarkdownForIndexing(markdown);
- expect(cleaned).toContain("bold");
- expect(cleaned).toContain("italic");
- expect(cleaned).not.toContain("**");
- expect(cleaned).not.toContain("*");
- });
-
- it("should remove list markers", () => {
- const markdown = "- Item 1\n- Item 2\n1. Ordered item";
- const cleaned = cleanMarkdownForIndexing(markdown);
- expect(cleaned).toContain("Item 1");
- expect(cleaned).toContain("Item 2");
- expect(cleaned).toContain("Ordered item");
- expect(cleaned).not.toMatch(/^[-*+]\s/m);
- expect(cleaned).not.toMatch(/^\d+\.\s/m);
- });
-
- it("should handle empty input", () => {
- expect(cleanMarkdownForIndexing("")).toBe("");
- expect(cleanMarkdownForIndexing(null as any)).toBe("");
- });
-
- it("should normalize excessive whitespace", () => {
- const markdown = "Text\n\n\n\nwith\n\n\n\nmany\n\n\n\nlines";
- const cleaned = cleanMarkdownForIndexing(markdown);
- expect(cleaned).not.toMatch(/\n{3,}/);
- });
- });
-
- describe("chunkText", () => {
- describe("basic functionality", () => {
- it("should return empty array for empty text", async () => {
- const chunks = await chunkText("");
- expect(chunks).toEqual([]);
- });
-
- it("should return single chunk for short text", async () => {
- const text = "This is a short piece of text.";
- const chunks = await chunkText(text);
- expect(chunks).toHaveLength(1);
- expect(chunks[0].text).toBe(text.trim());
- expect(chunks[0].ordinal).toBe(0);
- });
-
- it("should split long text into multiple chunks", async () => {
- // Create a text that exceeds max tokens
- const longText = Array(200)
- .fill("This is a sentence with multiple words.")
- .join(" ");
- const chunks = await chunkText(longText, { maxTokens: 500 });
- expect(chunks.length).toBeGreaterThan(1);
- });
-
- it("should respect maxTokens limit", async () => {
- const longText = Array(100)
- .fill("This is a sentence.")
- .join(" ");
- const maxTokens = 300;
- const chunks = await chunkText(longText, { maxTokens });
-
- chunks.forEach(chunk => {
- expect(chunk.tokenCount).toBeLessThanOrEqual(maxTokens);
- });
- });
- });
-
- describe("chunk metadata", () => {
- it("should set correct ordinals", async () => {
- const longText = Array(100)
- .fill("This is a sentence.")
- .join(" ");
- const chunks = await chunkText(longText, { maxTokens: 300 });
-
- chunks.forEach((chunk, index) => {
- expect(chunk.ordinal).toBe(index);
- });
- });
-
- it("should calculate token counts", async () => {
- const text = "This is test text.";
- const chunks = await chunkText(text);
-
- expect(chunks[0].tokenCount).toBeGreaterThan(0);
- expect(chunks[0].tokenCount).toBe(estimateTokenCount(text.trim()));
- });
-
- it("should calculate character counts", async () => {
- const text = "This is test text.";
- const chunks = await chunkText(text);
-
- expect(chunks[0].charCount).toBe(text.length);
- });
-
- it("should set start and end offsets", async () => {
- const text = "First sentence. Second sentence.";
- const chunks = await chunkText(text);
-
- expect(chunks[0].startOffset).toBe(0);
- expect(chunks[0].endOffset).toBeGreaterThan(0);
- expect(chunks[0].endOffset).toBeLessThanOrEqual(text.length);
- });
- });
-
- describe("offset tracking", () => {
- it("should track offsets correctly for multiple chunks", async () => {
- const longText = Array(100)
- .fill("This is a sentence with several words.")
- .join(" ");
- const chunks = await chunkText(longText, { maxTokens: 300 });
-
- // First chunk should start at 0
- expect(chunks[0].startOffset).toBe(0);
-
- // Each chunk's offsets should be sequential (allowing for overlap)
- for (let i = 1; i < chunks.length; i++) {
- expect(chunks[i].startOffset).toBeGreaterThanOrEqual(0);
- expect(chunks[i].endOffset).toBeGreaterThan(chunks[i].startOffset);
- }
- });
-
- it("should maintain consistent offsets in structure-preserving mode", async () => {
- const text = "# Heading\n\nParagraph one. " +
- Array(50).fill("More text.").join(" ") +
- "\n\n# Another Heading\n\nParagraph two.";
-
- const chunks = await chunkText(text, {
- maxTokens: 300,
- preserveStructure: true
- });
-
- chunks.forEach(chunk => {
- expect(chunk.startOffset).toBeGreaterThanOrEqual(0);
- expect(chunk.endOffset).toBeGreaterThan(chunk.startOffset);
- expect(chunk.endOffset).toBeLessThanOrEqual(text.length + 100); // Allow some margin for processing
- });
- });
-
- it("should maintain consistent offsets in sentence-based mode", async () => {
- const text = Array(80)
- .fill("This is a sentence.")
- .join(" ");
-
- const chunks = await chunkText(text, {
- maxTokens: 300,
- preserveStructure: false
- });
-
- chunks.forEach(chunk => {
- expect(chunk.startOffset).toBeGreaterThanOrEqual(0);
- expect(chunk.endOffset).toBeGreaterThan(chunk.startOffset);
- });
- });
- });
-
- describe("chunking strategies", () => {
- it("should preserve markdown structure when enabled", async () => {
- const text =
- "# Heading 1\n\nFirst paragraph.\n\n" +
- "## Heading 2\n\nSecond paragraph with more content.\n\n" +
- "- List item 1\n- List item 2\n\n" +
- "Final paragraph.";
-
- const chunks = await chunkText(text, {
- preserveStructure: true,
- maxTokens: 200
- });
-
- expect(chunks.length).toBeGreaterThan(0);
- // Structure preservation should try to keep sections together
- expect(chunks.some(c => c.text.includes("Heading"))).toBe(true);
- });
-
- it("should use sentence-based chunking when structure preservation is disabled", async () => {
- const text = Array(50)
- .fill("This is a sentence. Another sentence here.")
- .join(" ");
-
- const chunks = await chunkText(text, {
- preserveStructure: false,
- maxTokens: 300
- });
-
- expect(chunks.length).toBeGreaterThan(1);
- chunks.forEach(chunk => {
- expect(chunk.text.length).toBeGreaterThan(0);
- });
- });
-
- it("should handle large sections that need splitting", async () => {
- const largeSection = Array(100)
- .fill("Sentence in a large section.")
- .join(" ");
- const text = `# Large Section\n\n${largeSection}`;
-
- const chunks = await chunkText(text, {
- preserveStructure: true,
- maxTokens: 300
- });
-
- expect(chunks.length).toBeGreaterThan(1);
- });
- });
-
- describe("overlap handling", () => {
- it("should create overlapping chunks", async () => {
- const longText = Array(80)
- .fill("This is a unique sentence number.")
- .join(" ");
-
- const chunks = await chunkText(longText, {
- maxTokens: 300,
- overlapTokens: 50
- });
-
- if (chunks.length > 1) {
- // Chunks should have some overlap in content
- expect(chunks.length).toBeGreaterThan(1);
- // Each chunk should exist and have content
- chunks.forEach(chunk => {
- expect(chunk.text.length).toBeGreaterThan(0);
- });
- }
- });
-
- it("should apply overlap correctly with proper context", async () => {
- const sentences = Array(60)
- .fill(0)
- .map((_, i) => `Sentence number ${i}.`)
- .join(" ");
-
- const chunks = await chunkText(sentences, {
- maxTokens: 200,
- overlapTokens: 30
- });
-
- if (chunks.length > 1) {
- // Verify chunks are created with proper content
- chunks.forEach((chunk, i) => {
- expect(chunk.ordinal).toBe(i);
- expect(chunk.text.length).toBeGreaterThan(0);
- });
- }
- });
- });
-
- describe("edge cases", () => {
- it("should handle text with only whitespace", async () => {
- const chunks = await chunkText(" \n\n ");
- expect(chunks).toEqual([]);
- });
-
- it("should handle text with special characters", async () => {
- const text = "Text with émojis 🔥 and spëcial çharacters!";
- const chunks = await chunkText(text);
-
- expect(chunks).toHaveLength(1);
- expect(chunks[0].text).toContain("émojis");
- expect(chunks[0].text).toContain("🔥");
- });
-
- it("should handle very long single sentences", async () => {
- // A very long single sentence cannot be split at sentence boundaries
- // The chunker will keep it as one chunk even if it exceeds maxTokens
- const longSentence = "This is a very long sentence that " +
- Array(200).fill("keeps going and going").join(" ") +
- ".";
-
- const chunks = await chunkText(longSentence, { maxTokens: 500 });
-
- // Should create at least one chunk
- expect(chunks.length).toBeGreaterThan(0);
- // When a single sentence is too long, it stays as one chunk
- // This is expected behavior to maintain sentence integrity
- expect(chunks[0].text).toContain("very long sentence");
- });
-
- it("should handle text with multiple consecutive newlines", async () => {
- const text = "First paragraph.\n\n\n\nSecond paragraph.";
- const chunks = await chunkText(text);
-
- expect(chunks.length).toBeGreaterThan(0);
- expect(chunks[0].text).toContain("First paragraph");
- });
-
- it("should handle mixed content types", async () => {
- const text =
- "# Heading\n\n" +
- "Paragraph with text.\n\n" +
- "```code\nconst x = 1;\n```\n\n" +
- "- List item\n" +
- "- Another item\n\n" +
- "Final paragraph.";
-
- const chunks = await chunkText(text);
- expect(chunks.length).toBeGreaterThan(0);
- });
- });
-
- describe("options validation", () => {
- it("should use default options when none provided", async () => {
- const text = Array(50).fill("Text here.").join(" ");
- const chunks = await chunkText(text);
-
- expect(chunks).toBeDefined();
- expect(Array.isArray(chunks)).toBe(true);
- });
-
- it("should respect custom targetTokens", async () => {
- const text = Array(100).fill("Text here.").join(" ");
- const chunks = await chunkText(text, { targetTokens: 200 });
-
- expect(chunks.length).toBeGreaterThan(0);
- });
-
- it("should respect custom minTokens", async () => {
- const text = Array(100).fill("Text here.").join(" ");
- const chunks = await chunkText(text, { minTokens: 100 });
-
- expect(chunks.length).toBeGreaterThan(0);
- });
-
- it("should respect custom overlapTokens", async () => {
- const text = Array(100).fill("Text here.").join(" ");
- const chunks = await chunkText(text, {
- maxTokens: 300,
- overlapTokens: 75
- });
-
- expect(chunks.length).toBeGreaterThan(0);
- });
- });
-
- describe("chunk quality", () => {
- it("should not create empty chunks", async () => {
- const text = Array(100).fill("Content here.").join(" ");
- const chunks = await chunkText(text, { maxTokens: 300 });
-
- chunks.forEach(chunk => {
- expect(chunk.text.trim().length).toBeGreaterThan(0);
- expect(chunk.tokenCount).toBeGreaterThan(0);
- expect(chunk.charCount).toBeGreaterThan(0);
- });
- });
-
- it("should maintain text integrity", async () => {
- const text = "This is important content that must be preserved.";
- const chunks = await chunkText(text);
-
- const reconstructed = chunks.map(c => c.text).join(" ");
- expect(reconstructed).toContain("important content");
- });
-
- it("should create reasonably sized chunks", async () => {
- const text = Array(100).fill("Some text.").join(" ");
- const maxTokens = 400;
- const chunks = await chunkText(text, { maxTokens });
-
- chunks.forEach(chunk => {
- expect(chunk.tokenCount).toBeGreaterThan(0);
- expect(chunk.tokenCount).toBeLessThanOrEqual(maxTokens);
- });
- });
- });
-
- describe("performance", () => {
- it("should handle large documents efficiently", async () => {
- const largeText = Array(500)
- .fill("This is a sentence in a large document.")
- .join(" ");
-
- const startTime = Date.now();
- const chunks = await chunkText(largeText, { maxTokens: 500 });
- const duration = Date.now() - startTime;
-
- expect(chunks.length).toBeGreaterThan(0);
- expect(duration).toBeLessThan(5000); // Should complete in under 5 seconds
- });
-
- it("should handle documents with complex structure", async () => {
- const sections = Array(20).fill(0).map((_, i) =>
- `# Section ${i}\n\nContent for section ${i}.\n\n` +
- Array(10).fill("More text here.").join(" ")
- ).join("\n\n");
-
- const chunks = await chunkText(sections, {
- preserveStructure: true,
- maxTokens: 500
- });
-
- expect(chunks.length).toBeGreaterThan(0);
- });
- });
- });
-});
-
diff --git a/apps/api/src/lib/search-index/chunker.ts b/apps/api/src/lib/search-index/chunker.ts
deleted file mode 100644
index 4c8512c4b..000000000
--- a/apps/api/src/lib/search-index/chunker.ts
+++ /dev/null
@@ -1,374 +0,0 @@
-import { logger as _logger } from "../logger";
-import type { Logger } from "winston";
-
-export interface TextChunk {
- text: string;
- ordinal: number;
- tokenCount: number;
- charCount: number;
- startOffset: number;
- endOffset: number;
-}
-
-export interface ChunkingOptions {
- targetTokens?: number; // Target tokens per chunk (default: 750)
- minTokens?: number; // Minimum tokens per chunk (default: 600)
- maxTokens?: number; // Maximum tokens per chunk (default: 900)
- overlapTokens?: number; // Overlap between chunks (default: 100)
- preserveStructure?: boolean; // Preserve markdown structure (default: true)
-}
-
-const DEFAULT_OPTIONS: Required<ChunkingOptions> = {
- targetTokens: 750,
- minTokens: 600,
- maxTokens: 900,
- overlapTokens: 100,
- preserveStructure: true,
-};
-
-/**
- * Estimate token count using simple heuristic (1 token ≈ 4 chars for English)
- * This is approximate but fast. For exact counts, use tiktoken.
- */
-export function estimateTokenCount(text: string): number {
- // Remove extra whitespace
- const normalized = text.replace(/\s+/g, " ").trim();
- // Rough approximation: 1 token ≈ 4 characters
- return Math.ceil(normalized.length / 4);
-}
-
-/**
- * Split text into sentences, preserving structure
- */
-function splitIntoSentences(text: string): string[] {
- // Split on sentence boundaries, but preserve structure
- const sentences: string[] = [];
-
- // Match sentences ending with .!? followed by space/newline/end
- // Also handle abbreviations like "Dr." "Mr." "etc."
- const sentenceRegex = /(?<![A-Z][a-z]\.)\s*([.!?]+)\s+(?=[A-Z]|$)/g;
-
- let lastIndex = 0;
- let match: RegExpExecArray | null;
-
- while ((match = sentenceRegex.exec(text)) !== null) {
- const sentence = text.slice(lastIndex, match.index + match[1].length).trim();
- if (sentence.length > 0) {
- sentences.push(sentence);
- }
- lastIndex = match.index + match[0].length;
- }
-
- // Add remaining text
- if (lastIndex < text.length) {
- const remaining = text.slice(lastIndex).trim();
- if (remaining.length > 0) {
- sentences.push(remaining);
- }
- }
-
- return sentences.filter(s => s.length > 0);
-}
-
-/**
- * Detect markdown structure boundaries (headings, lists, code blocks)
- */
-function detectStructureBoundaries(text: string): number[] {
- const boundaries: number[] = [0];
-
- const lines = text.split("\n");
- let offset = 0;
-
- for (let i = 0; i < lines.length; i++) {
- const line = lines[i];
-
- // Detect headings
- if (/^#{1,6}\s/.test(line)) {
- boundaries.push(offset);
- }
-
- // Detect code blocks
- if (/^```/.test(line)) {
- boundaries.push(offset);
- }
-
- // Detect list items (with proper structure)
- if (/^[\s]*[-*+]\s/.test(line) || /^[\s]*\d+\.\s/.test(line)) {
- boundaries.push(offset);
- }
-
- offset += line.length + 1; // +1 for newline
- }
-
- boundaries.push(text.length);
-
- return [...new Set(boundaries)].sort((a, b) => a - b);
-}
-
-/**
- * Main chunking function: splits text into semantically coherent chunks
- */
-export async function chunkText(
- text: string,
- options: ChunkingOptions = {},
- logger?: Logger,
-): Promise<TextChunk[]> {
- const opts = { ...DEFAULT_OPTIONS, ...options };
- const log = logger ?? _logger.child({ module: "search-chunker" });
-
- // Handle empty or very short text
- if (!text || text.trim().length === 0) {
- return [];
- }
-
- const totalTokens = estimateTokenCount(text);
-
- // If text is short enough, return as single chunk
- if (totalTokens <= opts.maxTokens) {
- return [{
- text: text.trim(),
- ordinal: 0,
- tokenCount: totalTokens,
- charCount: text.length,
- startOffset: 0,
- endOffset: text.length,
- }];
- }
-
- const chunks: TextChunk[] = [];
-
- // Strategy 1: If preserving structure, respect markdown boundaries
- if (opts.preserveStructure) {
- const boundaries = detectStructureBoundaries(text);
- const sections = boundaries.slice(0, -1).map((start, i) => ({
- text: text.slice(start, boundaries[i + 1]),
- offset: start,
- }));
-
- let currentChunk = "";
- let currentOffset = 0;
- let currentTokens = 0;
-
- for (const section of sections) {
- const sectionTokens = estimateTokenCount(section.text);
-
- // If section alone exceeds max, split it further
- if (sectionTokens > opts.maxTokens) {
- // Flush current chunk if any
- if (currentChunk.length > 0) {
- chunks.push({
- text: currentChunk.trim(),
- ordinal: chunks.length,
- tokenCount: currentTokens,
- charCount: currentChunk.length,
- startOffset: currentOffset,
- endOffset: currentOffset + currentChunk.length,
- });
- currentChunk = "";
- currentTokens = 0;
- }
-
- // Split large section by sentences
- const sentences = splitIntoSentences(section.text);
- let sentenceChunk = "";
- let sentenceTokens = 0;
- let sentenceChunkOffset = section.offset;
-
- for (const sentence of sentences) {
- const sentenceTokenCount = estimateTokenCount(sentence);
-
- if (sentenceTokens + sentenceTokenCount > opts.maxTokens && sentenceChunk.length > 0) {
- // Save overlap from the previous chunk before flushing
- const overlapText = sentenceChunk.slice(-opts.overlapTokens * 4); // Rough char estimate
- const chunkLength = sentenceChunk.length;
-
- chunks.push({
- text: sentenceChunk.trim(),
- ordinal: chunks.length,
- tokenCount: sentenceTokens,
- charCount: chunkLength,
- startOffset: sentenceChunkOffset,
- endOffset: sentenceChunkOffset + chunkLength,
- });
-
- // Update offset for next chunk - account for overlap
- // The next chunk will start with overlapText, so its offset should point
- // to where the overlap begins in the source (not where previous chunk ended)
- sentenceChunkOffset += chunkLength - overlapText.length;
-
- // Start new chunk with overlap from previous chunk + new sentence
- sentenceChunk = overlapText + " " + sentence;
- sentenceTokens = estimateTokenCount(sentenceChunk);
- } else {
- sentenceChunk += (sentenceChunk.length > 0 ? " " : "") + sentence;
- sentenceTokens += sentenceTokenCount;
- }
- }
-
- if (sentenceChunk.length > 0) {
- chunks.push({
- text: sentenceChunk.trim(),
- ordinal: chunks.length,
- tokenCount: sentenceTokens,
- charCount: sentenceChunk.length,
- startOffset: sentenceChunkOffset,
- endOffset: sentenceChunkOffset + sentenceChunk.length,
- });
- }
-
- currentOffset = section.offset + section.text.length;
- currentChunk = "";
- currentTokens = 0;
- continue;
- }
-
- // Try to add section to current chunk
- if (currentTokens + sectionTokens <= opts.maxTokens) {
- currentChunk += (currentChunk.length > 0 ? "\n" : "") + section.text;
- currentTokens += sectionTokens;
- if (currentChunk.length === section.text.length) {
- currentOffset = section.offset;
- }
- } else {
- // Current chunk is full, flush it
- if (currentChunk.length > 0) {
- chunks.push({
- text: currentChunk.trim(),
- ordinal: chunks.length,
- tokenCount: currentTokens,
- charCount: currentChunk.length,
- startOffset: currentOffset,
- endOffset: currentOffset + currentChunk.length,
- });
- }
-
- // Start new chunk with current section
- currentChunk = section.text;
- currentOffset = section.offset;
- currentTokens = sectionTokens;
- }
- }
-
- // Flush remaining chunk
- if (currentChunk.length > 0) {
- chunks.push({
- text: currentChunk.trim(),
- ordinal: chunks.length,
- tokenCount: currentTokens,
- charCount: currentChunk.length,
- startOffset: currentOffset,
- endOffset: currentOffset + currentChunk.length,
- });
- }
- } else {
- // Strategy 2: Simple sentence-based chunking without structure preservation
- const sentences = splitIntoSentences(text);
- let currentChunk = "";
- let currentTokens = 0;
- let currentOffset = 0;
-
- for (const sentence of sentences) {
- const sentenceTokens = estimateTokenCount(sentence);
-
- if (currentTokens + sentenceTokens > opts.maxTokens && currentChunk.length > 0) {
- const chunkLength = currentChunk.length;
- chunks.push({
- text: currentChunk.trim(),
- ordinal: chunks.length,
- tokenCount: currentTokens,
- charCount: chunkLength,
- startOffset: currentOffset,
- endOffset: currentOffset + chunkLength,
- });
-
- // Add overlap for context
- const lastSentences = currentChunk.split(/[.!?]+\s+/).slice(-2).join(". ");
-
- // Update offset for next chunk - account for overlap
- // The next chunk will start with lastSentences, so its offset should point
- // to where the overlap begins in the source (not where previous chunk ended)
- currentOffset += chunkLength - lastSentences.length;
-
- currentChunk = lastSentences + " " + sentence;
- currentTokens = estimateTokenCount(currentChunk);
- } else {
- currentChunk += (currentChunk.length > 0 ? " " : "") + sentence;
- currentTokens += sentenceTokens;
- }
-
- if (chunks.length === 0 && currentChunk === sentence) {
- currentOffset = 0;
- }
- }
-
- // Flush remaining
- if (currentChunk.length > 0) {
- chunks.push({
- text: currentChunk.trim(),
- ordinal: chunks.length,
- tokenCount: currentTokens,
- charCount: currentChunk.length,
- startOffset: currentOffset,
- endOffset: currentOffset + currentChunk.length,
- });
- }
- }
-
- log.debug("Chunked text", {
- totalTokens,
- chunks: chunks.length,
- avgTokensPerChunk: chunks.length > 0
- ? Math.round(chunks.reduce((sum, c) => sum + c.tokenCount, 0) / chunks.length)
- : 0,
- });
-
- return chunks;
-}
-
-/**
- * Extract clean text from markdown for indexing
- * Removes markdown syntax but preserves structure and content
- */
-export function cleanMarkdownForIndexing(markdown: string): string {
- if (!markdown) return "";
-
- let text = markdown;
-
- // Remove code blocks but keep language hints
- text = text.replace(/```(\w+)?\n([\s\S]*?)```/g, (_, lang, code) => {
- return lang ? `Code (${lang}): ${code.split("\n").slice(0, 3).join(" ")}...` : "";
- });
-
- // Remove inline code (keep content)
- text = text.replace(/`([^`]+)`/g, "$1");
-
- // Remove images (keep alt text)
- text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1");
-
- // Remove links (keep text)
- text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
-
- // Remove headers (#) but keep text
- text = text.replace(/^#{1,6}\s+(.+)$/gm, "$1");
-
- // Remove bold/italic
- text = text.replace(/(\*\*|__)(.*?)\1/g, "$2");
- text = text.replace(/(\*|_)(.*?)\1/g, "$2");
-
- // Remove list markers but keep structure
- text = text.replace(/^[\s]*[-*+]\s+/gm, "");
- text = text.replace(/^[\s]*\d+\.\s+/gm, "");
-
- // Remove blockquotes
- text = text.replace(/^>\s+/gm, "");
-
- // Remove horizontal rules
- text = text.replace(/^[-*_]{3,}$/gm, "");
-
- // Normalize whitespace
- text = text.replace(/\n{3,}/g, "\n\n");
- text = text.replace(/ +/g, " ");
-
- return text.trim();
-}
-
diff --git a/apps/api/src/lib/search-index/embeddings.ts b/apps/api/src/lib/search-index/embeddings.ts
deleted file mode 100644
index 5c8bde8cb..000000000
--- a/apps/api/src/lib/search-index/embeddings.ts
+++ /dev/null
@@ -1,291 +0,0 @@
-import { embed, EmbeddingModel } from "ai";
-import { getEmbeddingModel } from "../generic-ai";
-import { logger as _logger } from "../logger";
-import type { Logger } from "winston";
-import { withSpan, setSpanAttributes } from "../otel-tracer";
-
-export interface EmbeddingResult {
- embedding: number[];
- tokens: number;
-}
-
-export interface BatchEmbeddingResult {
- embeddings: number[][];
- totalTokens: number;
- failed: number[];
-}
-
-export interface EmbeddingOptions {
- model?: string;
- dimensions?: number;
- maxRetries?: number;
- retryDelay?: number;
- batchSize?: number;
-}
-
-const DEFAULT_OPTIONS: Required<EmbeddingOptions> = {
- model: "text-embedding-3-small",
- dimensions: 1536,
- maxRetries: 3,
- retryDelay: 1000, // ms
- batchSize: 100,
-};
-
-/**
- * Generate embedding for a single text
- */
-export async function generateEmbedding(
- text: string,
- options: EmbeddingOptions = {},
- logger?: Logger,
-): Promise<EmbeddingResult> {
- return await withSpan("firecrawl-generate-embedding", async span => {
- const opts = { ...DEFAULT_OPTIONS, ...options };
- const log = logger ?? _logger.child({ module: "search-embeddings" });
-
- setSpanAttributes(span, {
- "embedding.model": opts.model,
- "embedding.dimensions": opts.dimensions,
- "embedding.text_length": text.length,
- });
-
- if (!text || text.trim().length === 0) {
- setSpanAttributes(span, { "embedding.empty_text": true });
- return {
- embedding: new Array(opts.dimensions).fill(0),
- tokens: 0,
- };
- }
-
- // Truncate if too long (OpenAI limit: 8191 tokens for text-embedding-3-small)
- const maxChars = 8191 * 4; // Rough approximation
- const truncatedText = text.length > maxChars ? text.slice(0, maxChars) : text;
-
- let lastError: Error | null = null;
-
- for (let attempt = 0; attempt < opts.maxRetries; attempt++) {
- try {
- const embeddingModel: EmbeddingModel<string> = getEmbeddingModel(
- opts.model,
- "openai",
- );
-
- const result = await embed({
- model: embeddingModel,
- value: truncatedText,
- });
-
- setSpanAttributes(span, {
- "embedding.success": true,
- "embedding.tokens": result.usage.tokens,
- "embedding.attempt": attempt + 1,
- });
-
- log.debug("Generated embedding", {
- textLength: text.length,
- tokens: result.usage.tokens,
- dimensions: result.embedding.length,
- });
-
- return {
- embedding: result.embedding,
- tokens: result.usage.tokens,
- };
- } catch (error) {
- lastError = error as Error;
-
- setSpanAttributes(span, {
- "embedding.error": true,
- "embedding.attempt": attempt + 1,
- "embedding.error_message": lastError.message,
- });
-
- log.warn("Failed to generate embedding", {
- error: lastError.message,
- attempt: attempt + 1,
- maxRetries: opts.maxRetries,
- });
-
- // Exponential backoff
- if (attempt < opts.maxRetries - 1) {
- const delay = opts.retryDelay * Math.pow(2, attempt);
- await new Promise(resolve => setTimeout(resolve, delay));
- }
- }
- }
-
- log.error("Failed to generate embedding after all retries", {
- error: lastError?.message,
- maxRetries: opts.maxRetries,
- });
-
- // Return zero vector as fallback
- return {
- embedding: new Array(opts.dimensions).fill(0),
- tokens: 0,
- };
- });
-}
-
-/**
- * Generate embeddings for multiple texts in batches
- * More efficient for bulk operations
- */
-export async function generateEmbeddingsBatch(
- texts: string[],
- options: EmbeddingOptions = {},
- logger?: Logger,
-): Promise<BatchEmbeddingResult> {
- return await withSpan("firecrawl-generate-embeddings-batch", async span => {
- const opts = { ...DEFAULT_OPTIONS, ...options };
- const log = logger ?? _logger.child({ module: "search-embeddings-batch" });
-
- setSpanAttributes(span, {
- "embedding.batch_size": texts.length,
- "embedding.model": opts.model,
- });
-
- if (texts.length === 0) {
- return {
- embeddings: [],
- totalTokens: 0,
- failed: [],
- };
- }
-
- const embeddings: number[][] = new Array(texts.length);
- const failed: number[] = [];
- let totalTokens = 0;
-
- // Process in batches
- const batches: string[][] = [];
- for (let i = 0; i < texts.length; i += opts.batchSize) {
- batches.push(texts.slice(i, i + opts.batchSize));
- }
-
- log.info("Processing embedding batches", {
- totalTexts: texts.length,
- batches: batches.length,
- batchSize: opts.batchSize,
- });
-
- for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
- const batch = batches[batchIndex];
- const batchOffset = batchIndex * opts.batchSize;
-
- log.debug("Processing batch", {
- batch: batchIndex + 1,
- total: batches.length,
- size: batch.length,
- });
-
- // Process batch items in parallel
- const batchResults = await Promise.allSettled(
- batch.map((text, i) =>
- generateEmbedding(text, options, log).then(result => ({
- index: batchOffset + i,
- ...result,
- }))
- ),
- );
-
- // Collect results
- for (const result of batchResults) {
- if (result.status === "fulfilled") {
- const { index, embedding, tokens } = result.value;
- embeddings[index] = embedding;
- totalTokens += tokens;
- } else {
- const index = batchOffset + batchResults.indexOf(result);
- failed.push(index);
- embeddings[index] = new Array(opts.dimensions).fill(0);
-
- log.warn("Failed to generate embedding in batch", {
- index,
- error: result.reason?.message,
- });
- }
- }
-
- // Rate limiting: small delay between batches
- if (batchIndex < batches.length - 1) {
- await new Promise(resolve => setTimeout(resolve, 100));
- }
- }
-
- setSpanAttributes(span, {
- "embedding.total_tokens": totalTokens,
- "embedding.failed_count": failed.length,
- "embedding.success_rate": ((texts.length - failed.length) / texts.length),
- });
-
- log.info("Batch embedding complete", {
- total: texts.length,
- successful: texts.length - failed.length,
- failed: failed.length,
- totalTokens,
- });
-
- return {
- embeddings,
- totalTokens,
- failed,
- };
- });
-}
-
-/**
- * Estimate embedding cost for text
- * Based on OpenAI pricing: $0.02 / 1M tokens
- */
-export function estimateEmbeddingCost(tokenCount: number): number {
- return (tokenCount / 1_000_000) * 0.02;
-}
-
-/**
- * Check if embeddings are available (API key configured)
- */
-export function isEmbeddingEnabled(): boolean {
- return !!process.env.OPENAI_API_KEY;
-}
-
-/**
- * Normalize embedding vector (convert to unit vector)
- * Useful for cosine similarity when using dot product
- */
-export function normalizeEmbedding(embedding: number[]): number[] {
- const magnitude = Math.sqrt(
- embedding.reduce((sum, val) => sum + val * val, 0),
- );
-
- if (magnitude === 0) return embedding;
-
- return embedding.map(val => val / magnitude);
-}
-
-/**
- * Calculate cosine similarity between two embeddings
- */
-export function cosineSimilarity(a: number[], b: number[]): number {
- if (a.length !== b.length) {
- throw new Error("Embeddings must have the same dimensions");
- }
-
- let dotProduct = 0;
- let magnitudeA = 0;
- let magnitudeB = 0;
-
- for (let i = 0; i < a.length; i++) {
- dotProduct += a[i] * b[i];
- magnitudeA += a[i] * a[i];
- magnitudeB += b[i] * b[i];
- }
-
- magnitudeA = Math.sqrt(magnitudeA);
- magnitudeB = Math.sqrt(magnitudeB);
-
- if (magnitudeA === 0 || magnitudeB === 0) return 0;
-
- return dotProduct / (magnitudeA * magnitudeB);
-}
-
diff --git a/apps/api/src/lib/search-index/index.ts b/apps/api/src/lib/search-index/index.ts
deleted file mode 100644
index 2e946ef7f..000000000
--- a/apps/api/src/lib/search-index/index.ts
+++ /dev/null
@@ -1,56 +0,0 @@
-
-export {
- chunkText,
- cleanMarkdownForIndexing,
- estimateTokenCount,
- type TextChunk,
- type ChunkingOptions,
-} from "./chunker";
-
-export {
- generateEmbedding,
- generateEmbeddingsBatch,
- isEmbeddingEnabled,
- estimateEmbeddingCost,
- normalizeEmbedding,
- cosineSimilarity,
- type EmbeddingResult,
- type BatchEmbeddingResult,
- type EmbeddingOptions,
-} from "./embeddings";
-
-export {
- indexDocumentForSearch,
- deleteDocumentFromSearch,
- searchDocumentExists,
- type SearchDocumentInput,
- type SearchIndexResult,
-} from "./service";
-
-export {
- search,
- searchChunks,
- getSearchStats,
- type SearchQuery,
- type SearchFilters,
- type SearchResult,
- type SearchResponse,
-} from "./query";
-
-export {
- addSearchIndexJob,
- processSearchIndexJobs,
- getSearchIndexQueueLength,
-} from "./queue";
-
-export {
- upsertToPinecone,
- searchPinecone,
- deleteFromPinecone,
- getPineconeStats,
- isPineconeEnabled,
- buildPineconeFilter,
- getPineconeIndex,
- type PineconeRecord,
-} from "./pinecone-service";
-
diff --git a/apps/api/src/lib/search-index/pinecone-service.ts b/apps/api/src/lib/search-index/pinecone-service.ts
deleted file mode 100644
index c2809855e..000000000
--- a/apps/api/src/lib/search-index/pinecone-service.ts
+++ /dev/null
@@ -1,297 +0,0 @@
-import { Pinecone } from "@pinecone-database/pinecone";
-import { logger as _logger } from "../logger";
-import type { Logger } from "winston";
-import { withSpan, setSpanAttributes } from "../otel-tracer";
-
-// Pinecone client (singleton)
-let pineconeClient: Pinecone | null = null;
-let pineconeIndex: any = null;
-
-/**
- * Initialize Pinecone client
- */
-function getPineconeClient(): Pinecone {
- if (!pineconeClient) {
- const apiKey = process.env.PINECONE_API_KEY;
-
- if (!apiKey) {
- throw new Error("PINECONE_API_KEY not set");
- }
-
- pineconeClient = new Pinecone({
- apiKey,
- });
-
- _logger.info("Pinecone client initialized");
- }
-
- return pineconeClient;
-}
-
-/**
- * Get Pinecone index
- */
-export function getPineconeIndex() {
- if (!pineconeIndex) {
- const client = getPineconeClient();
- const indexName = process.env.PINECONE_INDEX_NAME || "firecrawl-search";
-
- pineconeIndex = client.index(indexName);
-
- _logger.info("Pinecone index initialized", { indexName });
- }
-
- return pineconeIndex;
-}
-
-/**
- * Check if Pinecone is enabled
- * Note: PINECONE_INDEX_NAME is optional and defaults to "firecrawl-search" in getPineconeIndex()
- */
-export function isPineconeEnabled(): boolean {
- return !!process.env.PINECONE_API_KEY;
-}
-
-export interface PineconeRecord {
- id: string;
- values: number[];
- metadata: {
- url: string;
- domain: string;
- title?: string;
- freshness_score: number;
- quality_score: number;
- country?: string;
- is_mobile: boolean;
- chunk_ordinal?: number; // For chunks
- doc_id?: string; // For chunks
- };
-}
-
-/**
- * Upsert embeddings to Pinecone
- */
-export async function upsertToPinecone(
- records: PineconeRecord[],
- namespace: string = "documents",
- logger?: Logger,
-): Promise<void> {
- return await withSpan("firecrawl-pinecone-upsert", async span => {
- const log = logger ?? _logger.child({ module: "pinecone-service" });
-
- setSpanAttributes(span, {
- "pinecone.operation": "upsert",
- "pinecone.namespace": namespace,
- "pinecone.record_count": records.length,
- });
-
- if (records.length === 0) {
- return;
- }
-
- try {
- const index = getPineconeIndex();
- const ns = index.namespace(namespace);
-
- // Upsert in batches of 100 (Pinecone limit)
- const batchSize = 100;
- for (let i = 0; i < records.length; i += batchSize) {
- const batch = records.slice(i, i + batchSize);
-
- await ns.upsert(batch);
-
- log.debug("Upserted batch to Pinecone", {
- batch: Math.floor(i / batchSize) + 1,
- size: batch.length,
- });
- }
-
- setSpanAttributes(span, {
- "pinecone.upsert_successful": true,
- });
-
- log.info("Upserted to Pinecone", {
- namespace,
- records: records.length,
- });
- } catch (error) {
- log.error("Failed to upsert to Pinecone", {
- error: (error as Error).message,
- namespace,
- recordCount: records.length,
- });
-
- setSpanAttributes(span, {
- "pinecone.upsert_error": true,
- "pinecone.error_message": (error as Error).message,
- });
-
- throw error;
- }
- });
-}
-
-/**
- * Search Pinecone for similar vectors
- */
-export async function searchPinecone(
- queryEmbedding: number[],
- limit: number = 100,
- filter?: Record<string, any>,
- namespace: string = "documents",
- logger?: Logger,
-): Promise<Array<{
- id: string;
- score: number;
- metadata: Record<string, any>;
-}>> {
- return await withSpan("firecrawl-pinecone-query", async span => {
- const log = logger ?? _logger.child({ module: "pinecone-service" });
-
- setSpanAttributes(span, {
- "pinecone.operation": "query",
- "pinecone.namespace": namespace,
- "pinecone.limit": limit,
- "pinecone.has_filter": !!filter,
- });
-
- try {
- const index = getPineconeIndex();
- const ns = index.namespace(namespace);
-
- const queryResponse = await ns.query({
- vector: queryEmbedding,
- topK: limit,
- filter,
- includeMetadata: true,
- includeValues: false,
- });
-
- const results = (queryResponse.matches || []).map(match => ({
- id: match.id,
- score: match.score || 0,
- metadata: match.metadata || {},
- }));
-
- setSpanAttributes(span, {
- "pinecone.query_successful": true,
- "pinecone.results_count": results.length,
- });
-
- log.debug("Queried Pinecone", {
- namespace,
- limit,
- results: results.length,
- });
-
- return results;
- } catch (error) {
- log.error("Failed to query Pinecone", {
- error: (error as Error).message,
- namespace,
- limit,
- });
-
- setSpanAttributes(span, {
- "pinecone.query_error": true,
- "pinecone.error_message": (error as Error).message,
- });
-
- throw error;
- }
- });
-}
-
-/**
- * Delete from Pinecone
- */
-export async function deleteFromPinecone(
- ids: string[],
- namespace: string = "documents",
- logger?: Logger,
-): Promise<void> {
- const log = logger ?? _logger.child({ module: "pinecone-service" });
-
- try {
- const index = getPineconeIndex();
- const ns = index.namespace(namespace);
-
- await ns.deleteMany(ids);
-
- log.info("Deleted from Pinecone", {
- namespace,
- count: ids.length,
- });
- } catch (error) {
- log.error("Failed to delete from Pinecone", {
- error: (error as Error).message,
- namespace,
- count: ids.length,
- });
- }
-}
-
-/**
- * Get Pinecone index stats
- */
-export async function getPineconeStats(
- namespace: string = "documents",
-): Promise<{
- vectorCount: number;
- dimension: number;
- indexFullness: number;
-}> {
- try {
- const index = getPineconeIndex();
- const stats = await index.describeIndexStats();
-
- const nsStats = stats.namespaces?.[namespace];
-
- return {
- vectorCount: nsStats?.recordCount || 0,
- dimension: stats.dimension || 1536,
- indexFullness: stats.indexFullness || 0,
- };
- } catch (error) {
- _logger.error("Failed to get Pinecone stats", {
- error: (error as Error).message,
- });
-
- return {
- vectorCount: 0,
- dimension: 1536,
- indexFullness: 0,
- };
- }
-}
-
-/**
- * Build Pinecone filter from search filters
- */
-export function buildPineconeFilter(filters: {
- domain?: string;
- country?: string;
- isMobile?: boolean;
- minFreshness?: number;
-}): Record<string, any> | undefined {
- const filter: Record<string, any> = {};
-
- if (filters.domain) {
- filter.domain = { $eq: filters.domain };
- }
-
- if (filters.country) {
- filter.country = { $eq: filters.country };
- }
-
- if (filters.isMobile !== undefined) {
- filter.is_mobile = { $eq: filters.isMobile };
- }
-
- if (filters.minFreshness !== undefined) {
- filter.freshness_score = { $gte: filters.minFreshness };
- }
-
- return Object.keys(filter).length > 0 ? filter : undefined;
-}
-
diff --git a/apps/api/src/lib/search-index/query.ts b/apps/api/src/lib/search-index/query.ts
deleted file mode 100644
index 2c9bff4ee..000000000
--- a/apps/api/src/lib/search-index/query.ts
+++ /dev/null
@@ -1,644 +0,0 @@
-import { SupabaseClient } from "@supabase/supabase-js";
-import { logger as _logger } from "../logger";
-import type { Logger } from "winston";
-import { withSpan, setSpanAttributes } from "../otel-tracer";
-import { generateEmbedding, isEmbeddingEnabled } from "./embeddings";
-import {
- searchPinecone,
- isPineconeEnabled,
- buildPineconeFilter,
-} from "./pinecone-service";
-
-export interface SearchQuery {
- query: string;
- limit?: number;
- offset?: number;
- filters?: SearchFilters;
- mode?: "hybrid" | "keyword" | "semantic";
-}
-
-export interface SearchFilters {
- domain?: string;
- country?: string;
- isMobile?: boolean;
- minFreshness?: number;
- language?: string;
-}
-
-export interface SearchResult {
- documentId: string;
- url: string;
- title: string | null;
- description: string | null;
- domain: string;
- snippet?: string;
- score: number;
- bm25Rank: number | null;
- vectorRank: number | null;
- freshnessScore: number;
- qualityScore: number;
- lastCrawledAt: string;
-}
-
-export interface SearchResponse {
- results: SearchResult[];
- total: number;
- query: string;
- mode: string;
- took: number; // milliseconds
-}
-
-/**
- * Main search function: hybrid search with RRF ranking
- */
-export async function search(
- supabase: SupabaseClient,
- searchQuery: SearchQuery,
- logger?: Logger,
-): Promise<SearchResponse> {
- return await withSpan("firecrawl-search", async span => {
- const log = logger ?? _logger.child({ module: "search-query" });
- const startTime = Date.now();
-
- const {
- query,
- limit = 50,
- offset = 0,
- filters = {},
- mode = "hybrid",
- } = searchQuery;
-
- setSpanAttributes(span, {
- "search.query": query,
- "search.limit": limit,
- "search.mode": mode,
- "search.filters": JSON.stringify(filters),
- });
-
- if (!query || query.trim().length === 0) {
- return {
- results: [],
- total: 0,
- query,
- mode,
- took: Date.now() - startTime,
- };
- }
-
- try {
- let results: SearchResult[] = [];
-
- // Fetch enough results to support pagination
- const fetchLimit = limit + offset;
-
- // Choose search strategy based on mode
- if (mode === "keyword") {
- results = await keywordSearch(supabase, query, fetchLimit, filters, log);
- } else if (mode === "semantic") {
- results = await semanticSearch(supabase, query, fetchLimit, filters, log);
- } else {
- // Hybrid: combine BM25 + vector with RRF
- results = await hybridSearch(supabase, query, fetchLimit, filters, log);
- }
-
- // Apply offset and limit for pagination
- const paginatedResults = results.slice(offset, offset + limit);
-
- const took = Date.now() - startTime;
-
- setSpanAttributes(span, {
- "search.results_count": results.length,
- "search.took_ms": took,
- });
-
- log.info("Search completed", {
- query,
- mode,
- results: results.length,
- took,
- });
-
- return {
- results: paginatedResults,
- total: results.length,
- query,
- mode,
- took,
- };
- } catch (error) {
- log.error("Search failed", {
- error: (error as Error).message,
- query,
- });
-
- setSpanAttributes(span, {
- "search.error": true,
- "search.error_message": (error as Error).message,
- });
-
- return {
- results: [],
- total: 0,
- query,
- mode,
- took: Date.now() - startTime,
- };
- }
- });
-}
-
-/**
- * Hybrid search: BM25 (Postgres) + Vector (Pinecone) with RRF ranking
- */
-async function hybridSearch(
- supabase: SupabaseClient,
- query: string,
- limit: number,
- filters: SearchFilters,
- logger: Logger,
-): Promise<SearchResult[]> {
- // Generate query embedding for vector search
- let queryEmbedding: number[] | null = null;
-
- if (isEmbeddingEnabled() && isPineconeEnabled()) {
- try {
- const embResult = await generateEmbedding(query, {}, logger);
- queryEmbedding = embResult.embedding;
- } catch (error) {
- logger.warn("Failed to generate query embedding, falling back to keyword search", {
- error: (error as Error).message,
- });
- }
- }
-
- // If no embedding, fall back to keyword search
- if (!queryEmbedding) {
- return keywordSearch(supabase, query, limit, filters, logger);
- }
-
- // Parallel search: BM25 in Postgres + Vector in Pinecone
- const [bm25Results, vectorResults] = await Promise.all([
- // BM25 search in Postgres
- supabase.rpc("bm25_search", {
- query_text: query,
- result_limit: 100,
- country_filter: filters.country ?? null,
- domain_filter: filters.domain ?? null,
- is_mobile_filter: filters.isMobile ?? null,
- min_freshness: filters.minFreshness ?? 0.0,
- }),
-
- // Vector search in Pinecone
- (async () => {
- try {
- const pineconeFilter = buildPineconeFilter(filters);
- const results = await searchPinecone(
- queryEmbedding!,
- 100,
- pineconeFilter,
- "documents",
- logger,
- );
- return results;
- } catch (error) {
- logger.warn("Pinecone search failed, continuing with BM25 only", {
- error: (error as Error).message,
- });
- return [];
- }
- })(),
- ]);
-
- if (bm25Results.error) {
- logger.error("BM25 search failed", { error: bm25Results.error.message });
- throw new Error(`BM25 search failed: ${bm25Results.error.message}`);
- }
-
- // Extract Pinecone doc IDs
- const pineconeDocIds = vectorResults
- .map(r => r.metadata.doc_id)
- .filter(Boolean) as string[];
-
- // Get metadata for Pinecone results from Postgres
- const { data: pineconeMetadata } = await supabase.rpc("get_documents_metadata", {
- doc_ids: pineconeDocIds,
- });
-
- // Create map of doc_id -> metadata
- const metadataMap = new Map(
- (pineconeMetadata || []).map((m: any) => [m.doc_id, m])
- );
-
- // Merge results with RRF
- const bm25Map = new Map<string, any>(
- (bm25Results.data || []).map((r: any, index: number) => [
- r.doc_id,
- { rank: index + 1, ...r },
- ])
- );
-
- const vectorMap = new Map<string, { rank: number; score: number }>(
- vectorResults.map((r, index) => [
- r.metadata.doc_id || r.id,
- { rank: index + 1, score: r.score },
- ])
- );
-
- // Combine all unique doc IDs
- const allDocIds = new Set([
- ...bm25Map.keys(),
- ...vectorMap.keys(),
- ]);
-
- const k = 60; // RRF constant
- const merged: SearchResult[] = [];
-
- for (const docId of allDocIds) {
- const bm25 = bm25Map.get(docId);
- const vector = vectorMap.get(docId);
-
- // RRF score
- const bm25Rank = bm25?.rank ?? null;
- const vectorRank = vector?.rank ?? null;
-
- const rrf =
- (bm25Rank ? 1 / (k + bm25Rank) : 0) +
- (vectorRank ? 1 / (k + vectorRank) : 0);
-
- // Get metadata (prefer from BM25 result, fallback to Pinecone metadata)
- const metadata: any = bm25 || metadataMap.get(docId);
-
- if (!metadata) continue;
-
- const combinedScore =
- rrf * (metadata.freshness_score || 1.0) * (metadata.quality_score || 1.0);
-
- merged.push({
- documentId: docId,
- url: metadata.url,
- title: metadata.title,
- description: metadata.description,
- domain: metadata.domain,
- score: combinedScore,
- bm25Rank,
- vectorRank,
- freshnessScore: metadata.freshness_score || 1.0,
- qualityScore: metadata.quality_score || 1.0,
- lastCrawledAt: metadata.last_crawled_at,
- });
- }
-
- // Sort by combined score
- merged.sort((a, b) => b.score - a.score);
-
- return merged.slice(0, limit);
-}
-
-/**
- * Keyword-only search (BM25)
- */
-async function keywordSearch(
- supabase: SupabaseClient,
- query: string,
- limit: number,
- filters: SearchFilters,
- logger: Logger,
-): Promise<SearchResult[]> {
- let queryBuilder = supabase
- .from("search_documents")
- .select(
- "id, resolved_url, title, description, domain, freshness_score, quality_score, last_crawled_at, content_ts",
- )
- .textSearch("content_ts", query, {
- type: "plain",
- config: "english",
- });
-
- // Apply filters
- if (filters.domain) {
- queryBuilder = queryBuilder.eq("domain", filters.domain);
- }
- if (filters.country) {
- queryBuilder = queryBuilder.eq("country", filters.country);
- }
- if (filters.isMobile !== undefined) {
- queryBuilder = queryBuilder.eq("is_mobile", filters.isMobile);
- }
- if (filters.minFreshness) {
- queryBuilder = queryBuilder.gte("freshness_score", filters.minFreshness);
- }
- if (filters.language) {
- queryBuilder = queryBuilder.eq("language", filters.language);
- }
-
- queryBuilder = queryBuilder.limit(limit);
-
- const { data, error } = await queryBuilder;
-
- if (error) {
- logger.error("Keyword search failed", { error: error.message });
- throw new Error(`Keyword search failed: ${error.message}`);
- }
-
- return (data ?? []).map((row: any, index: number) => ({
- documentId: row.id,
- url: row.resolved_url,
- title: row.title,
- description: row.description,
- domain: row.domain,
- score: row.freshness_score * row.quality_score * (1 / (index + 1)),
- bm25Rank: index + 1,
- vectorRank: null,
- freshnessScore: row.freshness_score,
- qualityScore: row.quality_score,
- lastCrawledAt: row.last_crawled_at,
- }));
-}
-
-/**
- * Semantic-only search (vector similarity via Pinecone)
- */
-async function semanticSearch(
- supabase: SupabaseClient,
- query: string,
- limit: number,
- filters: SearchFilters,
- logger: Logger,
-): Promise<SearchResult[]> {
- if (!isEmbeddingEnabled() || !isPineconeEnabled()) {
- logger.warn("Embeddings or Pinecone not enabled, falling back to keyword search");
- return keywordSearch(supabase, query, limit, filters, logger);
- }
-
- // Generate query embedding
- const embResult = await generateEmbedding(query, {}, logger);
- const queryEmbedding = embResult.embedding;
-
- // Search Pinecone with filters
- const pineconeFilter = buildPineconeFilter(filters);
- const vectorResults = await searchPinecone(
- queryEmbedding,
- limit,
- pineconeFilter,
- "documents",
- logger,
- );
-
- if (vectorResults.length === 0) {
- return [];
- }
-
- // Get metadata from Postgres
- const docIds = vectorResults
- .map(r => r.metadata.doc_id)
- .filter(Boolean) as string[];
-
- const { data: metadata, error } = await supabase.rpc("get_documents_metadata", {
- doc_ids: docIds,
- });
-
- if (error) {
- logger.error("Failed to fetch metadata for semantic search", { error: error.message });
- throw new Error(`Metadata fetch failed: ${error.message}`);
- }
-
- // Create metadata map
- const metadataMap = new Map(
- (metadata || []).map((m: any) => [m.doc_id, m])
- );
-
- // Combine Pinecone results with Postgres metadata
- return vectorResults.map((result, index) => {
- const docId = result.metadata.doc_id || result.id;
- const meta: any = metadataMap.get(docId);
-
- if (!meta) {
- // Fallback to Pinecone metadata
- return {
- documentId: docId,
- url: result.metadata.url || "",
- title: result.metadata.title || null,
- description: null,
- domain: result.metadata.domain || "",
- score: result.score * (result.metadata.freshness_score || 1.0) * (result.metadata.quality_score || 1.0),
- bm25Rank: null,
- vectorRank: index + 1,
- freshnessScore: result.metadata.freshness_score || 1.0,
- qualityScore: result.metadata.quality_score || 1.0,
- lastCrawledAt: new Date().toISOString(),
- };
- }
-
- return {
- documentId: docId,
- url: meta.url,
- title: meta.title,
- description: meta.description,
- domain: meta.domain,
- score: result.score * meta.freshness_score * meta.quality_score,
- bm25Rank: null,
- vectorRank: index + 1,
- freshnessScore: meta.freshness_score,
- qualityScore: meta.quality_score,
- lastCrawledAt: meta.last_crawled_at,
- };
- }).filter(r => r.url !== "");
-}
-
-/**
- * Search within chunks (for precise snippet retrieval via Pinecone)
- */
-export async function searchChunks(
- supabase: SupabaseClient,
- query: string,
- limit: number = 20,
- filters: SearchFilters = {},
- logger?: Logger,
-): Promise<Array<{
- chunkId: string;
- documentId: string;
- url: string;
- title: string | null;
- text: string;
- score: number;
- ordinal: number;
-}>> {
- const log = logger ?? _logger.child({ module: "search-chunks" });
-
- // Generate query embedding
- let queryEmbedding: number[] | null = null;
-
- if (isEmbeddingEnabled() && isPineconeEnabled()) {
- try {
- const embResult = await generateEmbedding(query, {}, log);
- queryEmbedding = embResult.embedding;
- } catch (error) {
- log.warn("Failed to generate query embedding for chunk search", {
- error: (error as Error).message,
- });
- }
- }
-
- if (!queryEmbedding) {
- // Fall back to keyword search on chunks (Postgres)
- const { data, error } = await supabase
- .from("search_chunks")
- .select("id, doc_id, text, ordinal")
- .textSearch("text_ts", query, {
- type: "plain",
- config: "english",
- })
- .not("text", "is", null)
- .limit(limit);
-
- if (error) {
- log.error("Chunk keyword search failed", { error: error.message });
- return [];
- }
-
- // Join with documents to get metadata
- const docIds = [...new Set((data ?? []).map(c => c.doc_id))];
- const { data: docsData } = await supabase
- .from("search_documents")
- .select("id, resolved_url, title")
- .in("id", docIds);
-
- const docMap = new Map((docsData ?? []).map(d => [d.id, d]));
-
- return (data ?? []).map((chunk, index) => {
- const doc = docMap.get(chunk.doc_id);
- return {
- chunkId: chunk.id,
- documentId: chunk.doc_id,
- url: doc?.resolved_url ?? "",
- title: doc?.title ?? null,
- text: chunk.text || "",
- score: 1 / (index + 1),
- ordinal: chunk.ordinal,
- };
- });
- }
-
- // Vector chunk search via Pinecone
- const pineconeFilter = buildPineconeFilter(filters);
-
- // Add chunk-specific filter
- const chunkFilter = {
- ...pineconeFilter,
- chunk_ordinal: { $exists: true }, // Only get chunk records, not document records
- };
-
- const vectorResults = await searchPinecone(
- queryEmbedding,
- limit,
- chunkFilter,
- "documents",
- log,
- );
-
- if (vectorResults.length === 0) {
- return [];
- }
-
- // Get chunk data from Postgres (for text and ordinal)
- const docIds = [...new Set(
- vectorResults.map(r => r.metadata.doc_id).filter(Boolean)
- )] as string[];
-
- const { data: chunksData } = await supabase
- .from("search_chunks")
- .select("id, doc_id, text, ordinal")
- .in("doc_id", docIds);
-
- const { data: docsData } = await supabase
- .from("search_documents")
- .select("id, resolved_url, title")
- .in("id", docIds);
-
- const chunkMap = new Map(
- (chunksData ?? [])
- .filter(c => c.doc_id && c.ordinal !== null)
- .map(c => [`${c.doc_id}_${c.ordinal}`, c])
- );
- const docMap = new Map((docsData ?? []).map(d => [d.id, d]));
-
- // Combine Pinecone results with Postgres data
- return vectorResults.map((result, index) => {
- const docId = result.metadata.doc_id;
- const ordinal = result.metadata.chunk_ordinal;
- const chunkKey = `${docId}_${ordinal}`;
-
- const chunk = chunkMap.get(chunkKey);
- const doc = docMap.get(docId);
-
- return {
- chunkId: result.id,
- documentId: docId || "",
- url: doc?.resolved_url || result.metadata.url || "",
- title: doc?.title || result.metadata.title || null,
- text: chunk?.text || "",
- score: result.score,
- ordinal: ordinal ?? 0,
- };
- }).filter(r => r.url !== "");
-}
-
-/**
- * Get search statistics (including Pinecone stats)
- */
-export async function getSearchStats(
- supabase: SupabaseClient,
-): Promise<{
- totalDocuments: number;
- totalChunks: number;
- documentsWithEmbeddings: number;
- chunksWithEmbeddings: number;
- avgFreshness: number;
- avgQuality: number;
- uniqueDomains: number;
- pineconeVectors?: number;
- pineconeIndexFullness?: number;
-}> {
- const { data, error } = await supabase.from("search_index_stats").select("*");
-
- // Get Pinecone stats if available
- let pineconeStats = { vectorCount: 0, indexFullness: 0 };
- if (isPineconeEnabled()) {
- try {
- const { getPineconeStats } = await import("./pinecone-service.js");
- pineconeStats = await getPineconeStats("documents");
- } catch (error) {
- _logger.warn("Failed to get Pinecone stats", {
- error: (error as Error).message,
- });
- }
- }
-
- if (error || !data || data.length === 0) {
- return {
- totalDocuments: 0,
- totalChunks: 0,
- documentsWithEmbeddings: 0,
- chunksWithEmbeddings: 0,
- avgFreshness: 0,
- avgQuality: 0,
- uniqueDomains: 0,
- pineconeVectors: pineconeStats.vectorCount,
- pineconeIndexFullness: pineconeStats.indexFullness,
- };
- }
-
- const stats = data[0];
- return {
- totalDocuments: stats.total_documents ?? 0,
- totalChunks: stats.total_chunks ?? 0,
- documentsWithEmbeddings: stats.documents_in_pinecone ?? 0,
- chunksWithEmbeddings: stats.chunks_in_pinecone ?? 0,
- avgFreshness: stats.avg_freshness ?? 0,
- avgQuality: stats.avg_quality ?? 0,
- uniqueDomains: stats.unique_domains ?? 0,
- pineconeVectors: pineconeStats.vectorCount,
- pineconeIndexFullness: pineconeStats.indexFullness,
- };
-}
-
diff --git a/apps/api/src/lib/search-index/queue.ts b/apps/api/src/lib/search-index/queue.ts
deleted file mode 100644
index a8c6c87b1..000000000
--- a/apps/api/src/lib/search-index/queue.ts
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- * Search Index Queue Processor
- *
- * Handles async indexing jobs using Redis queue.
- * Similar to existing index insert queue, but for search index.
- *
- * Flow:
- * 1. Scraper adds job to queue after successful scrape
- * 2. Background worker picks up jobs in batches
- * 3. Text is chunked and embeddings generated
- * 4. Data is written to search_documents and search_chunks
- */
-
-import { redisEvictConnection } from "../../services/redis";
-import { logger as _logger } from "../logger";
-import { indexDocumentForSearch, type SearchDocumentInput } from "./service";
-import { search_index_supabase_service } from "../../services/search-index-db";
-
-const SEARCH_INDEX_QUEUE_KEY = "search-index-queue";
-const SEARCH_INDEX_BATCH_SIZE = 10; // Smaller batch size due to embedding generation
-
-export interface SearchIndexJob {
- url: string;
- resolvedUrl: string;
- title?: string;
- description?: string;
- markdown: string;
- html: string;
- statusCode: number;
- gcsPath?: string;
- screenshotUrl?: string;
- language?: string;
- country?: string;
- isMobile?: boolean;
-}
-
-/**
- * Add a job to the search index queue
- */
-export async function addSearchIndexJob(job: SearchIndexJob): Promise<void> {
- try {
- await redisEvictConnection.rpush(
- SEARCH_INDEX_QUEUE_KEY,
- JSON.stringify(job),
- );
-
- _logger.debug("Added job to search index queue", {
- url: job.url,
- });
- } catch (error) {
- _logger.error("Failed to add job to search index queue", {
- error: (error as Error).message,
- url: job.url,
- });
- }
-}
-
-/**
- * Get jobs from the queue
- */
-async function getSearchIndexJobs(): Promise<SearchIndexJob[]> {
- try {
- const jobs =
- (await redisEvictConnection.lpop(
- SEARCH_INDEX_QUEUE_KEY,
- SEARCH_INDEX_BATCH_SIZE,
- )) ?? [];
-
- return jobs.map(x => JSON.parse(x) as SearchIndexJob);
- } catch (error) {
- _logger.error("Failed to get jobs from search index queue", {
- error: (error as Error).message,
- });
- return [];
- }
-}
-
-/**
- * Process search index jobs (called by background worker)
- */
-export async function processSearchIndexJobs(): Promise<void> {
- const jobs = await getSearchIndexJobs();
-
- if (jobs.length === 0) {
- return;
- }
-
- _logger.info("Processing search index jobs", {
- jobCount: jobs.length,
- });
-
- let successCount = 0;
- let failedCount = 0;
-
- // Process jobs sequentially to avoid overwhelming embedding API
- for (const job of jobs) {
- try {
- const logger = _logger.child({
- module: "search-index-queue",
- url: job.url,
- });
-
- const result = await indexDocumentForSearch(
- search_index_supabase_service,
- {
- url: job.url,
- resolvedUrl: job.resolvedUrl,
- title: job.title,
- description: job.description,
- markdown: job.markdown,
- html: job.html,
- statusCode: job.statusCode,
- gcsPath: job.gcsPath,
- screenshotUrl: job.screenshotUrl,
- language: job.language,
- country: job.country,
- isMobile: job.isMobile,
- },
- logger,
- );
-
- if (result.error) {
- failedCount++;
- logger.error("Failed to index document for search", {
- error: result.error,
- });
- } else {
- successCount++;
- logger.info("Indexed document for search", {
- documentId: result.documentId,
- chunks: result.chunkCount,
- tokens: result.totalTokens,
- });
- }
- } catch (error) {
- failedCount++;
- _logger.error("Failed to process search index job", {
- error: (error as Error).message,
- url: job.url,
- });
- }
- }
-
- _logger.info("Finished processing search index jobs", {
- total: jobs.length,
- successful: successCount,
- failed: failedCount,
- });
-}
-
-/**
- * Get queue length
- */
-export async function getSearchIndexQueueLength(): Promise<number> {
- try {
- return (await redisEvictConnection.llen(SEARCH_INDEX_QUEUE_KEY)) ?? 0;
- } catch (error) {
- _logger.error("Failed to get search index queue length", {
- error: (error as Error).message,
- });
- return 0;
- }
-}
-
diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.ts
deleted file mode 100644
index 32e07e83a..000000000
--- a/apps/api/src/lib/search-index/service.ts
+++ /dev/null
@@ -1,684 +0,0 @@
-import { SupabaseClient } from "@supabase/supabase-js";
-import { logger as _logger } from "../logger";
-import type { Logger } from "winston";
-import { withSpan, setSpanAttributes } from "../otel-tracer";
-import {
- chunkText,
- cleanMarkdownForIndexing,
- estimateTokenCount,
- type TextChunk,
-} from "./chunker";
-import {
- generateEmbedding,
- generateEmbeddingsBatch,
- isEmbeddingEnabled,
- type EmbeddingResult,
-} from "./embeddings";
-import {
- upsertToPinecone,
- isPineconeEnabled,
- type PineconeRecord,
-} from "./pinecone-service";
-import crypto from "crypto";
-import psl from "psl";
-
-export interface SearchDocumentInput {
- url: string;
- resolvedUrl: string;
- title?: string;
- description?: string;
- markdown: string;
- html: string;
- statusCode: number;
- gcsPath?: string;
- screenshotUrl?: string;
- language?: string;
- country?: string;
- isMobile?: boolean;
-}
-
-export interface SearchIndexResult {
- documentId: string;
- chunkCount: number;
- totalTokens: number;
- embeddingsGenerated: boolean;
- error?: string;
-}
-
-/**
- * Main function: Index a document for search
- */
-export async function indexDocumentForSearch(
- supabase: SupabaseClient,
- input: SearchDocumentInput,
- logger?: Logger,
-): Promise<SearchIndexResult> {
- return await withSpan("firecrawl-index-document-for-search", async span => {
- const log = logger ?? _logger.child({ module: "search-index-service" });
-
- setSpanAttributes(span, {
- "search.url": input.url,
- "search.status_code": input.statusCode,
- "search.is_mobile": input.isMobile ?? false,
- });
-
- try {
- // 1. Extract and clean text from markdown
- const cleanText = cleanMarkdownForIndexing(input.markdown);
-
- if (!cleanText || cleanText.length < 100) {
- log.warn("Document too short to index", {
- url: input.url,
- textLength: cleanText.length,
- });
- return {
- documentId: "",
- chunkCount: 0,
- totalTokens: 0,
- embeddingsGenerated: false,
- error: "Document too short",
- };
- }
-
- setSpanAttributes(span, {
- "search.clean_text_length": cleanText.length,
- "search.clean_text_tokens": estimateTokenCount(cleanText),
- });
-
- // 2. Chunk the text
- const chunks = await chunkText(cleanText, {
- targetTokens: 750,
- minTokens: 600,
- maxTokens: 900,
- overlapTokens: 100,
- preserveStructure: true,
- }, log);
-
- if (chunks.length === 0) {
- log.warn("No chunks generated", { url: input.url });
- return {
- documentId: "",
- chunkCount: 0,
- totalTokens: 0,
- embeddingsGenerated: false,
- error: "No chunks generated",
- };
- }
-
- setSpanAttributes(span, {
- "search.chunk_count": chunks.length,
- "search.total_tokens": chunks.reduce((sum, c) => sum + c.tokenCount, 0),
- });
-
- // 3. Calculate URL hash and domain
- const urlHash = Buffer.from(
- crypto.createHash("sha256").update(normalizeSearchURL(input.url)).digest("hex"),
- "hex",
- );
- const domain = extractDomain(input.resolvedUrl);
-
- // 4. Generate embeddings (if enabled)
- const embeddingsEnabled = isEmbeddingEnabled() && isPineconeEnabled();
- let documentEmbedding: number[] | null = null;
- let chunkEmbeddings: number[][] = [];
- let totalTokens = 0;
- let pineconeRecords: PineconeRecord[] = [];
- let embeddingsActuallyGenerated = false;
-
- if (embeddingsEnabled) {
- try {
- // Generate document-level embedding (from title + description + first chunk)
- const documentText = [
- input.title,
- input.description,
- chunks[0]?.text.slice(0, 500),
- ]
- .filter(Boolean)
- .join(" ");
-
- const docEmbResult = await generateEmbedding(documentText, {}, log);
- documentEmbedding = docEmbResult.embedding;
- totalTokens += docEmbResult.tokens;
-
- // Generate chunk embeddings in batch
- const chunkTexts = chunks.map(c => c.text);
- const batchResult = await generateEmbeddingsBatch(chunkTexts, {
- batchSize: 50,
- }, log);
-
- chunkEmbeddings = batchResult.embeddings;
- totalTokens += batchResult.totalTokens;
-
- // Prepare Pinecone records (document + chunks)
- // Document-level record
- pineconeRecords.push({
- id: `doc_${crypto.randomUUID()}`,
- values: documentEmbedding,
- metadata: {
- url: input.resolvedUrl,
- domain: extractDomain(input.resolvedUrl),
- title: input.title,
- freshness_score: 1.0,
- quality_score: 1.0, // Will update later
- country: input.country,
- is_mobile: input.isMobile ?? false,
- },
- });
-
- // Chunk-level records
- chunks.forEach((chunk, i) => {
- if (chunkEmbeddings[i]) {
- pineconeRecords.push({
- id: `chunk_${crypto.randomUUID()}`,
- values: chunkEmbeddings[i],
- metadata: {
- url: input.resolvedUrl,
- domain: extractDomain(input.resolvedUrl),
- title: input.title,
- freshness_score: 1.0,
- quality_score: 1.0,
- country: input.country,
- is_mobile: input.isMobile ?? false,
- chunk_ordinal: i,
- doc_id: "", // Will update after DB insert
- },
- });
- }
- });
-
- // Mark as successfully generated
- embeddingsActuallyGenerated = true;
-
- setSpanAttributes(span, {
- "search.embeddings_generated": true,
- "search.embedding_tokens": totalTokens,
- "search.pinecone_records": pineconeRecords.length,
- });
- } catch (error) {
- log.error("Failed to generate embeddings", {
- error: (error as Error).message,
- url: input.url,
- });
- setSpanAttributes(span, {
- "search.embeddings_error": true,
- "search.embeddings_error_message": (error as Error).message,
- });
- // Reset flag since embeddings failed
- embeddingsActuallyGenerated = false;
- }
- }
-
- // 5. Create tsvector for full-text search
- const contentTsVector = await createTsVector(supabase, cleanText);
-
- // 6. Upsert document into search_documents
- // Check if document already exists by url_hash
- // Note: url_hash is BYTEA, need to query with proper format
- const urlHashHex = "\\x" + urlHash.toString('hex');
- const { data: existingDocs, error: lookupError } = await supabase
- .from("search_documents")
- .select("id, content_hash")
- .eq("url_hash", urlHashHex)
- .limit(1);
-
- if (lookupError) {
- log.warn("Error looking up existing document, will attempt insert", {
- error: lookupError.message,
- url: input.url,
- });
- }
-
- const existingDoc = existingDocs?.[0];
- const newContentHash = Buffer.from(
- crypto.createHash("sha256").update(cleanText).digest("hex"),
- "hex",
- );
-
- let documentId: string;
- let isUpdate = false;
-
- if (existingDoc) {
- // Document exists - update it
- documentId = existingDoc.id;
- isUpdate = true;
-
- // Delete existing chunks (will be replaced)
- await supabase
- .from("search_chunks")
- .delete()
- .eq("doc_id", documentId);
-
- // Update document
- const { error: updateError } = await supabase
- .from("search_documents")
- .update({
- resolved_url: input.resolvedUrl,
- title: input.title?.slice(0, 200) ?? null,
- description: input.description?.slice(0, 500) ?? null,
- language: input.language ?? "en",
- gcs_path: input.gcsPath ?? null,
- screenshot_url: input.screenshotUrl ?? null,
- content_ts: contentTsVector,
- country: input.country ?? null,
- pinecone_synced: false, // Will sync after successful Pinecone upsert
- is_mobile: input.isMobile ?? false,
- status_code: input.statusCode,
- content_hash: newContentHash,
- freshness_score: 1.0, // Reset to fresh
- quality_score: calculateQualityScore(input, chunks),
- last_crawled_at: new Date().toISOString(),
- updated_at: new Date().toISOString(),
- })
- .eq("id", documentId);
-
- if (updateError) {
- log.error("Failed to update document", {
- error: updateError.message,
- url: input.url,
- documentId,
- });
-
- setSpanAttributes(span, {
- "search.update_error": true,
- "search.update_error_message": updateError.message,
- });
-
- return {
- documentId: "",
- chunkCount: 0,
- totalTokens: 0,
- embeddingsGenerated: false,
- error: updateError.message,
- };
- }
- } else {
- // New document - insert
- documentId = crypto.randomUUID();
-
- const { error: insertError } = await supabase
- .from("search_documents")
- .insert({
- id: documentId,
- url_hash: urlHashHex,
- url: normalizeSearchURL(input.url),
- resolved_url: input.resolvedUrl,
- title: input.title?.slice(0, 200) ?? null,
- description: input.description?.slice(0, 500) ?? null,
- language: input.language ?? "en",
- gcs_path: input.gcsPath ?? null,
- gcp_path_id: input.gcsPath ?? null,
- screenshot_url: input.screenshotUrl ?? null,
- content_ts: contentTsVector,
- domain,
- country: input.country ?? null,
- is_mobile: input.isMobile ?? false,
- status_code: input.statusCode,
- content_hash: newContentHash,
- freshness_score: 1.0,
- quality_score: calculateQualityScore(input, chunks),
- last_crawled_at: new Date().toISOString(),
- pinecone_synced: false,
- });
-
- if (insertError) {
- log.error("Failed to insert document", {
- error: insertError.message,
- url: input.url,
- });
-
- setSpanAttributes(span, {
- "search.insert_error": true,
- "search.insert_error_message": insertError.message,
- });
-
- return {
- documentId: "",
- chunkCount: 0,
- totalTokens: 0,
- embeddingsGenerated: false,
- error: insertError.message,
- };
- }
- }
-
- // 7. Insert chunks into search_chunks
- // Only insert first 2 chunks for snippet display
- // Rest are stored only in Pinecone (saves 80% storage!)
- const chunkInserts = chunks
- .slice(0, 2) // Only first 2 chunks
- .map((chunk, i) => ({
- doc_id: documentId,
- ordinal: i,
- text: chunk.text,
- token_count: chunk.tokenCount,
- char_count: chunk.charCount,
- prev_chunk_id: null,
- pinecone_synced: false,
- }));
-
- const { error: chunkError } = await supabase
- .from("search_chunks")
- .insert(chunkInserts);
-
- if (chunkError) {
- log.error("Failed to insert chunks", {
- error: chunkError.message,
- url: input.url,
- documentId,
- });
-
- // Try to clean up the document
- await supabase.from("search_documents").delete().eq("id", documentId);
-
- return {
- documentId: "",
- chunkCount: 0,
- totalTokens: 0,
- embeddingsGenerated: false,
- error: chunkError.message,
- };
- }
-
- // 8. Upsert to Pinecone (async, don't block)
- if (embeddingsEnabled && pineconeRecords.length > 0) {
- // Update Pinecone records with actual doc_id
- pineconeRecords.forEach(record => {
- if (record.metadata.chunk_ordinal !== undefined) {
- record.metadata.doc_id = documentId;
- }
- });
-
- try {
- await upsertToPinecone(pineconeRecords, "documents", log);
-
- // Mark as synced
- await supabase
- .from("search_documents")
- .update({
- pinecone_synced: true,
- pinecone_synced_at: new Date().toISOString(),
- })
- .eq("id", documentId);
-
- await supabase
- .from("search_chunks")
- .update({ pinecone_synced: true })
- .eq("doc_id", documentId);
-
- log.info("Synced to Pinecone", {
- documentId,
- records: pineconeRecords.length,
- });
- } catch (error) {
- log.error("Failed to sync to Pinecone", {
- error: (error as Error).message,
- documentId,
- });
- // Continue anyway - can retry later
- }
- }
-
- // 9. Update sync state
- await updateSyncState(supabase, {
- documentsIndexed: 1,
- chunksIndexed: Math.min(chunks.length, 2), // Only inserted first 2
- embeddingsGenerated: embeddingsActuallyGenerated ? 1 + chunks.length : 0,
- });
-
- log.info("Document indexed successfully", {
- url: input.url,
- documentId,
- chunks: chunks.length,
- totalTokens,
- embeddingsGenerated: embeddingsActuallyGenerated,
- isUpdate,
- });
-
- setSpanAttributes(span, {
- "search.success": true,
- "search.document_id": documentId,
- "search.is_update": isUpdate,
- });
-
- return {
- documentId,
- chunkCount: chunks.length,
- totalTokens,
- embeddingsGenerated: embeddingsActuallyGenerated,
- };
- } catch (error) {
- log.error("Failed to index document", {
- error: (error as Error).message,
- url: input.url,
- });
-
- setSpanAttributes(span, {
- "search.error": true,
- "search.error_message": (error as Error).message,
- });
-
- return {
- documentId: "",
- chunkCount: 0,
- totalTokens: 0,
- embeddingsGenerated: false,
- error: (error as Error).message,
- };
- }
- });
-}
-
-/**
- * Normalize URL for search index (remove query params, fragments, etc.)
- * Preserves the original protocol (http/https) to avoid collisions and maintain reachability
- */
-function normalizeSearchURL(url: string): string {
- try {
- const urlObj = new URL(url);
- urlObj.hash = "";
- urlObj.search = ""; // Remove query params for canonical URL
- // Note: Preserve original protocol (don't force https) to avoid:
- // 1. Making HTTP-only pages unreachable
- // 2. Collisions between distinct HTTP/HTTPS resources
-
- if (urlObj.hostname.startsWith("www.")) {
- urlObj.hostname = urlObj.hostname.slice(4);
- }
-
- if (urlObj.pathname.endsWith("/")) {
- urlObj.pathname = urlObj.pathname.slice(0, -1);
- }
-
- return urlObj.toString();
- } catch {
- return url;
- }
-}
-
-/**
- * Extract domain from URL
- */
-function extractDomain(url: string): string {
- try {
- const urlObj = new URL(url);
- const parsed = psl.parse(urlObj.hostname);
-
- if (parsed.domain) {
- return parsed.domain;
- }
-
- return urlObj.hostname;
- } catch {
- return "";
- }
-}
-
-/**
- * Calculate quality score based on content characteristics
- */
-function calculateQualityScore(
- input: SearchDocumentInput,
- chunks: TextChunk[],
-): number {
- let score = 1.0;
-
- // Factor 1: Has title and description
- if (input.title && input.title.length > 10) score *= 1.1;
- if (input.description && input.description.length > 50) score *= 1.1;
-
- // Factor 2: Content length (sweet spot: 2000-10000 chars)
- const totalChars = chunks.reduce((sum, c) => sum + c.charCount, 0);
- if (totalChars >= 2000 && totalChars <= 10000) {
- score *= 1.2;
- } else if (totalChars < 500) {
- score *= 0.5;
- } else if (totalChars > 50000) {
- score *= 0.8;
- }
-
- // Factor 3: Chunk count (good chunking indicates structured content)
- if (chunks.length >= 3 && chunks.length <= 20) {
- score *= 1.1;
- }
-
- // Factor 4: Status code
- if (input.statusCode >= 200 && input.statusCode < 300) {
- score *= 1.0;
- } else {
- score *= 0.3;
- }
-
- // Normalize to [0, 1]
- return Math.min(Math.max(score, 0.1), 2.0);
-}
-
-/**
- * Create tsvector from text (for BM25 search)
- */
-async function createTsVector(
- supabase: SupabaseClient,
- text: string,
-): Promise<string | null> {
- try {
- const { data, error } = await supabase.rpc("to_tsvector_english", {
- text: text.slice(0, 100000), // Limit to 100k chars
- });
-
- if (error) {
- _logger.warn("Failed to create tsvector", { error: error.message });
- return null;
- }
-
- return data;
- } catch (error) {
- _logger.warn("Failed to create tsvector", {
- error: (error as Error).message,
- });
- return null;
- }
-}
-
-/**
- * Update sync state with metrics
- */
-async function updateSyncState(
- supabase: SupabaseClient,
- metrics: {
- documentsIndexed?: number;
- chunksIndexed?: number;
- embeddingsGenerated?: number;
- failedDocuments?: number;
- },
-): Promise<void> {
- try {
- const updates: any = {
- updated_at: new Date().toISOString(),
- };
-
- if (metrics.documentsIndexed) {
- // Use SQL to increment
- await supabase.rpc("increment_search_sync_state", {
- field: "total_documents_indexed",
- amount: metrics.documentsIndexed,
- });
- }
-
- if (metrics.chunksIndexed) {
- await supabase.rpc("increment_search_sync_state", {
- field: "total_chunks_indexed",
- amount: metrics.chunksIndexed,
- });
- }
-
- if (metrics.failedDocuments) {
- await supabase.rpc("increment_search_sync_state", {
- field: "failed_documents",
- amount: metrics.failedDocuments,
- });
- }
-
- // Update embedding quota tracking
- const today = new Date().toISOString().split("T")[0];
- await supabase.rpc("update_embedding_quota", {
- embeddings_count: metrics.embeddingsGenerated ?? 0,
- today_date: today,
- });
- } catch (error) {
- _logger.warn("Failed to update sync state", {
- error: (error as Error).message,
- });
- }
-}
-
-/**
- * Delete document from search index
- */
-export async function deleteDocumentFromSearch(
- supabase: SupabaseClient,
- urlHash: Buffer,
- logger?: Logger,
-): Promise<void> {
- const log = logger ?? _logger.child({ module: "search-index-service" });
-
- try {
- const urlHashHex = "\\x" + urlHash.toString('hex');
- const { error } = await supabase
- .from("search_documents")
- .delete()
- .eq("url_hash", urlHashHex);
-
- if (error) {
- log.error("Failed to delete document from search index", {
- error: error.message,
- });
- }
- } catch (error) {
- log.error("Failed to delete document from search index", {
- error: (error as Error).message,
- });
- }
-}
-
-/**
- * Check if document already exists in search index
- */
-export async function searchDocumentExists(
- supabase: SupabaseClient,
- urlHash: Buffer,
-): Promise<boolean> {
- try {
- const urlHashHex = "\\x" + urlHash.toString('hex');
- const { data, error } = await supabase
- .from("search_documents")
- .select("id")
- .eq("url_hash", urlHashHex)
- .limit(1);
-
- if (error) return false;
-
- return (data?.length ?? 0) > 0;
- } catch {
- return false;
- }
-}
-