Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolasaebc2f0c6374
diff --git a/apps/api/src/lib/search-index/__tests__/chunker.test.ts b/apps/api/src/lib/search-index/__tests__/chunker.test.tsdeleted file mode 100644index 1b2f8c26b..000000000--- a/apps/api/src/lib/search-index/__tests__/chunker.test.ts+++ /dev/null@@ -1,474 +0,0 @@-import {- chunkText,- cleanMarkdownForIndexing,- estimateTokenCount,- type TextChunk,- type ChunkingOptions,-} from "../chunker";--describe("Chunker", () => {- describe("estimateTokenCount", () => {- it("should estimate token count for simple text", () => {- const text = "Hello world";- const tokens = estimateTokenCount(text);- expect(tokens).toBeGreaterThan(0);- expect(tokens).toBeLessThan(10);- });-- it("should estimate token count for longer text", () => {- const text = "This is a longer piece of text that should have more tokens than the previous example.";- const tokens = estimateTokenCount(text);- expect(tokens).toBeGreaterThan(10);- expect(tokens).toBeLessThan(50);- });-- it("should handle empty string", () => {- const tokens = estimateTokenCount("");- expect(tokens).toBe(0);- });-- it("should normalize whitespace before counting", () => {- const text1 = "Hello world";- const text2 = "Hello world";- expect(estimateTokenCount(text1)).toBe(estimateTokenCount(text2));- });- });-- describe("cleanMarkdownForIndexing", () => {- it("should remove code blocks", () => {- const markdown = "Some text\n```javascript\nconst x = 1;\n```\nMore text";- const cleaned = cleanMarkdownForIndexing(markdown);- expect(cleaned).not.toContain("```");- expect(cleaned).toContain("Some text");- expect(cleaned).toContain("More text");- });-- it("should remove inline code but keep content", () => {- const markdown = "This is `inline code` in text";- const cleaned = cleanMarkdownForIndexing(markdown);- expect(cleaned).not.toContain("`");- expect(cleaned).toContain("inline code");- });-- it("should remove images but keep alt text", () => {- const markdown = "Check this  out";- const cleaned = cleanMarkdownForIndexing(markdown);- expect(cleaned).toContain("alt text");- expect(cleaned).not.toContain(";- });-- it("should remove links but keep text", () => {- const markdown = "Visit [Google](https://google.com) for search";- const cleaned = cleanMarkdownForIndexing(markdown);- expect(cleaned).toContain("Google");- expect(cleaned).not.toContain("[");- expect(cleaned).not.toContain("](");- });-- it("should remove headers but keep text", () => {- const markdown = "# Heading 1\n## Heading 2\n### Heading 3";- const cleaned = cleanMarkdownForIndexing(markdown);- expect(cleaned).toContain("Heading 1");- expect(cleaned).toContain("Heading 2");- expect(cleaned).toContain("Heading 3");- expect(cleaned).not.toContain("#");- });-- it("should remove bold and italic markers", () => {- const markdown = "This is **bold** and *italic* text";- const cleaned = cleanMarkdownForIndexing(markdown);- expect(cleaned).toContain("bold");- expect(cleaned).toContain("italic");- expect(cleaned).not.toContain("**");- expect(cleaned).not.toContain("*");- });-- it("should remove list markers", () => {- const markdown = "- Item 1\n- Item 2\n1. Ordered item";- const cleaned = cleanMarkdownForIndexing(markdown);- expect(cleaned).toContain("Item 1");- expect(cleaned).toContain("Item 2");- expect(cleaned).toContain("Ordered item");- expect(cleaned).not.toMatch(/^[-*+]\s/m);- expect(cleaned).not.toMatch(/^\d+\.\s/m);- });-- it("should handle empty input", () => {- expect(cleanMarkdownForIndexing("")).toBe("");- expect(cleanMarkdownForIndexing(null as any)).toBe("");- });-- it("should normalize excessive whitespace", () => {- const markdown = "Text\n\n\n\nwith\n\n\n\nmany\n\n\n\nlines";- const cleaned = cleanMarkdownForIndexing(markdown);- expect(cleaned).not.toMatch(/\n{3,}/);- });- });-- describe("chunkText", () => {- describe("basic functionality", () => {- it("should return empty array for empty text", async () => {- const chunks = await chunkText("");- expect(chunks).toEqual([]);- });-- it("should return single chunk for short text", async () => {- const text = "This is a short piece of text.";- const chunks = await chunkText(text);- expect(chunks).toHaveLength(1);- expect(chunks[0].text).toBe(text.trim());- expect(chunks[0].ordinal).toBe(0);- });-- it("should split long text into multiple chunks", async () => {- // Create a text that exceeds max tokens- const longText = Array(200)- .fill("This is a sentence with multiple words.")- .join(" ");- const chunks = await chunkText(longText, { maxTokens: 500 });- expect(chunks.length).toBeGreaterThan(1);- });-- it("should respect maxTokens limit", async () => {- const longText = Array(100)- .fill("This is a sentence.")- .join(" ");- const maxTokens = 300;- const chunks = await chunkText(longText, { maxTokens });-- chunks.forEach(chunk => {- expect(chunk.tokenCount).toBeLessThanOrEqual(maxTokens);- });- });- });-- describe("chunk metadata", () => {- it("should set correct ordinals", async () => {- const longText = Array(100)- .fill("This is a sentence.")- .join(" ");- const chunks = await chunkText(longText, { maxTokens: 300 });-- chunks.forEach((chunk, index) => {- expect(chunk.ordinal).toBe(index);- });- });-- it("should calculate token counts", async () => {- const text = "This is test text.";- const chunks = await chunkText(text);-- expect(chunks[0].tokenCount).toBeGreaterThan(0);- expect(chunks[0].tokenCount).toBe(estimateTokenCount(text.trim()));- });-- it("should calculate character counts", async () => {- const text = "This is test text.";- const chunks = await chunkText(text);-- expect(chunks[0].charCount).toBe(text.length);- });-- it("should set start and end offsets", async () => {- const text = "First sentence. Second sentence.";- const chunks = await chunkText(text);-- expect(chunks[0].startOffset).toBe(0);- expect(chunks[0].endOffset).toBeGreaterThan(0);- expect(chunks[0].endOffset).toBeLessThanOrEqual(text.length);- });- });-- describe("offset tracking", () => {- it("should track offsets correctly for multiple chunks", async () => {- const longText = Array(100)- .fill("This is a sentence with several words.")- .join(" ");- const chunks = await chunkText(longText, { maxTokens: 300 });-- // First chunk should start at 0- expect(chunks[0].startOffset).toBe(0);-- // Each chunk's offsets should be sequential (allowing for overlap)- for (let i = 1; i < chunks.length; i++) {- expect(chunks[i].startOffset).toBeGreaterThanOrEqual(0);- expect(chunks[i].endOffset).toBeGreaterThan(chunks[i].startOffset);- }- });-- it("should maintain consistent offsets in structure-preserving mode", async () => {- const text = "# Heading\n\nParagraph one. " +- Array(50).fill("More text.").join(" ") +- "\n\n# Another Heading\n\nParagraph two.";-- const chunks = await chunkText(text, {- maxTokens: 300,- preserveStructure: true- });-- chunks.forEach(chunk => {- expect(chunk.startOffset).toBeGreaterThanOrEqual(0);- expect(chunk.endOffset).toBeGreaterThan(chunk.startOffset);- expect(chunk.endOffset).toBeLessThanOrEqual(text.length + 100); // Allow some margin for processing- });- });-- it("should maintain consistent offsets in sentence-based mode", async () => {- const text = Array(80)- .fill("This is a sentence.")- .join(" ");-- const chunks = await chunkText(text, {- maxTokens: 300,- preserveStructure: false- });-- chunks.forEach(chunk => {- expect(chunk.startOffset).toBeGreaterThanOrEqual(0);- expect(chunk.endOffset).toBeGreaterThan(chunk.startOffset);- });- });- });-- describe("chunking strategies", () => {- it("should preserve markdown structure when enabled", async () => {- const text =- "# Heading 1\n\nFirst paragraph.\n\n" +- "## Heading 2\n\nSecond paragraph with more content.\n\n" +- "- List item 1\n- List item 2\n\n" +- "Final paragraph.";-- const chunks = await chunkText(text, {- preserveStructure: true,- maxTokens: 200- });-- expect(chunks.length).toBeGreaterThan(0);- // Structure preservation should try to keep sections together- expect(chunks.some(c => c.text.includes("Heading"))).toBe(true);- });-- it("should use sentence-based chunking when structure preservation is disabled", async () => {- const text = Array(50)- .fill("This is a sentence. Another sentence here.")- .join(" ");-- const chunks = await chunkText(text, {- preserveStructure: false,- maxTokens: 300- });-- expect(chunks.length).toBeGreaterThan(1);- chunks.forEach(chunk => {- expect(chunk.text.length).toBeGreaterThan(0);- });- });-- it("should handle large sections that need splitting", async () => {- const largeSection = Array(100)- .fill("Sentence in a large section.")- .join(" ");- const text = `# Large Section\n\n${largeSection}`;-- const chunks = await chunkText(text, {- preserveStructure: true,- maxTokens: 300- });-- expect(chunks.length).toBeGreaterThan(1);- });- });-- describe("overlap handling", () => {- it("should create overlapping chunks", async () => {- const longText = Array(80)- .fill("This is a unique sentence number.")- .join(" ");-- const chunks = await chunkText(longText, {- maxTokens: 300,- overlapTokens: 50- });-- if (chunks.length > 1) {- // Chunks should have some overlap in content- expect(chunks.length).toBeGreaterThan(1);- // Each chunk should exist and have content- chunks.forEach(chunk => {- expect(chunk.text.length).toBeGreaterThan(0);- });- }- });-- it("should apply overlap correctly with proper context", async () => {- const sentences = Array(60)- .fill(0)- .map((_, i) => `Sentence number ${i}.`)- .join(" ");-- const chunks = await chunkText(sentences, {- maxTokens: 200,- overlapTokens: 30- });-- if (chunks.length > 1) {- // Verify chunks are created with proper content- chunks.forEach((chunk, i) => {- expect(chunk.ordinal).toBe(i);- expect(chunk.text.length).toBeGreaterThan(0);- });- }- });- });-- describe("edge cases", () => {- it("should handle text with only whitespace", async () => {- const chunks = await chunkText(" \n\n ");- expect(chunks).toEqual([]);- });-- it("should handle text with special characters", async () => {- const text = "Text with émojis 🔥 and spëcial çharacters!";- const chunks = await chunkText(text);-- expect(chunks).toHaveLength(1);- expect(chunks[0].text).toContain("émojis");- expect(chunks[0].text).toContain("🔥");- });-- it("should handle very long single sentences", async () => {- // A very long single sentence cannot be split at sentence boundaries- // The chunker will keep it as one chunk even if it exceeds maxTokens- const longSentence = "This is a very long sentence that " +- Array(200).fill("keeps going and going").join(" ") +- ".";-- const chunks = await chunkText(longSentence, { maxTokens: 500 });-- // Should create at least one chunk- expect(chunks.length).toBeGreaterThan(0);- // When a single sentence is too long, it stays as one chunk- // This is expected behavior to maintain sentence integrity- expect(chunks[0].text).toContain("very long sentence");- });-- it("should handle text with multiple consecutive newlines", async () => {- const text = "First paragraph.\n\n\n\nSecond paragraph.";- const chunks = await chunkText(text);-- expect(chunks.length).toBeGreaterThan(0);- expect(chunks[0].text).toContain("First paragraph");- });-- it("should handle mixed content types", async () => {- const text =- "# Heading\n\n" +- "Paragraph with text.\n\n" +- "```code\nconst x = 1;\n```\n\n" +- "- List item\n" +- "- Another item\n\n" +- "Final paragraph.";-- const chunks = await chunkText(text);- expect(chunks.length).toBeGreaterThan(0);- });- });-- describe("options validation", () => {- it("should use default options when none provided", async () => {- const text = Array(50).fill("Text here.").join(" ");- const chunks = await chunkText(text);-- expect(chunks).toBeDefined();- expect(Array.isArray(chunks)).toBe(true);- });-- it("should respect custom targetTokens", async () => {- const text = Array(100).fill("Text here.").join(" ");- const chunks = await chunkText(text, { targetTokens: 200 });-- expect(chunks.length).toBeGreaterThan(0);- });-- it("should respect custom minTokens", async () => {- const text = Array(100).fill("Text here.").join(" ");- const chunks = await chunkText(text, { minTokens: 100 });-- expect(chunks.length).toBeGreaterThan(0);- });-- it("should respect custom overlapTokens", async () => {- const text = Array(100).fill("Text here.").join(" ");- const chunks = await chunkText(text, {- maxTokens: 300,- overlapTokens: 75- });-- expect(chunks.length).toBeGreaterThan(0);- });- });-- describe("chunk quality", () => {- it("should not create empty chunks", async () => {- const text = Array(100).fill("Content here.").join(" ");- const chunks = await chunkText(text, { maxTokens: 300 });-- chunks.forEach(chunk => {- expect(chunk.text.trim().length).toBeGreaterThan(0);- expect(chunk.tokenCount).toBeGreaterThan(0);- expect(chunk.charCount).toBeGreaterThan(0);- });- });-- it("should maintain text integrity", async () => {- const text = "This is important content that must be preserved.";- const chunks = await chunkText(text);-- const reconstructed = chunks.map(c => c.text).join(" ");- expect(reconstructed).toContain("important content");- });-- it("should create reasonably sized chunks", async () => {- const text = Array(100).fill("Some text.").join(" ");- const maxTokens = 400;- const chunks = await chunkText(text, { maxTokens });-- chunks.forEach(chunk => {- expect(chunk.tokenCount).toBeGreaterThan(0);- expect(chunk.tokenCount).toBeLessThanOrEqual(maxTokens);- });- });- });-- describe("performance", () => {- it("should handle large documents efficiently", async () => {- const largeText = Array(500)- .fill("This is a sentence in a large document.")- .join(" ");-- const startTime = Date.now();- const chunks = await chunkText(largeText, { maxTokens: 500 });- const duration = Date.now() - startTime;-- expect(chunks.length).toBeGreaterThan(0);- expect(duration).toBeLessThan(5000); // Should complete in under 5 seconds- });-- it("should handle documents with complex structure", async () => {- const sections = Array(20).fill(0).map((_, i) =>- `# Section ${i}\n\nContent for section ${i}.\n\n` +- Array(10).fill("More text here.").join(" ")- ).join("\n\n");-- const chunks = await chunkText(sections, {- preserveStructure: true,- maxTokens: 500- });-- expect(chunks.length).toBeGreaterThan(0);- });- });- });-});-diff --git a/apps/api/src/lib/search-index/chunker.ts b/apps/api/src/lib/search-index/chunker.tsdeleted file mode 100644index 4c8512c4b..000000000--- a/apps/api/src/lib/search-index/chunker.ts+++ /dev/null@@ -1,374 +0,0 @@-import { logger as _logger } from "../logger";-import type { Logger } from "winston";--export interface TextChunk {- text: string;- ordinal: number;- tokenCount: number;- charCount: number;- startOffset: number;- endOffset: number;-}--export interface ChunkingOptions {- targetTokens?: number; // Target tokens per chunk (default: 750)- minTokens?: number; // Minimum tokens per chunk (default: 600)- maxTokens?: number; // Maximum tokens per chunk (default: 900)- overlapTokens?: number; // Overlap between chunks (default: 100)- preserveStructure?: boolean; // Preserve markdown structure (default: true)-}--const DEFAULT_OPTIONS: Required<ChunkingOptions> = {- targetTokens: 750,- minTokens: 600,- maxTokens: 900,- overlapTokens: 100,- preserveStructure: true,-};--/**- * Estimate token count using simple heuristic (1 token ≈ 4 chars for English)- * This is approximate but fast. For exact counts, use tiktoken.- */-export function estimateTokenCount(text: string): number {- // Remove extra whitespace- const normalized = text.replace(/\s+/g, " ").trim();- // Rough approximation: 1 token ≈ 4 characters- return Math.ceil(normalized.length / 4);-}--/**- * Split text into sentences, preserving structure- */-function splitIntoSentences(text: string): string[] {- // Split on sentence boundaries, but preserve structure- const sentences: string[] = [];-- // Match sentences ending with .!? followed by space/newline/end- // Also handle abbreviations like "Dr." "Mr." "etc."- const sentenceRegex = /(?<![A-Z][a-z]\.)\s*([.!?]+)\s+(?=[A-Z]|$)/g;-- let lastIndex = 0;- let match: RegExpExecArray | null;-- while ((match = sentenceRegex.exec(text)) !== null) {- const sentence = text.slice(lastIndex, match.index + match[1].length).trim();- if (sentence.length > 0) {- sentences.push(sentence);- }- lastIndex = match.index + match[0].length;- }-- // Add remaining text- if (lastIndex < text.length) {- const remaining = text.slice(lastIndex).trim();- if (remaining.length > 0) {- sentences.push(remaining);- }- }-- return sentences.filter(s => s.length > 0);-}--/**- * Detect markdown structure boundaries (headings, lists, code blocks)- */-function detectStructureBoundaries(text: string): number[] {- const boundaries: number[] = [0];-- const lines = text.split("\n");- let offset = 0;-- for (let i = 0; i < lines.length; i++) {- const line = lines[i];-- // Detect headings- if (/^#{1,6}\s/.test(line)) {- boundaries.push(offset);- }-- // Detect code blocks- if (/^```/.test(line)) {- boundaries.push(offset);- }-- // Detect list items (with proper structure)- if (/^[\s]*[-*+]\s/.test(line) || /^[\s]*\d+\.\s/.test(line)) {- boundaries.push(offset);- }-- offset += line.length + 1; // +1 for newline- }-- boundaries.push(text.length);-- return [...new Set(boundaries)].sort((a, b) => a - b);-}--/**- * Main chunking function: splits text into semantically coherent chunks- */-export async function chunkText(- text: string,- options: ChunkingOptions = {},- logger?: Logger,-): Promise<TextChunk[]> {- const opts = { ...DEFAULT_OPTIONS, ...options };- const log = logger ?? _logger.child({ module: "search-chunker" });-- // Handle empty or very short text- if (!text || text.trim().length === 0) {- return [];- }-- const totalTokens = estimateTokenCount(text);-- // If text is short enough, return as single chunk- if (totalTokens <= opts.maxTokens) {- return [{- text: text.trim(),- ordinal: 0,- tokenCount: totalTokens,- charCount: text.length,- startOffset: 0,- endOffset: text.length,- }];- }-- const chunks: TextChunk[] = [];-- // Strategy 1: If preserving structure, respect markdown boundaries- if (opts.preserveStructure) {- const boundaries = detectStructureBoundaries(text);- const sections = boundaries.slice(0, -1).map((start, i) => ({- text: text.slice(start, boundaries[i + 1]),- offset: start,- }));-- let currentChunk = "";- let currentOffset = 0;- let currentTokens = 0;-- for (const section of sections) {- const sectionTokens = estimateTokenCount(section.text);-- // If section alone exceeds max, split it further- if (sectionTokens > opts.maxTokens) {- // Flush current chunk if any- if (currentChunk.length > 0) {- chunks.push({- text: currentChunk.trim(),- ordinal: chunks.length,- tokenCount: currentTokens,- charCount: currentChunk.length,- startOffset: currentOffset,- endOffset: currentOffset + currentChunk.length,- });- currentChunk = "";- currentTokens = 0;- }-- // Split large section by sentences- const sentences = splitIntoSentences(section.text);- let sentenceChunk = "";- let sentenceTokens = 0;- let sentenceChunkOffset = section.offset;-- for (const sentence of sentences) {- const sentenceTokenCount = estimateTokenCount(sentence);-- if (sentenceTokens + sentenceTokenCount > opts.maxTokens && sentenceChunk.length > 0) {- // Save overlap from the previous chunk before flushing- const overlapText = sentenceChunk.slice(-opts.overlapTokens * 4); // Rough char estimate- const chunkLength = sentenceChunk.length;-- chunks.push({- text: sentenceChunk.trim(),- ordinal: chunks.length,- tokenCount: sentenceTokens,- charCount: chunkLength,- startOffset: sentenceChunkOffset,- endOffset: sentenceChunkOffset + chunkLength,- });-- // Update offset for next chunk - account for overlap- // The next chunk will start with overlapText, so its offset should point- // to where the overlap begins in the source (not where previous chunk ended)- sentenceChunkOffset += chunkLength - overlapText.length;-- // Start new chunk with overlap from previous chunk + new sentence- sentenceChunk = overlapText + " " + sentence;- sentenceTokens = estimateTokenCount(sentenceChunk);- } else {- sentenceChunk += (sentenceChunk.length > 0 ? " " : "") + sentence;- sentenceTokens += sentenceTokenCount;- }- }-- if (sentenceChunk.length > 0) {- chunks.push({- text: sentenceChunk.trim(),- ordinal: chunks.length,- tokenCount: sentenceTokens,- charCount: sentenceChunk.length,- startOffset: sentenceChunkOffset,- endOffset: sentenceChunkOffset + sentenceChunk.length,- });- }-- currentOffset = section.offset + section.text.length;- currentChunk = "";- currentTokens = 0;- continue;- }-- // Try to add section to current chunk- if (currentTokens + sectionTokens <= opts.maxTokens) {- currentChunk += (currentChunk.length > 0 ? "\n" : "") + section.text;- currentTokens += sectionTokens;- if (currentChunk.length === section.text.length) {- currentOffset = section.offset;- }- } else {- // Current chunk is full, flush it- if (currentChunk.length > 0) {- chunks.push({- text: currentChunk.trim(),- ordinal: chunks.length,- tokenCount: currentTokens,- charCount: currentChunk.length,- startOffset: currentOffset,- endOffset: currentOffset + currentChunk.length,- });- }-- // Start new chunk with current section- currentChunk = section.text;- currentOffset = section.offset;- currentTokens = sectionTokens;- }- }-- // Flush remaining chunk- if (currentChunk.length > 0) {- chunks.push({- text: currentChunk.trim(),- ordinal: chunks.length,- tokenCount: currentTokens,- charCount: currentChunk.length,- startOffset: currentOffset,- endOffset: currentOffset + currentChunk.length,- });- }- } else {- // Strategy 2: Simple sentence-based chunking without structure preservation- const sentences = splitIntoSentences(text);- let currentChunk = "";- let currentTokens = 0;- let currentOffset = 0;-- for (const sentence of sentences) {- const sentenceTokens = estimateTokenCount(sentence);-- if (currentTokens + sentenceTokens > opts.maxTokens && currentChunk.length > 0) {- const chunkLength = currentChunk.length;- chunks.push({- text: currentChunk.trim(),- ordinal: chunks.length,- tokenCount: currentTokens,- charCount: chunkLength,- startOffset: currentOffset,- endOffset: currentOffset + chunkLength,- });-- // Add overlap for context- const lastSentences = currentChunk.split(/[.!?]+\s+/).slice(-2).join(". ");-- // Update offset for next chunk - account for overlap- // The next chunk will start with lastSentences, so its offset should point- // to where the overlap begins in the source (not where previous chunk ended)- currentOffset += chunkLength - lastSentences.length;-- currentChunk = lastSentences + " " + sentence;- currentTokens = estimateTokenCount(currentChunk);- } else {- currentChunk += (currentChunk.length > 0 ? " " : "") + sentence;- currentTokens += sentenceTokens;- }-- if (chunks.length === 0 && currentChunk === sentence) {- currentOffset = 0;- }- }-- // Flush remaining- if (currentChunk.length > 0) {- chunks.push({- text: currentChunk.trim(),- ordinal: chunks.length,- tokenCount: currentTokens,- charCount: currentChunk.length,- startOffset: currentOffset,- endOffset: currentOffset + currentChunk.length,- });- }- }-- log.debug("Chunked text", {- totalTokens,- chunks: chunks.length,- avgTokensPerChunk: chunks.length > 0- ? Math.round(chunks.reduce((sum, c) => sum + c.tokenCount, 0) / chunks.length)- : 0,- });-- return chunks;-}--/**- * Extract clean text from markdown for indexing- * Removes markdown syntax but preserves structure and content- */-export function cleanMarkdownForIndexing(markdown: string): string {- if (!markdown) return "";-- let text = markdown;-- // Remove code blocks but keep language hints- text = text.replace(/```(\w+)?\n([\s\S]*?)```/g, (_, lang, code) => {- return lang ? `Code (${lang}): ${code.split("\n").slice(0, 3).join(" ")}...` : "";- });-- // Remove inline code (keep content)- text = text.replace(/`([^`]+)`/g, "$1");-- // Remove images (keep alt text)- text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1");-- // Remove links (keep text)- text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");-- // Remove headers (#) but keep text- text = text.replace(/^#{1,6}\s+(.+)$/gm, "$1");-- // Remove bold/italic- text = text.replace(/(\*\*|__)(.*?)\1/g, "$2");- text = text.replace(/(\*|_)(.*?)\1/g, "$2");-- // Remove list markers but keep structure- text = text.replace(/^[\s]*[-*+]\s+/gm, "");- text = text.replace(/^[\s]*\d+\.\s+/gm, "");-- // Remove blockquotes- text = text.replace(/^>\s+/gm, "");-- // Remove horizontal rules- text = text.replace(/^[-*_]{3,}$/gm, "");-- // Normalize whitespace- text = text.replace(/\n{3,}/g, "\n\n");- text = text.replace(/ +/g, " ");-- return text.trim();-}-diff --git a/apps/api/src/lib/search-index/embeddings.ts b/apps/api/src/lib/search-index/embeddings.tsdeleted file mode 100644index 5c8bde8cb..000000000--- a/apps/api/src/lib/search-index/embeddings.ts+++ /dev/null@@ -1,291 +0,0 @@-import { embed, EmbeddingModel } from "ai";-import { getEmbeddingModel } from "../generic-ai";-import { logger as _logger } from "../logger";-import type { Logger } from "winston";-import { withSpan, setSpanAttributes } from "../otel-tracer";--export interface EmbeddingResult {- embedding: number[];- tokens: number;-}--export interface BatchEmbeddingResult {- embeddings: number[][];- totalTokens: number;- failed: number[];-}--export interface EmbeddingOptions {- model?: string;- dimensions?: number;- maxRetries?: number;- retryDelay?: number;- batchSize?: number;-}--const DEFAULT_OPTIONS: Required<EmbeddingOptions> = {- model: "text-embedding-3-small",- dimensions: 1536,- maxRetries: 3,- retryDelay: 1000, // ms- batchSize: 100,-};--/**- * Generate embedding for a single text- */-export async function generateEmbedding(- text: string,- options: EmbeddingOptions = {},- logger?: Logger,-): Promise<EmbeddingResult> {- return await withSpan("firecrawl-generate-embedding", async span => {- const opts = { ...DEFAULT_OPTIONS, ...options };- const log = logger ?? _logger.child({ module: "search-embeddings" });-- setSpanAttributes(span, {- "embedding.model": opts.model,- "embedding.dimensions": opts.dimensions,- "embedding.text_length": text.length,- });-- if (!text || text.trim().length === 0) {- setSpanAttributes(span, { "embedding.empty_text": true });- return {- embedding: new Array(opts.dimensions).fill(0),- tokens: 0,- };- }-- // Truncate if too long (OpenAI limit: 8191 tokens for text-embedding-3-small)- const maxChars = 8191 * 4; // Rough approximation- const truncatedText = text.length > maxChars ? text.slice(0, maxChars) : text;-- let lastError: Error | null = null;-- for (let attempt = 0; attempt < opts.maxRetries; attempt++) {- try {- const embeddingModel: EmbeddingModel<string> = getEmbeddingModel(- opts.model,- "openai",- );-- const result = await embed({- model: embeddingModel,- value: truncatedText,- });-- setSpanAttributes(span, {- "embedding.success": true,- "embedding.tokens": result.usage.tokens,- "embedding.attempt": attempt + 1,- });-- log.debug("Generated embedding", {- textLength: text.length,- tokens: result.usage.tokens,- dimensions: result.embedding.length,- });-- return {- embedding: result.embedding,- tokens: result.usage.tokens,- };- } catch (error) {- lastError = error as Error;-- setSpanAttributes(span, {- "embedding.error": true,- "embedding.attempt": attempt + 1,- "embedding.error_message": lastError.message,- });-- log.warn("Failed to generate embedding", {- error: lastError.message,- attempt: attempt + 1,- maxRetries: opts.maxRetries,- });-- // Exponential backoff- if (attempt < opts.maxRetries - 1) {- const delay = opts.retryDelay * Math.pow(2, attempt);- await new Promise(resolve => setTimeout(resolve, delay));- }- }- }-- log.error("Failed to generate embedding after all retries", {- error: lastError?.message,- maxRetries: opts.maxRetries,- });-- // Return zero vector as fallback- return {- embedding: new Array(opts.dimensions).fill(0),- tokens: 0,- };- });-}--/**- * Generate embeddings for multiple texts in batches- * More efficient for bulk operations- */-export async function generateEmbeddingsBatch(- texts: string[],- options: EmbeddingOptions = {},- logger?: Logger,-): Promise<BatchEmbeddingResult> {- return await withSpan("firecrawl-generate-embeddings-batch", async span => {- const opts = { ...DEFAULT_OPTIONS, ...options };- const log = logger ?? _logger.child({ module: "search-embeddings-batch" });-- setSpanAttributes(span, {- "embedding.batch_size": texts.length,- "embedding.model": opts.model,- });-- if (texts.length === 0) {- return {- embeddings: [],- totalTokens: 0,- failed: [],- };- }-- const embeddings: number[][] = new Array(texts.length);- const failed: number[] = [];- let totalTokens = 0;-- // Process in batches- const batches: string[][] = [];- for (let i = 0; i < texts.length; i += opts.batchSize) {- batches.push(texts.slice(i, i + opts.batchSize));- }-- log.info("Processing embedding batches", {- totalTexts: texts.length,- batches: batches.length,- batchSize: opts.batchSize,- });-- for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {- const batch = batches[batchIndex];- const batchOffset = batchIndex * opts.batchSize;-- log.debug("Processing batch", {- batch: batchIndex + 1,- total: batches.length,- size: batch.length,- });-- // Process batch items in parallel- const batchResults = await Promise.allSettled(- batch.map((text, i) =>- generateEmbedding(text, options, log).then(result => ({- index: batchOffset + i,- ...result,- }))- ),- );-- // Collect results- for (const result of batchResults) {- if (result.status === "fulfilled") {- const { index, embedding, tokens } = result.value;- embeddings[index] = embedding;- totalTokens += tokens;- } else {- const index = batchOffset + batchResults.indexOf(result);- failed.push(index);- embeddings[index] = new Array(opts.dimensions).fill(0);-- log.warn("Failed to generate embedding in batch", {- index,- error: result.reason?.message,- });- }- }-- // Rate limiting: small delay between batches- if (batchIndex < batches.length - 1) {- await new Promise(resolve => setTimeout(resolve, 100));- }- }-- setSpanAttributes(span, {- "embedding.total_tokens": totalTokens,- "embedding.failed_count": failed.length,- "embedding.success_rate": ((texts.length - failed.length) / texts.length),- });-- log.info("Batch embedding complete", {- total: texts.length,- successful: texts.length - failed.length,- failed: failed.length,- totalTokens,- });-- return {- embeddings,- totalTokens,- failed,- };- });-}--/**- * Estimate embedding cost for text- * Based on OpenAI pricing: $0.02 / 1M tokens- */-export function estimateEmbeddingCost(tokenCount: number): number {- return (tokenCount / 1_000_000) * 0.02;-}--/**- * Check if embeddings are available (API key configured)- */-export function isEmbeddingEnabled(): boolean {- return !!process.env.OPENAI_API_KEY;-}--/**- * Normalize embedding vector (convert to unit vector)- * Useful for cosine similarity when using dot product- */-export function normalizeEmbedding(embedding: number[]): number[] {- const magnitude = Math.sqrt(- embedding.reduce((sum, val) => sum + val * val, 0),- );-- if (magnitude === 0) return embedding;-- return embedding.map(val => val / magnitude);-}--/**- * Calculate cosine similarity between two embeddings- */-export function cosineSimilarity(a: number[], b: number[]): number {- if (a.length !== b.length) {- throw new Error("Embeddings must have the same dimensions");- }-- let dotProduct = 0;- let magnitudeA = 0;- let magnitudeB = 0;-- for (let i = 0; i < a.length; i++) {- dotProduct += a[i] * b[i];- magnitudeA += a[i] * a[i];- magnitudeB += b[i] * b[i];- }-- magnitudeA = Math.sqrt(magnitudeA);- magnitudeB = Math.sqrt(magnitudeB);-- if (magnitudeA === 0 || magnitudeB === 0) return 0;-- return dotProduct / (magnitudeA * magnitudeB);-}-diff --git a/apps/api/src/lib/search-index/index.ts b/apps/api/src/lib/search-index/index.tsdeleted file mode 100644index 2e946ef7f..000000000--- a/apps/api/src/lib/search-index/index.ts+++ /dev/null@@ -1,56 +0,0 @@--export {- chunkText,- cleanMarkdownForIndexing,- estimateTokenCount,- type TextChunk,- type ChunkingOptions,-} from "./chunker";--export {- generateEmbedding,- generateEmbeddingsBatch,- isEmbeddingEnabled,- estimateEmbeddingCost,- normalizeEmbedding,- cosineSimilarity,- type EmbeddingResult,- type BatchEmbeddingResult,- type EmbeddingOptions,-} from "./embeddings";--export {- indexDocumentForSearch,- deleteDocumentFromSearch,- searchDocumentExists,- type SearchDocumentInput,- type SearchIndexResult,-} from "./service";--export {- search,- searchChunks,- getSearchStats,- type SearchQuery,- type SearchFilters,- type SearchResult,- type SearchResponse,-} from "./query";--export {- addSearchIndexJob,- processSearchIndexJobs,- getSearchIndexQueueLength,-} from "./queue";--export {- upsertToPinecone,- searchPinecone,- deleteFromPinecone,- getPineconeStats,- isPineconeEnabled,- buildPineconeFilter,- getPineconeIndex,- type PineconeRecord,-} from "./pinecone-service";-diff --git a/apps/api/src/lib/search-index/pinecone-service.ts b/apps/api/src/lib/search-index/pinecone-service.tsdeleted file mode 100644index c2809855e..000000000--- a/apps/api/src/lib/search-index/pinecone-service.ts+++ /dev/null@@ -1,297 +0,0 @@-import { Pinecone } from "@pinecone-database/pinecone";-import { logger as _logger } from "../logger";-import type { Logger } from "winston";-import { withSpan, setSpanAttributes } from "../otel-tracer";--// Pinecone client (singleton)-let pineconeClient: Pinecone | null = null;-let pineconeIndex: any = null;--/**- * Initialize Pinecone client- */-function getPineconeClient(): Pinecone {- if (!pineconeClient) {- const apiKey = process.env.PINECONE_API_KEY;-- if (!apiKey) {- throw new Error("PINECONE_API_KEY not set");- }-- pineconeClient = new Pinecone({- apiKey,- });-- _logger.info("Pinecone client initialized");- }-- return pineconeClient;-}--/**- * Get Pinecone index- */-export function getPineconeIndex() {- if (!pineconeIndex) {- const client = getPineconeClient();- const indexName = process.env.PINECONE_INDEX_NAME || "firecrawl-search";-- pineconeIndex = client.index(indexName);-- _logger.info("Pinecone index initialized", { indexName });- }-- return pineconeIndex;-}--/**- * Check if Pinecone is enabled- * Note: PINECONE_INDEX_NAME is optional and defaults to "firecrawl-search" in getPineconeIndex()- */-export function isPineconeEnabled(): boolean {- return !!process.env.PINECONE_API_KEY;-}--export interface PineconeRecord {- id: string;- values: number[];- metadata: {- url: string;- domain: string;- title?: string;- freshness_score: number;- quality_score: number;- country?: string;- is_mobile: boolean;- chunk_ordinal?: number; // For chunks- doc_id?: string; // For chunks- };-}--/**- * Upsert embeddings to Pinecone- */-export async function upsertToPinecone(- records: PineconeRecord[],- namespace: string = "documents",- logger?: Logger,-): Promise<void> {- return await withSpan("firecrawl-pinecone-upsert", async span => {- const log = logger ?? _logger.child({ module: "pinecone-service" });-- setSpanAttributes(span, {- "pinecone.operation": "upsert",- "pinecone.namespace": namespace,- "pinecone.record_count": records.length,- });-- if (records.length === 0) {- return;- }-- try {- const index = getPineconeIndex();- const ns = index.namespace(namespace);-- // Upsert in batches of 100 (Pinecone limit)- const batchSize = 100;- for (let i = 0; i < records.length; i += batchSize) {- const batch = records.slice(i, i + batchSize);-- await ns.upsert(batch);-- log.debug("Upserted batch to Pinecone", {- batch: Math.floor(i / batchSize) + 1,- size: batch.length,- });- }-- setSpanAttributes(span, {- "pinecone.upsert_successful": true,- });-- log.info("Upserted to Pinecone", {- namespace,- records: records.length,- });- } catch (error) {- log.error("Failed to upsert to Pinecone", {- error: (error as Error).message,- namespace,- recordCount: records.length,- });-- setSpanAttributes(span, {- "pinecone.upsert_error": true,- "pinecone.error_message": (error as Error).message,- });-- throw error;- }- });-}--/**- * Search Pinecone for similar vectors- */-export async function searchPinecone(- queryEmbedding: number[],- limit: number = 100,- filter?: Record<string, any>,- namespace: string = "documents",- logger?: Logger,-): Promise<Array<{- id: string;- score: number;- metadata: Record<string, any>;-}>> {- return await withSpan("firecrawl-pinecone-query", async span => {- const log = logger ?? _logger.child({ module: "pinecone-service" });-- setSpanAttributes(span, {- "pinecone.operation": "query",- "pinecone.namespace": namespace,- "pinecone.limit": limit,- "pinecone.has_filter": !!filter,- });-- try {- const index = getPineconeIndex();- const ns = index.namespace(namespace);-- const queryResponse = await ns.query({- vector: queryEmbedding,- topK: limit,- filter,- includeMetadata: true,- includeValues: false,- });-- const results = (queryResponse.matches || []).map(match => ({- id: match.id,- score: match.score || 0,- metadata: match.metadata || {},- }));-- setSpanAttributes(span, {- "pinecone.query_successful": true,- "pinecone.results_count": results.length,- });-- log.debug("Queried Pinecone", {- namespace,- limit,- results: results.length,- });-- return results;- } catch (error) {- log.error("Failed to query Pinecone", {- error: (error as Error).message,- namespace,- limit,- });-- setSpanAttributes(span, {- "pinecone.query_error": true,- "pinecone.error_message": (error as Error).message,- });-- throw error;- }- });-}--/**- * Delete from Pinecone- */-export async function deleteFromPinecone(- ids: string[],- namespace: string = "documents",- logger?: Logger,-): Promise<void> {- const log = logger ?? _logger.child({ module: "pinecone-service" });-- try {- const index = getPineconeIndex();- const ns = index.namespace(namespace);-- await ns.deleteMany(ids);-- log.info("Deleted from Pinecone", {- namespace,- count: ids.length,- });- } catch (error) {- log.error("Failed to delete from Pinecone", {- error: (error as Error).message,- namespace,- count: ids.length,- });- }-}--/**- * Get Pinecone index stats- */-export async function getPineconeStats(- namespace: string = "documents",-): Promise<{- vectorCount: number;- dimension: number;- indexFullness: number;-}> {- try {- const index = getPineconeIndex();- const stats = await index.describeIndexStats();-- const nsStats = stats.namespaces?.[namespace];-- return {- vectorCount: nsStats?.recordCount || 0,- dimension: stats.dimension || 1536,- indexFullness: stats.indexFullness || 0,- };- } catch (error) {- _logger.error("Failed to get Pinecone stats", {- error: (error as Error).message,- });-- return {- vectorCount: 0,- dimension: 1536,- indexFullness: 0,- };- }-}--/**- * Build Pinecone filter from search filters- */-export function buildPineconeFilter(filters: {- domain?: string;- country?: string;- isMobile?: boolean;- minFreshness?: number;-}): Record<string, any> | undefined {- const filter: Record<string, any> = {};-- if (filters.domain) {- filter.domain = { $eq: filters.domain };- }-- if (filters.country) {- filter.country = { $eq: filters.country };- }-- if (filters.isMobile !== undefined) {- filter.is_mobile = { $eq: filters.isMobile };- }-- if (filters.minFreshness !== undefined) {- filter.freshness_score = { $gte: filters.minFreshness };- }-- return Object.keys(filter).length > 0 ? filter : undefined;-}-diff --git a/apps/api/src/lib/search-index/query.ts b/apps/api/src/lib/search-index/query.tsdeleted file mode 100644index 2c9bff4ee..000000000--- a/apps/api/src/lib/search-index/query.ts+++ /dev/null@@ -1,644 +0,0 @@-import { SupabaseClient } from "@supabase/supabase-js";-import { logger as _logger } from "../logger";-import type { Logger } from "winston";-import { withSpan, setSpanAttributes } from "../otel-tracer";-import { generateEmbedding, isEmbeddingEnabled } from "./embeddings";-import {- searchPinecone,- isPineconeEnabled,- buildPineconeFilter,-} from "./pinecone-service";--export interface SearchQuery {- query: string;- limit?: number;- offset?: number;- filters?: SearchFilters;- mode?: "hybrid" | "keyword" | "semantic";-}--export interface SearchFilters {- domain?: string;- country?: string;- isMobile?: boolean;- minFreshness?: number;- language?: string;-}--export interface SearchResult {- documentId: string;- url: string;- title: string | null;- description: string | null;- domain: string;- snippet?: string;- score: number;- bm25Rank: number | null;- vectorRank: number | null;- freshnessScore: number;- qualityScore: number;- lastCrawledAt: string;-}--export interface SearchResponse {- results: SearchResult[];- total: number;- query: string;- mode: string;- took: number; // milliseconds-}--/**- * Main search function: hybrid search with RRF ranking- */-export async function search(- supabase: SupabaseClient,- searchQuery: SearchQuery,- logger?: Logger,-): Promise<SearchResponse> {- return await withSpan("firecrawl-search", async span => {- const log = logger ?? _logger.child({ module: "search-query" });- const startTime = Date.now();-- const {- query,- limit = 50,- offset = 0,- filters = {},- mode = "hybrid",- } = searchQuery;-- setSpanAttributes(span, {- "search.query": query,- "search.limit": limit,- "search.mode": mode,- "search.filters": JSON.stringify(filters),- });-- if (!query || query.trim().length === 0) {- return {- results: [],- total: 0,- query,- mode,- took: Date.now() - startTime,- };- }-- try {- let results: SearchResult[] = [];-- // Fetch enough results to support pagination- const fetchLimit = limit + offset;-- // Choose search strategy based on mode- if (mode === "keyword") {- results = await keywordSearch(supabase, query, fetchLimit, filters, log);- } else if (mode === "semantic") {- results = await semanticSearch(supabase, query, fetchLimit, filters, log);- } else {- // Hybrid: combine BM25 + vector with RRF- results = await hybridSearch(supabase, query, fetchLimit, filters, log);- }-- // Apply offset and limit for pagination- const paginatedResults = results.slice(offset, offset + limit);-- const took = Date.now() - startTime;-- setSpanAttributes(span, {- "search.results_count": results.length,- "search.took_ms": took,- });-- log.info("Search completed", {- query,- mode,- results: results.length,- took,- });-- return {- results: paginatedResults,- total: results.length,- query,- mode,- took,- };- } catch (error) {- log.error("Search failed", {- error: (error as Error).message,- query,- });-- setSpanAttributes(span, {- "search.error": true,- "search.error_message": (error as Error).message,- });-- return {- results: [],- total: 0,- query,- mode,- took: Date.now() - startTime,- };- }- });-}--/**- * Hybrid search: BM25 (Postgres) + Vector (Pinecone) with RRF ranking- */-async function hybridSearch(- supabase: SupabaseClient,- query: string,- limit: number,- filters: SearchFilters,- logger: Logger,-): Promise<SearchResult[]> {- // Generate query embedding for vector search- let queryEmbedding: number[] | null = null;-- if (isEmbeddingEnabled() && isPineconeEnabled()) {- try {- const embResult = await generateEmbedding(query, {}, logger);- queryEmbedding = embResult.embedding;- } catch (error) {- logger.warn("Failed to generate query embedding, falling back to keyword search", {- error: (error as Error).message,- });- }- }-- // If no embedding, fall back to keyword search- if (!queryEmbedding) {- return keywordSearch(supabase, query, limit, filters, logger);- }-- // Parallel search: BM25 in Postgres + Vector in Pinecone- const [bm25Results, vectorResults] = await Promise.all([- // BM25 search in Postgres- supabase.rpc("bm25_search", {- query_text: query,- result_limit: 100,- country_filter: filters.country ?? null,- domain_filter: filters.domain ?? null,- is_mobile_filter: filters.isMobile ?? null,- min_freshness: filters.minFreshness ?? 0.0,- }),-- // Vector search in Pinecone- (async () => {- try {- const pineconeFilter = buildPineconeFilter(filters);- const results = await searchPinecone(- queryEmbedding!,- 100,- pineconeFilter,- "documents",- logger,- );- return results;- } catch (error) {- logger.warn("Pinecone search failed, continuing with BM25 only", {- error: (error as Error).message,- });- return [];- }- })(),- ]);-- if (bm25Results.error) {- logger.error("BM25 search failed", { error: bm25Results.error.message });- throw new Error(`BM25 search failed: ${bm25Results.error.message}`);- }-- // Extract Pinecone doc IDs- const pineconeDocIds = vectorResults- .map(r => r.metadata.doc_id)- .filter(Boolean) as string[];-- // Get metadata for Pinecone results from Postgres- const { data: pineconeMetadata } = await supabase.rpc("get_documents_metadata", {- doc_ids: pineconeDocIds,- });-- // Create map of doc_id -> metadata- const metadataMap = new Map(- (pineconeMetadata || []).map((m: any) => [m.doc_id, m])- );-- // Merge results with RRF- const bm25Map = new Map<string, any>(- (bm25Results.data || []).map((r: any, index: number) => [- r.doc_id,- { rank: index + 1, ...r },- ])- );-- const vectorMap = new Map<string, { rank: number; score: number }>(- vectorResults.map((r, index) => [- r.metadata.doc_id || r.id,- { rank: index + 1, score: r.score },- ])- );-- // Combine all unique doc IDs- const allDocIds = new Set([- ...bm25Map.keys(),- ...vectorMap.keys(),- ]);-- const k = 60; // RRF constant- const merged: SearchResult[] = [];-- for (const docId of allDocIds) {- const bm25 = bm25Map.get(docId);- const vector = vectorMap.get(docId);-- // RRF score- const bm25Rank = bm25?.rank ?? null;- const vectorRank = vector?.rank ?? null;-- const rrf =- (bm25Rank ? 1 / (k + bm25Rank) : 0) +- (vectorRank ? 1 / (k + vectorRank) : 0);-- // Get metadata (prefer from BM25 result, fallback to Pinecone metadata)- const metadata: any = bm25 || metadataMap.get(docId);-- if (!metadata) continue;-- const combinedScore =- rrf * (metadata.freshness_score || 1.0) * (metadata.quality_score || 1.0);-- merged.push({- documentId: docId,- url: metadata.url,- title: metadata.title,- description: metadata.description,- domain: metadata.domain,- score: combinedScore,- bm25Rank,- vectorRank,- freshnessScore: metadata.freshness_score || 1.0,- qualityScore: metadata.quality_score || 1.0,- lastCrawledAt: metadata.last_crawled_at,- });- }-- // Sort by combined score- merged.sort((a, b) => b.score - a.score);-- return merged.slice(0, limit);-}--/**- * Keyword-only search (BM25)- */-async function keywordSearch(- supabase: SupabaseClient,- query: string,- limit: number,- filters: SearchFilters,- logger: Logger,-): Promise<SearchResult[]> {- let queryBuilder = supabase- .from("search_documents")- .select(- "id, resolved_url, title, description, domain, freshness_score, quality_score, last_crawled_at, content_ts",- )- .textSearch("content_ts", query, {- type: "plain",- config: "english",- });-- // Apply filters- if (filters.domain) {- queryBuilder = queryBuilder.eq("domain", filters.domain);- }- if (filters.country) {- queryBuilder = queryBuilder.eq("country", filters.country);- }- if (filters.isMobile !== undefined) {- queryBuilder = queryBuilder.eq("is_mobile", filters.isMobile);- }- if (filters.minFreshness) {- queryBuilder = queryBuilder.gte("freshness_score", filters.minFreshness);- }- if (filters.language) {- queryBuilder = queryBuilder.eq("language", filters.language);- }-- queryBuilder = queryBuilder.limit(limit);-- const { data, error } = await queryBuilder;-- if (error) {- logger.error("Keyword search failed", { error: error.message });- throw new Error(`Keyword search failed: ${error.message}`);- }-- return (data ?? []).map((row: any, index: number) => ({- documentId: row.id,- url: row.resolved_url,- title: row.title,- description: row.description,- domain: row.domain,- score: row.freshness_score * row.quality_score * (1 / (index + 1)),- bm25Rank: index + 1,- vectorRank: null,- freshnessScore: row.freshness_score,- qualityScore: row.quality_score,- lastCrawledAt: row.last_crawled_at,- }));-}--/**- * Semantic-only search (vector similarity via Pinecone)- */-async function semanticSearch(- supabase: SupabaseClient,- query: string,- limit: number,- filters: SearchFilters,- logger: Logger,-): Promise<SearchResult[]> {- if (!isEmbeddingEnabled() || !isPineconeEnabled()) {- logger.warn("Embeddings or Pinecone not enabled, falling back to keyword search");- return keywordSearch(supabase, query, limit, filters, logger);- }-- // Generate query embedding- const embResult = await generateEmbedding(query, {}, logger);- const queryEmbedding = embResult.embedding;-- // Search Pinecone with filters- const pineconeFilter = buildPineconeFilter(filters);- const vectorResults = await searchPinecone(- queryEmbedding,- limit,- pineconeFilter,- "documents",- logger,- );-- if (vectorResults.length === 0) {- return [];- }-- // Get metadata from Postgres- const docIds = vectorResults- .map(r => r.metadata.doc_id)- .filter(Boolean) as string[];-- const { data: metadata, error } = await supabase.rpc("get_documents_metadata", {- doc_ids: docIds,- });-- if (error) {- logger.error("Failed to fetch metadata for semantic search", { error: error.message });- throw new Error(`Metadata fetch failed: ${error.message}`);- }-- // Create metadata map- const metadataMap = new Map(- (metadata || []).map((m: any) => [m.doc_id, m])- );-- // Combine Pinecone results with Postgres metadata- return vectorResults.map((result, index) => {- const docId = result.metadata.doc_id || result.id;- const meta: any = metadataMap.get(docId);-- if (!meta) {- // Fallback to Pinecone metadata- return {- documentId: docId,- url: result.metadata.url || "",- title: result.metadata.title || null,- description: null,- domain: result.metadata.domain || "",- score: result.score * (result.metadata.freshness_score || 1.0) * (result.metadata.quality_score || 1.0),- bm25Rank: null,- vectorRank: index + 1,- freshnessScore: result.metadata.freshness_score || 1.0,- qualityScore: result.metadata.quality_score || 1.0,- lastCrawledAt: new Date().toISOString(),- };- }-- return {- documentId: docId,- url: meta.url,- title: meta.title,- description: meta.description,- domain: meta.domain,- score: result.score * meta.freshness_score * meta.quality_score,- bm25Rank: null,- vectorRank: index + 1,- freshnessScore: meta.freshness_score,- qualityScore: meta.quality_score,- lastCrawledAt: meta.last_crawled_at,- };- }).filter(r => r.url !== "");-}--/**- * Search within chunks (for precise snippet retrieval via Pinecone)- */-export async function searchChunks(- supabase: SupabaseClient,- query: string,- limit: number = 20,- filters: SearchFilters = {},- logger?: Logger,-): Promise<Array<{- chunkId: string;- documentId: string;- url: string;- title: string | null;- text: string;- score: number;- ordinal: number;-}>> {- const log = logger ?? _logger.child({ module: "search-chunks" });-- // Generate query embedding- let queryEmbedding: number[] | null = null;-- if (isEmbeddingEnabled() && isPineconeEnabled()) {- try {- const embResult = await generateEmbedding(query, {}, log);- queryEmbedding = embResult.embedding;- } catch (error) {- log.warn("Failed to generate query embedding for chunk search", {- error: (error as Error).message,- });- }- }-- if (!queryEmbedding) {- // Fall back to keyword search on chunks (Postgres)- const { data, error } = await supabase- .from("search_chunks")- .select("id, doc_id, text, ordinal")- .textSearch("text_ts", query, {- type: "plain",- config: "english",- })- .not("text", "is", null)- .limit(limit);-- if (error) {- log.error("Chunk keyword search failed", { error: error.message });- return [];- }-- // Join with documents to get metadata- const docIds = [...new Set((data ?? []).map(c => c.doc_id))];- const { data: docsData } = await supabase- .from("search_documents")- .select("id, resolved_url, title")- .in("id", docIds);-- const docMap = new Map((docsData ?? []).map(d => [d.id, d]));-- return (data ?? []).map((chunk, index) => {- const doc = docMap.get(chunk.doc_id);- return {- chunkId: chunk.id,- documentId: chunk.doc_id,- url: doc?.resolved_url ?? "",- title: doc?.title ?? null,- text: chunk.text || "",- score: 1 / (index + 1),- ordinal: chunk.ordinal,- };- });- }-- // Vector chunk search via Pinecone- const pineconeFilter = buildPineconeFilter(filters);-- // Add chunk-specific filter- const chunkFilter = {- ...pineconeFilter,- chunk_ordinal: { $exists: true }, // Only get chunk records, not document records- };-- const vectorResults = await searchPinecone(- queryEmbedding,- limit,- chunkFilter,- "documents",- log,- );-- if (vectorResults.length === 0) {- return [];- }-- // Get chunk data from Postgres (for text and ordinal)- const docIds = [...new Set(- vectorResults.map(r => r.metadata.doc_id).filter(Boolean)- )] as string[];-- const { data: chunksData } = await supabase- .from("search_chunks")- .select("id, doc_id, text, ordinal")- .in("doc_id", docIds);-- const { data: docsData } = await supabase- .from("search_documents")- .select("id, resolved_url, title")- .in("id", docIds);-- const chunkMap = new Map(- (chunksData ?? [])- .filter(c => c.doc_id && c.ordinal !== null)- .map(c => [`${c.doc_id}_${c.ordinal}`, c])- );- const docMap = new Map((docsData ?? []).map(d => [d.id, d]));-- // Combine Pinecone results with Postgres data- return vectorResults.map((result, index) => {- const docId = result.metadata.doc_id;- const ordinal = result.metadata.chunk_ordinal;- const chunkKey = `${docId}_${ordinal}`;-- const chunk = chunkMap.get(chunkKey);- const doc = docMap.get(docId);-- return {- chunkId: result.id,- documentId: docId || "",- url: doc?.resolved_url || result.metadata.url || "",- title: doc?.title || result.metadata.title || null,- text: chunk?.text || "",- score: result.score,- ordinal: ordinal ?? 0,- };- }).filter(r => r.url !== "");-}--/**- * Get search statistics (including Pinecone stats)- */-export async function getSearchStats(- supabase: SupabaseClient,-): Promise<{- totalDocuments: number;- totalChunks: number;- documentsWithEmbeddings: number;- chunksWithEmbeddings: number;- avgFreshness: number;- avgQuality: number;- uniqueDomains: number;- pineconeVectors?: number;- pineconeIndexFullness?: number;-}> {- const { data, error } = await supabase.from("search_index_stats").select("*");-- // Get Pinecone stats if available- let pineconeStats = { vectorCount: 0, indexFullness: 0 };- if (isPineconeEnabled()) {- try {- const { getPineconeStats } = await import("./pinecone-service.js");- pineconeStats = await getPineconeStats("documents");- } catch (error) {- _logger.warn("Failed to get Pinecone stats", {- error: (error as Error).message,- });- }- }-- if (error || !data || data.length === 0) {- return {- totalDocuments: 0,- totalChunks: 0,- documentsWithEmbeddings: 0,- chunksWithEmbeddings: 0,- avgFreshness: 0,- avgQuality: 0,- uniqueDomains: 0,- pineconeVectors: pineconeStats.vectorCount,- pineconeIndexFullness: pineconeStats.indexFullness,- };- }-- const stats = data[0];- return {- totalDocuments: stats.total_documents ?? 0,- totalChunks: stats.total_chunks ?? 0,- documentsWithEmbeddings: stats.documents_in_pinecone ?? 0,- chunksWithEmbeddings: stats.chunks_in_pinecone ?? 0,- avgFreshness: stats.avg_freshness ?? 0,- avgQuality: stats.avg_quality ?? 0,- uniqueDomains: stats.unique_domains ?? 0,- pineconeVectors: pineconeStats.vectorCount,- pineconeIndexFullness: pineconeStats.indexFullness,- };-}-diff --git a/apps/api/src/lib/search-index/queue.ts b/apps/api/src/lib/search-index/queue.tsdeleted file mode 100644index a8c6c87b1..000000000--- a/apps/api/src/lib/search-index/queue.ts+++ /dev/null@@ -1,164 +0,0 @@-/**- * Search Index Queue Processor- *- * Handles async indexing jobs using Redis queue.- * Similar to existing index insert queue, but for search index.- *- * Flow:- * 1. Scraper adds job to queue after successful scrape- * 2. Background worker picks up jobs in batches- * 3. Text is chunked and embeddings generated- * 4. Data is written to search_documents and search_chunks- */--import { redisEvictConnection } from "../../services/redis";-import { logger as _logger } from "../logger";-import { indexDocumentForSearch, type SearchDocumentInput } from "./service";-import { search_index_supabase_service } from "../../services/search-index-db";--const SEARCH_INDEX_QUEUE_KEY = "search-index-queue";-const SEARCH_INDEX_BATCH_SIZE = 10; // Smaller batch size due to embedding generation--export interface SearchIndexJob {- url: string;- resolvedUrl: string;- title?: string;- description?: string;- markdown: string;- html: string;- statusCode: number;- gcsPath?: string;- screenshotUrl?: string;- language?: string;- country?: string;- isMobile?: boolean;-}--/**- * Add a job to the search index queue- */-export async function addSearchIndexJob(job: SearchIndexJob): Promise<void> {- try {- await redisEvictConnection.rpush(- SEARCH_INDEX_QUEUE_KEY,- JSON.stringify(job),- );-- _logger.debug("Added job to search index queue", {- url: job.url,- });- } catch (error) {- _logger.error("Failed to add job to search index queue", {- error: (error as Error).message,- url: job.url,- });- }-}--/**- * Get jobs from the queue- */-async function getSearchIndexJobs(): Promise<SearchIndexJob[]> {- try {- const jobs =- (await redisEvictConnection.lpop(- SEARCH_INDEX_QUEUE_KEY,- SEARCH_INDEX_BATCH_SIZE,- )) ?? [];-- return jobs.map(x => JSON.parse(x) as SearchIndexJob);- } catch (error) {- _logger.error("Failed to get jobs from search index queue", {- error: (error as Error).message,- });- return [];- }-}--/**- * Process search index jobs (called by background worker)- */-export async function processSearchIndexJobs(): Promise<void> {- const jobs = await getSearchIndexJobs();-- if (jobs.length === 0) {- return;- }-- _logger.info("Processing search index jobs", {- jobCount: jobs.length,- });-- let successCount = 0;- let failedCount = 0;-- // Process jobs sequentially to avoid overwhelming embedding API- for (const job of jobs) {- try {- const logger = _logger.child({- module: "search-index-queue",- url: job.url,- });-- const result = await indexDocumentForSearch(- search_index_supabase_service,- {- url: job.url,- resolvedUrl: job.resolvedUrl,- title: job.title,- description: job.description,- markdown: job.markdown,- html: job.html,- statusCode: job.statusCode,- gcsPath: job.gcsPath,- screenshotUrl: job.screenshotUrl,- language: job.language,- country: job.country,- isMobile: job.isMobile,- },- logger,- );-- if (result.error) {- failedCount++;- logger.error("Failed to index document for search", {- error: result.error,- });- } else {- successCount++;- logger.info("Indexed document for search", {- documentId: result.documentId,- chunks: result.chunkCount,- tokens: result.totalTokens,- });- }- } catch (error) {- failedCount++;- _logger.error("Failed to process search index job", {- error: (error as Error).message,- url: job.url,- });- }- }-- _logger.info("Finished processing search index jobs", {- total: jobs.length,- successful: successCount,- failed: failedCount,- });-}--/**- * Get queue length- */-export async function getSearchIndexQueueLength(): Promise<number> {- try {- return (await redisEvictConnection.llen(SEARCH_INDEX_QUEUE_KEY)) ?? 0;- } catch (error) {- _logger.error("Failed to get search index queue length", {- error: (error as Error).message,- });- return 0;- }-}-diff --git a/apps/api/src/lib/search-index/service.ts b/apps/api/src/lib/search-index/service.tsdeleted file mode 100644index 32e07e83a..000000000--- a/apps/api/src/lib/search-index/service.ts+++ /dev/null@@ -1,684 +0,0 @@-import { SupabaseClient } from "@supabase/supabase-js";-import { logger as _logger } from "../logger";-import type { Logger } from "winston";-import { withSpan, setSpanAttributes } from "../otel-tracer";-import {- chunkText,- cleanMarkdownForIndexing,- estimateTokenCount,- type TextChunk,-} from "./chunker";-import {- generateEmbedding,- generateEmbeddingsBatch,- isEmbeddingEnabled,- type EmbeddingResult,-} from "./embeddings";-import {- upsertToPinecone,- isPineconeEnabled,- type PineconeRecord,-} from "./pinecone-service";-import crypto from "crypto";-import psl from "psl";--export interface SearchDocumentInput {- url: string;- resolvedUrl: string;- title?: string;- description?: string;- markdown: string;- html: string;- statusCode: number;- gcsPath?: string;- screenshotUrl?: string;- language?: string;- country?: string;- isMobile?: boolean;-}--export interface SearchIndexResult {- documentId: string;- chunkCount: number;- totalTokens: number;- embeddingsGenerated: boolean;- error?: string;-}--/**- * Main function: Index a document for search- */-export async function indexDocumentForSearch(- supabase: SupabaseClient,- input: SearchDocumentInput,- logger?: Logger,-): Promise<SearchIndexResult> {- return await withSpan("firecrawl-index-document-for-search", async span => {- const log = logger ?? _logger.child({ module: "search-index-service" });-- setSpanAttributes(span, {- "search.url": input.url,- "search.status_code": input.statusCode,- "search.is_mobile": input.isMobile ?? false,- });-- try {- // 1. Extract and clean text from markdown- const cleanText = cleanMarkdownForIndexing(input.markdown);-- if (!cleanText || cleanText.length < 100) {- log.warn("Document too short to index", {- url: input.url,- textLength: cleanText.length,- });- return {- documentId: "",- chunkCount: 0,- totalTokens: 0,- embeddingsGenerated: false,- error: "Document too short",- };- }-- setSpanAttributes(span, {- "search.clean_text_length": cleanText.length,- "search.clean_text_tokens": estimateTokenCount(cleanText),- });-- // 2. Chunk the text- const chunks = await chunkText(cleanText, {- targetTokens: 750,- minTokens: 600,- maxTokens: 900,- overlapTokens: 100,- preserveStructure: true,- }, log);-- if (chunks.length === 0) {- log.warn("No chunks generated", { url: input.url });- return {- documentId: "",- chunkCount: 0,- totalTokens: 0,- embeddingsGenerated: false,- error: "No chunks generated",- };- }-- setSpanAttributes(span, {- "search.chunk_count": chunks.length,- "search.total_tokens": chunks.reduce((sum, c) => sum + c.tokenCount, 0),- });-- // 3. Calculate URL hash and domain- const urlHash = Buffer.from(- crypto.createHash("sha256").update(normalizeSearchURL(input.url)).digest("hex"),- "hex",- );- const domain = extractDomain(input.resolvedUrl);-- // 4. Generate embeddings (if enabled)- const embeddingsEnabled = isEmbeddingEnabled() && isPineconeEnabled();- let documentEmbedding: number[] | null = null;- let chunkEmbeddings: number[][] = [];- let totalTokens = 0;- let pineconeRecords: PineconeRecord[] = [];- let embeddingsActuallyGenerated = false;-- if (embeddingsEnabled) {- try {- // Generate document-level embedding (from title + description + first chunk)- const documentText = [- input.title,- input.description,- chunks[0]?.text.slice(0, 500),- ]- .filter(Boolean)- .join(" ");-- const docEmbResult = await generateEmbedding(documentText, {}, log);- documentEmbedding = docEmbResult.embedding;- totalTokens += docEmbResult.tokens;-- // Generate chunk embeddings in batch- const chunkTexts = chunks.map(c => c.text);- const batchResult = await generateEmbeddingsBatch(chunkTexts, {- batchSize: 50,- }, log);-- chunkEmbeddings = batchResult.embeddings;- totalTokens += batchResult.totalTokens;-- // Prepare Pinecone records (document + chunks)- // Document-level record- pineconeRecords.push({- id: `doc_${crypto.randomUUID()}`,- values: documentEmbedding,- metadata: {- url: input.resolvedUrl,- domain: extractDomain(input.resolvedUrl),- title: input.title,- freshness_score: 1.0,- quality_score: 1.0, // Will update later- country: input.country,- is_mobile: input.isMobile ?? false,- },- });-- // Chunk-level records- chunks.forEach((chunk, i) => {- if (chunkEmbeddings[i]) {- pineconeRecords.push({- id: `chunk_${crypto.randomUUID()}`,- values: chunkEmbeddings[i],- metadata: {- url: input.resolvedUrl,- domain: extractDomain(input.resolvedUrl),- title: input.title,- freshness_score: 1.0,- quality_score: 1.0,- country: input.country,- is_mobile: input.isMobile ?? false,- chunk_ordinal: i,- doc_id: "", // Will update after DB insert- },- });- }- });-- // Mark as successfully generated- embeddingsActuallyGenerated = true;-- setSpanAttributes(span, {- "search.embeddings_generated": true,- "search.embedding_tokens": totalTokens,- "search.pinecone_records": pineconeRecords.length,- });- } catch (error) {- log.error("Failed to generate embeddings", {- error: (error as Error).message,- url: input.url,- });- setSpanAttributes(span, {- "search.embeddings_error": true,- "search.embeddings_error_message": (error as Error).message,- });- // Reset flag since embeddings failed- embeddingsActuallyGenerated = false;- }- }-- // 5. Create tsvector for full-text search- const contentTsVector = await createTsVector(supabase, cleanText);-- // 6. Upsert document into search_documents- // Check if document already exists by url_hash- // Note: url_hash is BYTEA, need to query with proper format- const urlHashHex = "\\x" + urlHash.toString('hex');- const { data: existingDocs, error: lookupError } = await supabase- .from("search_documents")- .select("id, content_hash")- .eq("url_hash", urlHashHex)- .limit(1);-- if (lookupError) {- log.warn("Error looking up existing document, will attempt insert", {- error: lookupError.message,- url: input.url,- });- }-- const existingDoc = existingDocs?.[0];- const newContentHash = Buffer.from(- crypto.createHash("sha256").update(cleanText).digest("hex"),- "hex",- );-- let documentId: string;- let isUpdate = false;-- if (existingDoc) {- // Document exists - update it- documentId = existingDoc.id;- isUpdate = true;-- // Delete existing chunks (will be replaced)- await supabase- .from("search_chunks")- .delete()- .eq("doc_id", documentId);-- // Update document- const { error: updateError } = await supabase- .from("search_documents")- .update({- resolved_url: input.resolvedUrl,- title: input.title?.slice(0, 200) ?? null,- description: input.description?.slice(0, 500) ?? null,- language: input.language ?? "en",- gcs_path: input.gcsPath ?? null,- screenshot_url: input.screenshotUrl ?? null,- content_ts: contentTsVector,- country: input.country ?? null,- pinecone_synced: false, // Will sync after successful Pinecone upsert- is_mobile: input.isMobile ?? false,- status_code: input.statusCode,- content_hash: newContentHash,- freshness_score: 1.0, // Reset to fresh- quality_score: calculateQualityScore(input, chunks),- last_crawled_at: new Date().toISOString(),- updated_at: new Date().toISOString(),- })- .eq("id", documentId);-- if (updateError) {- log.error("Failed to update document", {- error: updateError.message,- url: input.url,- documentId,- });-- setSpanAttributes(span, {- "search.update_error": true,- "search.update_error_message": updateError.message,- });-- return {- documentId: "",- chunkCount: 0,- totalTokens: 0,- embeddingsGenerated: false,- error: updateError.message,- };- }- } else {- // New document - insert- documentId = crypto.randomUUID();-- const { error: insertError } = await supabase- .from("search_documents")- .insert({- id: documentId,- url_hash: urlHashHex,- url: normalizeSearchURL(input.url),- resolved_url: input.resolvedUrl,- title: input.title?.slice(0, 200) ?? null,- description: input.description?.slice(0, 500) ?? null,- language: input.language ?? "en",- gcs_path: input.gcsPath ?? null,- gcp_path_id: input.gcsPath ?? null,- screenshot_url: input.screenshotUrl ?? null,- content_ts: contentTsVector,- domain,- country: input.country ?? null,- is_mobile: input.isMobile ?? false,- status_code: input.statusCode,- content_hash: newContentHash,- freshness_score: 1.0,- quality_score: calculateQualityScore(input, chunks),- last_crawled_at: new Date().toISOString(),- pinecone_synced: false,- });-- if (insertError) {- log.error("Failed to insert document", {- error: insertError.message,- url: input.url,- });-- setSpanAttributes(span, {- "search.insert_error": true,- "search.insert_error_message": insertError.message,- });-- return {- documentId: "",- chunkCount: 0,- totalTokens: 0,- embeddingsGenerated: false,- error: insertError.message,- };- }- }-- // 7. Insert chunks into search_chunks- // Only insert first 2 chunks for snippet display- // Rest are stored only in Pinecone (saves 80% storage!)- const chunkInserts = chunks- .slice(0, 2) // Only first 2 chunks- .map((chunk, i) => ({- doc_id: documentId,- ordinal: i,- text: chunk.text,- token_count: chunk.tokenCount,- char_count: chunk.charCount,- prev_chunk_id: null,- pinecone_synced: false,- }));-- const { error: chunkError } = await supabase- .from("search_chunks")- .insert(chunkInserts);-- if (chunkError) {- log.error("Failed to insert chunks", {- error: chunkError.message,- url: input.url,- documentId,- });-- // Try to clean up the document- await supabase.from("search_documents").delete().eq("id", documentId);-- return {- documentId: "",- chunkCount: 0,- totalTokens: 0,- embeddingsGenerated: false,- error: chunkError.message,- };- }-- // 8. Upsert to Pinecone (async, don't block)- if (embeddingsEnabled && pineconeRecords.length > 0) {- // Update Pinecone records with actual doc_id- pineconeRecords.forEach(record => {- if (record.metadata.chunk_ordinal !== undefined) {- record.metadata.doc_id = documentId;- }- });-- try {- await upsertToPinecone(pineconeRecords, "documents", log);-- // Mark as synced- await supabase- .from("search_documents")- .update({- pinecone_synced: true,- pinecone_synced_at: new Date().toISOString(),- })- .eq("id", documentId);-- await supabase- .from("search_chunks")- .update({ pinecone_synced: true })- .eq("doc_id", documentId);-- log.info("Synced to Pinecone", {- documentId,- records: pineconeRecords.length,- });- } catch (error) {- log.error("Failed to sync to Pinecone", {- error: (error as Error).message,- documentId,- });- // Continue anyway - can retry later- }- }-- // 9. Update sync state- await updateSyncState(supabase, {- documentsIndexed: 1,- chunksIndexed: Math.min(chunks.length, 2), // Only inserted first 2- embeddingsGenerated: embeddingsActuallyGenerated ? 1 + chunks.length : 0,- });-- log.info("Document indexed successfully", {- url: input.url,- documentId,- chunks: chunks.length,- totalTokens,- embeddingsGenerated: embeddingsActuallyGenerated,- isUpdate,- });-- setSpanAttributes(span, {- "search.success": true,- "search.document_id": documentId,- "search.is_update": isUpdate,- });-- return {- documentId,- chunkCount: chunks.length,- totalTokens,- embeddingsGenerated: embeddingsActuallyGenerated,- };- } catch (error) {- log.error("Failed to index document", {- error: (error as Error).message,- url: input.url,- });-- setSpanAttributes(span, {- "search.error": true,- "search.error_message": (error as Error).message,- });-- return {- documentId: "",- chunkCount: 0,- totalTokens: 0,- embeddingsGenerated: false,- error: (error as Error).message,- };- }- });-}--/**- * Normalize URL for search index (remove query params, fragments, etc.)- * Preserves the original protocol (http/https) to avoid collisions and maintain reachability- */-function normalizeSearchURL(url: string): string {- try {- const urlObj = new URL(url);- urlObj.hash = "";- urlObj.search = ""; // Remove query params for canonical URL- // Note: Preserve original protocol (don't force https) to avoid:- // 1. Making HTTP-only pages unreachable- // 2. Collisions between distinct HTTP/HTTPS resources-- if (urlObj.hostname.startsWith("www.")) {- urlObj.hostname = urlObj.hostname.slice(4);- }-- if (urlObj.pathname.endsWith("/")) {- urlObj.pathname = urlObj.pathname.slice(0, -1);- }-- return urlObj.toString();- } catch {- return url;- }-}--/**- * Extract domain from URL- */-function extractDomain(url: string): string {- try {- const urlObj = new URL(url);- const parsed = psl.parse(urlObj.hostname);-- if (parsed.domain) {- return parsed.domain;- }-- return urlObj.hostname;- } catch {- return "";- }-}--/**- * Calculate quality score based on content characteristics- */-function calculateQualityScore(- input: SearchDocumentInput,- chunks: TextChunk[],-): number {- let score = 1.0;-- // Factor 1: Has title and description- if (input.title && input.title.length > 10) score *= 1.1;- if (input.description && input.description.length > 50) score *= 1.1;-- // Factor 2: Content length (sweet spot: 2000-10000 chars)- const totalChars = chunks.reduce((sum, c) => sum + c.charCount, 0);- if (totalChars >= 2000 && totalChars <= 10000) {- score *= 1.2;- } else if (totalChars < 500) {- score *= 0.5;- } else if (totalChars > 50000) {- score *= 0.8;- }-- // Factor 3: Chunk count (good chunking indicates structured content)- if (chunks.length >= 3 && chunks.length <= 20) {- score *= 1.1;- }-- // Factor 4: Status code- if (input.statusCode >= 200 && input.statusCode < 300) {- score *= 1.0;- } else {- score *= 0.3;- }-- // Normalize to [0, 1]- return Math.min(Math.max(score, 0.1), 2.0);-}--/**- * Create tsvector from text (for BM25 search)- */-async function createTsVector(- supabase: SupabaseClient,- text: string,-): Promise<string | null> {- try {- const { data, error } = await supabase.rpc("to_tsvector_english", {- text: text.slice(0, 100000), // Limit to 100k chars- });-- if (error) {- _logger.warn("Failed to create tsvector", { error: error.message });- return null;- }-- return data;- } catch (error) {- _logger.warn("Failed to create tsvector", {- error: (error as Error).message,- });- return null;- }-}--/**- * Update sync state with metrics- */-async function updateSyncState(- supabase: SupabaseClient,- metrics: {- documentsIndexed?: number;- chunksIndexed?: number;- embeddingsGenerated?: number;- failedDocuments?: number;- },-): Promise<void> {- try {- const updates: any = {- updated_at: new Date().toISOString(),- };-- if (metrics.documentsIndexed) {- // Use SQL to increment- await supabase.rpc("increment_search_sync_state", {- field: "total_documents_indexed",- amount: metrics.documentsIndexed,- });- }-- if (metrics.chunksIndexed) {- await supabase.rpc("increment_search_sync_state", {- field: "total_chunks_indexed",- amount: metrics.chunksIndexed,- });- }-- if (metrics.failedDocuments) {- await supabase.rpc("increment_search_sync_state", {- field: "failed_documents",- amount: metrics.failedDocuments,- });- }-- // Update embedding quota tracking- const today = new Date().toISOString().split("T")[0];- await supabase.rpc("update_embedding_quota", {- embeddings_count: metrics.embeddingsGenerated ?? 0,- today_date: today,- });- } catch (error) {- _logger.warn("Failed to update sync state", {- error: (error as Error).message,- });- }-}--/**- * Delete document from search index- */-export async function deleteDocumentFromSearch(- supabase: SupabaseClient,- urlHash: Buffer,- logger?: Logger,-): Promise<void> {- const log = logger ?? _logger.child({ module: "search-index-service" });-- try {- const urlHashHex = "\\x" + urlHash.toString('hex');- const { error } = await supabase- .from("search_documents")- .delete()- .eq("url_hash", urlHashHex);-- if (error) {- log.error("Failed to delete document from search index", {- error: error.message,- });- }- } catch (error) {- log.error("Failed to delete document from search index", {- error: (error as Error).message,- });- }-}--/**- * Check if document already exists in search index- */-export async function searchDocumentExists(- supabase: SupabaseClient,- urlHash: Buffer,-): Promise<boolean> {- try {- const urlHashHex = "\\x" + urlHash.toString('hex');- const { data, error } = await supabase- .from("search_documents")- .select("id")- .eq("url_hash", urlHashHex)- .limit(1);-- if (error) return false;-- return (data?.length ?? 0) > 0;- } catch {- return false;- }-}-