🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
Checkpoint: File edits
rUvbbee06aa43e4
diff --git a/v3/@claude-flow/embeddings/src/chunking.ts b/v3/@claude-flow/embeddings/src/chunking.tsnew file mode 100644index 0000000000..4035556416--- /dev/null+++ b/v3/@claude-flow/embeddings/src/chunking.ts@@ -0,0 +1,351 @@+/**+ * Document Chunking Utilities+ *+ * Features:+ * - Configurable chunk size and overlap+ * - Sentence-aware splitting+ * - Paragraph-aware splitting+ * - Token-based chunking (approximate)+ * - Metadata tracking for reconstruction+ */++/**+ * Chunking configuration+ */+export interface ChunkingConfig {+ /** Maximum chunk size in characters (default: 512) */+ maxChunkSize?: number;+ /** Overlap between chunks in characters (default: 50) */+ overlap?: number;+ /** Strategy for splitting (default: 'sentence') */+ strategy?: 'character' | 'sentence' | 'paragraph' | 'token';+ /** Minimum chunk size (default: 100) */+ minChunkSize?: number;+ /** Include metadata with chunks */+ includeMetadata?: boolean;+}++/**+ * Chunk result with metadata+ */+export interface Chunk {+ /** Chunk text content */+ text: string;+ /** Original index in document */+ index: number;+ /** Start position in original text */+ startPos: number;+ /** End position in original text */+ endPos: number;+ /** Character count */+ length: number;+ /** Approximate token count (chars / 4) */+ tokenCount: number;+}++/**+ * Chunked document result+ */+export interface ChunkedDocument {+ /** Array of chunks */+ chunks: Chunk[];+ /** Original text length */+ originalLength: number;+ /** Total chunks created */+ totalChunks: number;+ /** Configuration used */+ config: Required<ChunkingConfig>;+}++// Sentence boundary patterns+const SENTENCE_ENDINGS = /(?<=[.!?])\s+(?=[A-Z])/g;+const PARAGRAPH_BREAKS = /\n\n+/g;++/**+ * Split text into chunks with overlap+ */+export function chunkText(+ text: string,+ config: ChunkingConfig = {}+): ChunkedDocument {+ const finalConfig: Required<ChunkingConfig> = {+ maxChunkSize: config.maxChunkSize ?? 512,+ overlap: config.overlap ?? 50,+ strategy: config.strategy ?? 'sentence',+ minChunkSize: config.minChunkSize ?? 100,+ includeMetadata: config.includeMetadata ?? true,+ };++ // Normalize whitespace+ const normalizedText = text.replace(/\s+/g, ' ').trim();++ let chunks: Chunk[];++ switch (finalConfig.strategy) {+ case 'character':+ chunks = chunkByCharacter(normalizedText, finalConfig);+ break;+ case 'sentence':+ chunks = chunkBySentence(normalizedText, finalConfig);+ break;+ case 'paragraph':+ chunks = chunkByParagraph(text, finalConfig); // Keep original for paragraphs+ break;+ case 'token':+ chunks = chunkByToken(normalizedText, finalConfig);+ break;+ default:+ chunks = chunkBySentence(normalizedText, finalConfig);+ }++ return {+ chunks,+ originalLength: text.length,+ totalChunks: chunks.length,+ config: finalConfig,+ };+}++/**+ * Simple character-based chunking with overlap+ */+function chunkByCharacter(+ text: string,+ config: Required<ChunkingConfig>+): Chunk[] {+ const chunks: Chunk[] = [];+ const { maxChunkSize, overlap } = config;++ let pos = 0;+ let index = 0;++ while (pos < text.length) {+ const endPos = Math.min(pos + maxChunkSize, text.length);+ const chunkText = text.slice(pos, endPos);++ chunks.push({+ text: chunkText,+ index,+ startPos: pos,+ endPos,+ length: chunkText.length,+ tokenCount: Math.ceil(chunkText.length / 4),+ });++ // Move position with overlap+ pos = endPos - overlap;+ if (pos >= text.length - overlap) {+ break;+ }+ index++;+ }++ return chunks;+}++/**+ * Sentence-aware chunking - keeps sentences intact+ */+function chunkBySentence(+ text: string,+ config: Required<ChunkingConfig>+): Chunk[] {+ const { maxChunkSize, overlap, minChunkSize } = config;++ // Split into sentences+ const sentences = text.split(SENTENCE_ENDINGS).filter(s => s.trim().length > 0);++ const chunks: Chunk[] = [];+ let currentChunk = '';+ let currentStart = 0;+ let index = 0;+ let textPos = 0;++ for (const sentence of sentences) {+ const trimmedSentence = sentence.trim();++ // If adding this sentence exceeds max size, save current chunk+ if (currentChunk.length + trimmedSentence.length > maxChunkSize && currentChunk.length >= minChunkSize) {+ chunks.push({+ text: currentChunk.trim(),+ index,+ startPos: currentStart,+ endPos: textPos,+ length: currentChunk.length,+ tokenCount: Math.ceil(currentChunk.length / 4),+ });++ // Start new chunk with overlap (last part of previous chunk)+ const overlapText = currentChunk.slice(-overlap);+ currentChunk = overlapText + ' ' + trimmedSentence;+ currentStart = textPos - overlap;+ index++;+ } else {+ currentChunk += (currentChunk.length > 0 ? ' ' : '') + trimmedSentence;+ }++ textPos += trimmedSentence.length + 1;+ }++ // Add final chunk+ if (currentChunk.trim().length > 0) {+ chunks.push({+ text: currentChunk.trim(),+ index,+ startPos: currentStart,+ endPos: text.length,+ length: currentChunk.length,+ tokenCount: Math.ceil(currentChunk.length / 4),+ });+ }++ return chunks;+}++/**+ * Paragraph-aware chunking+ */+function chunkByParagraph(+ text: string,+ config: Required<ChunkingConfig>+): Chunk[] {+ const { maxChunkSize, minChunkSize } = config;++ // Split by paragraph breaks+ const paragraphs = text.split(PARAGRAPH_BREAKS).filter(p => p.trim().length > 0);++ const chunks: Chunk[] = [];+ let currentChunk = '';+ let currentStart = 0;+ let index = 0;+ let textPos = 0;++ for (const paragraph of paragraphs) {+ const trimmedPara = paragraph.trim();++ // If single paragraph exceeds max, fall back to sentence chunking+ if (trimmedPara.length > maxChunkSize) {+ if (currentChunk.length > 0) {+ chunks.push({+ text: currentChunk.trim(),+ index,+ startPos: currentStart,+ endPos: textPos,+ length: currentChunk.length,+ tokenCount: Math.ceil(currentChunk.length / 4),+ });+ index++;+ }++ // Chunk the large paragraph by sentence+ const subChunks = chunkBySentence(trimmedPara, config);+ for (const subChunk of subChunks) {+ chunks.push({+ ...subChunk,+ index,+ startPos: textPos + subChunk.startPos,+ endPos: textPos + subChunk.endPos,+ });+ index++;+ }++ currentChunk = '';+ currentStart = textPos + trimmedPara.length;+ } else if (currentChunk.length + trimmedPara.length > maxChunkSize && currentChunk.length >= minChunkSize) {+ chunks.push({+ text: currentChunk.trim(),+ index,+ startPos: currentStart,+ endPos: textPos,+ length: currentChunk.length,+ tokenCount: Math.ceil(currentChunk.length / 4),+ });++ currentChunk = trimmedPara;+ currentStart = textPos;+ index++;+ } else {+ currentChunk += (currentChunk.length > 0 ? '\n\n' : '') + trimmedPara;+ }++ textPos += trimmedPara.length + 2; // +2 for paragraph break+ }++ // Add final chunk+ if (currentChunk.trim().length > 0) {+ chunks.push({+ text: currentChunk.trim(),+ index,+ startPos: currentStart,+ endPos: text.length,+ length: currentChunk.length,+ tokenCount: Math.ceil(currentChunk.length / 4),+ });+ }++ return chunks;+}++/**+ * Token-based chunking (approximate - uses chars/4 as estimate)+ */+function chunkByToken(+ text: string,+ config: Required<ChunkingConfig>+): Chunk[] {+ // Convert token limits to character limits (rough estimate: 1 token ≈ 4 chars)+ const charConfig: Required<ChunkingConfig> = {+ ...config,+ maxChunkSize: config.maxChunkSize * 4,+ overlap: config.overlap * 4,+ minChunkSize: config.minChunkSize * 4,+ };++ // Use sentence-aware chunking with converted limits+ return chunkBySentence(text, charConfig);+}++/**+ * Estimate token count for text+ */+export function estimateTokens(text: string): number {+ // Simple estimation: ~4 characters per token on average+ return Math.ceil(text.length / 4);+}++/**+ * Reconstruct original text from chunks (approximate)+ */+export function reconstructFromChunks(chunks: Chunk[]): string {+ if (chunks.length === 0) return '';+ if (chunks.length === 1) return chunks[0].text;++ // Sort by index+ const sorted = [...chunks].sort((a, b) => a.index - b.index);++ // Simple concatenation (overlap removal is approximate)+ let result = sorted[0].text;++ for (let i = 1; i < sorted.length; i++) {+ const chunk = sorted[i];+ const prevChunk = sorted[i - 1];++ // Find overlap by looking for common suffix/prefix+ const overlapSize = Math.min(100, prevChunk.text.length, chunk.text.length);+ const prevSuffix = prevChunk.text.slice(-overlapSize);+ const currPrefix = chunk.text.slice(0, overlapSize);++ // Find longest common overlap+ let overlap = 0;+ for (let len = overlapSize; len > 0; len--) {+ if (currPrefix.startsWith(prevSuffix.slice(-len))) {+ overlap = len;+ break;+ }+ }++ result += ' ' + chunk.text.slice(overlap);+ }++ return result.replace(/\s+/g, ' ').trim();+}