Skip to content

Self-improving AI Employees for everyone

refactor(plugin): Wave 11.D.7 — document pipeline inlined

trohitgc21739325d40
diff --git a/server/nodes/document/_helpers.py b/server/nodes/document/_helpers.py
deleted file mode 100644
index 97783246..00000000
--- a/server/nodes/document/_helpers.py
+++ /dev/null
@@ -1,22 +0,0 @@
-"""Helpers for thin-delegating document plugins to their handlers."""
-
-from __future__ import annotations
-
-from typing import Any, Awaitable, Callable, Dict
-
-
-async def delegate(
- handler: Callable[..., Awaitable[Dict[str, Any]]],
- *,
- node_type: str,
- node_id: str,
- payload: Dict[str, Any],
- context: Dict[str, Any],
-) -> Any:
- response = await handler(
- node_id=node_id, node_type=node_type,
- parameters=payload, context=context,
- )
- if response.get("success"):
- return response.get("result") or response
- raise RuntimeError(response.get("error") or f"{node_type} failed")
diff --git a/server/nodes/document/document_parser.py b/server/nodes/document/document_parser.py
index bce35f02..50edc7ce 100644
--- a/server/nodes/document/document_parser.py
+++ b/server/nodes/document/document_parser.py
@@ -1,26 +1,53 @@
-"""Document Parser — Wave 11.C migration."""
+"""Document Parser — Wave 11.D.7 inlined."""
from __future__ import annotations
-from typing import Any, Literal, Optional
+import asyncio
+from pathlib import Path
+from typing import Literal, Optional
from pydantic import BaseModel, ConfigDict, Field
+from core.logging import get_logger
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
-from ._helpers import delegate
+logger = get_logger(__name__)
+
+
+def _parse_file_sync(path: Path, parser: str) -> str:
+ """Synchronous file parsing — runs in thread pool."""
+ if parser == 'pypdf':
+ from pypdf import PdfReader
+ return "\n\n".join(p.extract_text() or '' for p in PdfReader(str(path)).pages)
+ if parser == 'marker':
+ from marker.converters.pdf import PdfConverter
+ from marker.models import create_model_dict
+ converter = PdfConverter(artifact_dict=create_model_dict())
+ result = converter(str(path))
+ return result.markdown if hasattr(result, 'markdown') else str(result)
+ if parser == 'unstructured':
+ from unstructured.partition.auto import partition
+ return "\n\n".join(str(el) for el in partition(str(path)))
+ if parser == 'beautifulsoup':
+ from bs4 import BeautifulSoup
+ soup = BeautifulSoup(path.read_text(errors='ignore'), 'html.parser')
+ for s in soup(["script", "style"]):
+ s.decompose()
+ return soup.get_text(separator='\n')
+ raise ValueError(f"Unknown parser: {parser}")
class DocumentParserParams(BaseModel):
- file_path: str = Field(..., alias="filePath")
+ file_path: str = Field(default="", alias="filePath")
parser: Literal["pypdf", "marker", "unstructured", "beautifulsoup"] = "pypdf"
model_config = ConfigDict(populate_by_name=True, extra="allow")
class DocumentParserOutput(BaseModel):
- text: Optional[str] = None
- pages: Optional[int] = None
+ documents: Optional[list] = None
+ parsed_count: Optional[int] = None
+ failed: Optional[list] = None
model_config = ConfigDict(extra="allow")
@@ -45,9 +72,38 @@ class DocumentParserNode(ActionNode):
Output = DocumentParserOutput
@Operation("parse")
- async def parse(self, ctx: NodeContext, params: DocumentParserParams) -> Any:
- from services.handlers.document import handle_document_parser
- return await delegate(
- handle_document_parser, node_type=self.type, node_id=ctx.node_id,
- payload=params.model_dump(by_alias=True), context=ctx.raw,
+ async def parse(self, ctx: NodeContext, params: DocumentParserParams) -> DocumentParserOutput:
+ p = params.model_dump(by_alias=True)
+ files = p.get('files', [])
+ input_dir = p.get('inputDir', '')
+ parser = p.get('parser', 'pypdf')
+ file_pattern = p.get('filePattern', '*.pdf')
+ explicit_path = p.get('filePath') or p.get('file_path')
+
+ paths: list[Path] = []
+ if explicit_path:
+ paths.append(Path(explicit_path))
+ for f in files:
+ path = f.get('path', '') if isinstance(f, dict) else str(f)
+ if path:
+ paths.append(Path(path))
+ if input_dir and Path(input_dir).exists():
+ paths.extend(Path(input_dir).glob(file_pattern))
+
+ if not paths:
+ return DocumentParserOutput(documents=[], parsed_count=0, failed=[])
+
+ documents, failed = [], []
+ for path in paths:
+ try:
+ content = await asyncio.to_thread(_parse_file_sync, path, parser)
+ documents.append({
+ 'source': str(path), 'filename': path.name,
+ 'content': content, 'length': len(content), 'parser': parser,
+ })
+ except Exception as e:
+ failed.append({'file': str(path), 'error': str(e)})
+
+ return DocumentParserOutput(
+ documents=documents, parsed_count=len(documents), failed=failed,
)
diff --git a/server/nodes/document/embedding_generator.py b/server/nodes/document/embedding_generator.py
index 946a7ded..a99a9f7f 100644
--- a/server/nodes/document/embedding_generator.py
+++ b/server/nodes/document/embedding_generator.py
@@ -1,27 +1,31 @@
-"""Embedding Generator — Wave 11.C migration."""
+"""Embedding Generator — Wave 11.D.7 inlined."""
from __future__ import annotations
-from typing import Any, List, Literal, Optional
+import asyncio
+from typing import List, Literal, Optional
from pydantic import BaseModel, ConfigDict, Field
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
-from ._helpers import delegate
-
class EmbeddingGeneratorParams(BaseModel):
- chunks: List[str] = Field(default_factory=list)
+ chunks: List[Optional[dict]] = Field(default_factory=list)
provider: Literal["huggingface", "openai", "ollama"] = "huggingface"
model: str = Field(default="BAAI/bge-small-en-v1.5")
+ api_key: str = Field(default="", alias="apiKey")
- model_config = ConfigDict(extra="allow")
+ model_config = ConfigDict(populate_by_name=True, extra="allow")
class EmbeddingGeneratorOutput(BaseModel):
embeddings: Optional[list] = None
- dimension: Optional[int] = None
+ embedding_count: Optional[int] = None
+ dimensions: Optional[int] = None
+ chunks: Optional[list] = None
+ provider: Optional[str] = None
+ model: Optional[str] = None
model_config = ConfigDict(extra="allow")
@@ -46,9 +50,47 @@ class EmbeddingGeneratorNode(ActionNode):
Output = EmbeddingGeneratorOutput
@Operation("embed")
- async def embed(self, ctx: NodeContext, params: EmbeddingGeneratorParams) -> Any:
- from services.handlers.document import handle_embedding_generator
- return await delegate(
- handle_embedding_generator, node_type=self.type, node_id=ctx.node_id,
- payload=params.model_dump(by_alias=True), context=ctx.raw,
+ async def embed(self, ctx: NodeContext, params: EmbeddingGeneratorParams) -> EmbeddingGeneratorOutput:
+ p = params.model_dump(by_alias=True)
+ chunks = p.get('chunks', [])
+ provider = p.get('provider', 'huggingface')
+ model = p.get('model', 'BAAI/bge-small-en-v1.5')
+ api_key = p.get('apiKey', '')
+
+ if not chunks:
+ return EmbeddingGeneratorOutput(
+ embeddings=[], embedding_count=0, dimensions=0, chunks=[],
+ provider=provider, model=model,
+ )
+
+ texts = [c.get('content', '') if isinstance(c, dict) else str(c) for c in chunks]
+
+ if provider == 'huggingface':
+ try:
+ from langchain_huggingface import HuggingFaceEmbeddings
+ except ImportError:
+ raise RuntimeError(
+ "HuggingFace embeddings unavailable. "
+ "pip install langchain-huggingface sentence-transformers",
+ )
+ embedder = HuggingFaceEmbeddings(model_name=model)
+ elif provider == 'openai':
+ from langchain_openai import OpenAIEmbeddings
+ embedder = OpenAIEmbeddings(model=model, api_key=api_key)
+ elif provider == 'ollama':
+ from langchain_ollama import OllamaEmbeddings
+ embedder = OllamaEmbeddings(model=model)
+ else:
+ raise RuntimeError(f"Unknown provider: {provider}")
+
+ embeddings = await asyncio.to_thread(embedder.embed_documents, texts)
+ dimensions = len(embeddings[0]) if embeddings else 0
+
+ return EmbeddingGeneratorOutput(
+ embeddings=embeddings,
+ embedding_count=len(embeddings),
+ dimensions=dimensions,
+ chunks=chunks,
+ provider=provider,
+ model=model,
)
diff --git a/server/nodes/document/file_downloader.py b/server/nodes/document/file_downloader.py
index a429198c..988449b8 100644
--- a/server/nodes/document/file_downloader.py
+++ b/server/nodes/document/file_downloader.py
@@ -1,14 +1,19 @@
-"""File Downloader — Wave 11.C migration."""
+"""File Downloader — Wave 11.D.7 inlined."""
from __future__ import annotations
-from typing import Any, List, Optional
+import asyncio
+from pathlib import Path
+from typing import List, Optional
+from urllib.parse import unquote, urlparse
+import httpx
from pydantic import BaseModel, ConfigDict, Field
+from core.logging import get_logger
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
-from ._helpers import delegate
+logger = get_logger(__name__)
class FileDownloaderParams(BaseModel):
@@ -22,9 +27,11 @@ class FileDownloaderParams(BaseModel):
class FileDownloaderOutput(BaseModel):
- downloaded: Optional[list] = None
- skipped: Optional[list] = None
- failed: Optional[list] = None
+ downloaded: Optional[int] = None
+ skipped: Optional[int] = None
+ failed: Optional[int] = None
+ files: Optional[list] = None
+ output_dir: Optional[str] = None
model_config = ConfigDict(extra="allow")
@@ -49,9 +56,58 @@ class FileDownloaderNode(ActionNode):
Output = FileDownloaderOutput
@Operation("download")
- async def download(self, ctx: NodeContext, params: FileDownloaderParams) -> Any:
- from services.handlers.document import handle_file_downloader
- return await delegate(
- handle_file_downloader, node_type=self.type, node_id=ctx.node_id,
- payload=params.model_dump(by_alias=True), context=ctx.raw,
+ async def download(self, ctx: NodeContext, params: FileDownloaderParams) -> FileDownloaderOutput:
+ p = params.model_dump(by_alias=True)
+ items = p.get('items') or [{'url': u} for u in p.get('urls', [])]
+ workspace_dir = ctx.raw.get('workspace_dir', '')
+ default_dir = str(Path(workspace_dir) / 'downloads') if workspace_dir else p.get('outputDir', 'downloads')
+ output_dir = Path(p.get('outputDir') or default_dir)
+ max_workers = int(p.get('maxWorkers', 8))
+ skip_existing = p.get('skipExisting', True)
+ timeout = float(p.get('timeout', 60))
+
+ if not items:
+ return FileDownloaderOutput(
+ downloaded=0, skipped=0, failed=0, files=[], output_dir=str(output_dir),
+ )
+
+ output_dir.mkdir(parents=True, exist_ok=True)
+ semaphore = asyncio.Semaphore(max_workers)
+
+ async def dl(item):
+ async with semaphore:
+ url = item.get('url', '') if isinstance(item, dict) else str(item)
+ if not url:
+ return {'status': 'failed', 'error': 'Empty URL'}
+ filename = unquote(Path(urlparse(url).path).name or 'download')
+ file_path = output_dir / filename
+ if skip_existing and file_path.exists():
+ return {'status': 'skipped', 'path': str(file_path), 'url': url}
+ try:
+ async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+ resp = await client.get(url)
+ resp.raise_for_status()
+ file_path.write_bytes(resp.content)
+ return {
+ 'status': 'downloaded', 'path': str(file_path),
+ 'url': url, 'size': len(resp.content), 'filename': filename,
+ }
+ except Exception as e:
+ return {'status': 'failed', 'url': url, 'error': str(e)}
+
+ results = await asyncio.gather(*[dl(i) for i in items], return_exceptions=True)
+ downloaded, skipped, failed = [], [], []
+ for r in results:
+ if isinstance(r, Exception):
+ failed.append({'error': str(r)})
+ elif r.get('status') == 'downloaded':
+ downloaded.append(r)
+ elif r.get('status') == 'skipped':
+ skipped.append(r)
+ else:
+ failed.append(r)
+
+ return FileDownloaderOutput(
+ downloaded=len(downloaded), skipped=len(skipped), failed=len(failed),
+ files=downloaded, output_dir=str(output_dir),
)
diff --git a/server/nodes/document/http_scraper.py b/server/nodes/document/http_scraper.py
index 263cf323..ddc0215b 100644
--- a/server/nodes/document/http_scraper.py
+++ b/server/nodes/document/http_scraper.py
@@ -1,14 +1,23 @@
-"""HTTP Scraper — Wave 11.C migration."""
+"""HTTP Scraper — Wave 11.D.7 inlined.
+
+Scrapes links from web pages with optional date/page pagination.
+"""
from __future__ import annotations
+import json
+from datetime import datetime, timedelta
from typing import Any, Optional
+from urllib.parse import urljoin
+import httpx
+from bs4 import BeautifulSoup
from pydantic import BaseModel, ConfigDict, Field
+from core.logging import get_logger
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
-from ._helpers import delegate
+logger = get_logger(__name__)
class HttpScraperParams(BaseModel):
@@ -22,7 +31,8 @@ class HttpScraperParams(BaseModel):
class HttpScraperOutput(BaseModel):
items: Optional[list] = None
- count: Optional[int] = None
+ item_count: Optional[int] = None
+ errors: Optional[list] = None
model_config = ConfigDict(extra="allow")
@@ -47,9 +57,72 @@ class HttpScraperNode(ActionNode):
Output = HttpScraperOutput
@Operation("scrape")
- async def scrape(self, ctx: NodeContext, params: HttpScraperParams) -> Any:
- from services.handlers.document import handle_http_scraper
- return await delegate(
- handle_http_scraper, node_type=self.type, node_id=ctx.node_id,
- payload=params.model_dump(by_alias=True), context=ctx.raw,
- )
+ async def scrape(self, ctx: NodeContext, params: HttpScraperParams) -> HttpScraperOutput:
+ p = params.model_dump(by_alias=True)
+ url = p.get('url', '')
+ if not url:
+ raise RuntimeError("URL is required")
+
+ iteration_mode = p.get('iterationMode') or p.get('paginationMode', 'single')
+ link_selector = p.get('linkSelector', 'a[href$=".pdf"]')
+ headers_str = p.get('headers', '{}')
+ headers = json.loads(headers_str) if isinstance(headers_str, str) and headers_str else {}
+
+ urls_to_fetch = []
+ if iteration_mode == 'date':
+ start_date = p.get('startDate', '')
+ end_date = p.get('endDate', '')
+ placeholder = p.get('datePlaceholder', '{date}')
+ if not start_date or not end_date:
+ raise RuntimeError("Start/end dates required for date mode")
+ start = datetime.strptime(start_date, "%Y-%m-%d")
+ end = datetime.strptime(end_date, "%Y-%m-%d")
+ current = start
+ while current <= end:
+ urls_to_fetch.append((
+ url.replace(placeholder, current.strftime("%Y-%m-%d")),
+ {'date': current.isoformat()},
+ ))
+ current += timedelta(days=1)
+ elif iteration_mode == 'page':
+ start_page = int(p.get('startPage', 1))
+ end_page = int(p.get('endPage', 10))
+ for page in range(start_page, end_page + 1):
+ urls_to_fetch.append((url.replace('{page}', str(page)), {'page': page}))
+ else:
+ urls_to_fetch.append((url, {}))
+
+ proxy_url = None
+ if p.get('useProxy', False):
+ try:
+ from services.proxy.service import get_proxy_service
+ proxy_svc = get_proxy_service()
+ if proxy_svc and proxy_svc.is_enabled():
+ proxy_url = await proxy_svc.get_proxy_url(url, p)
+ except Exception as e:
+ logger.warning("[httpScraper] Proxy lookup failed", error=str(e))
+
+ items, errors = [], []
+ client_kwargs: dict = {"timeout": 30, "follow_redirects": True}
+ if proxy_url:
+ client_kwargs["proxy"] = proxy_url
+
+ async with httpx.AsyncClient(**client_kwargs) as client:
+ for fetch_url, meta in urls_to_fetch:
+ try:
+ response = await client.get(fetch_url, headers=headers)
+ response.raise_for_status()
+ soup = BeautifulSoup(response.text, 'html.parser')
+ for el in soup.select(link_selector):
+ href = el.get('href', '')
+ if href:
+ items.append({
+ 'url': urljoin(fetch_url, href),
+ 'text': el.get_text(strip=True),
+ 'source_url': fetch_url,
+ **meta,
+ })
+ except Exception as e:
+ errors.append(f"{fetch_url}: {e}")
+
+ return HttpScraperOutput(items=items, item_count=len(items), errors=errors)
diff --git a/server/nodes/document/text_chunker.py b/server/nodes/document/text_chunker.py
index d683b60c..511b58af 100644
--- a/server/nodes/document/text_chunker.py
+++ b/server/nodes/document/text_chunker.py
@@ -1,28 +1,26 @@
-"""Text Chunker — Wave 11.C migration."""
+"""Text Chunker — Wave 11.D.7 inlined."""
from __future__ import annotations
-from typing import Any, Literal, Optional
+from typing import Literal, Optional
from pydantic import BaseModel, ConfigDict, Field
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
-from ._helpers import delegate
-
class TextChunkerParams(BaseModel):
text: str = Field(default="")
strategy: Literal["recursive", "markdown", "token"] = "recursive"
chunk_size: int = Field(default=1000, alias="chunkSize", ge=100, le=8000)
- overlap: int = Field(default=200, ge=0, le=1000)
+ overlap: int = Field(default=200, alias="chunkOverlap", ge=0, le=1000)
model_config = ConfigDict(populate_by_name=True, extra="allow")
class TextChunkerOutput(BaseModel):
chunks: Optional[list] = None
- count: Optional[int] = None
+ chunk_count: Optional[int] = None
model_config = ConfigDict(extra="allow")
@@ -47,9 +45,35 @@ class TextChunkerNode(ActionNode):
Output = TextChunkerOutput
@Operation("chunk")
- async def chunk(self, ctx: NodeContext, params: TextChunkerParams) -> Any:
- from services.handlers.document import handle_text_chunker
- return await delegate(
- handle_text_chunker, node_type=self.type, node_id=ctx.node_id,
- payload=params.model_dump(by_alias=True), context=ctx.raw,
+ async def chunk(self, ctx: NodeContext, params: TextChunkerParams) -> TextChunkerOutput:
+ from langchain_text_splitters import (
+ MarkdownTextSplitter, RecursiveCharacterTextSplitter,
)
+
+ p = params.model_dump(by_alias=True)
+ documents = p.get('documents') or ([{'content': p.get('text', '')}] if p.get('text') else [])
+ chunk_size = int(p.get('chunkSize', 1000))
+ chunk_overlap = int(p.get('chunkOverlap') or p.get('overlap', 200))
+ strategy = p.get('strategy', 'recursive')
+
+ if not documents:
+ return TextChunkerOutput(chunks=[], chunk_count=0)
+
+ if strategy == 'markdown':
+ splitter = MarkdownTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+ else:
+ splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+ chunks = []
+ for doc in documents:
+ content = doc.get('content', '') if isinstance(doc, dict) else str(doc)
+ source = doc.get('source', 'input') if isinstance(doc, dict) else 'input'
+ if not content:
+ continue
+ for i, chunk_text in enumerate(splitter.split_text(content)):
+ chunks.append({
+ 'source': source, 'chunk_index': i,
+ 'content': chunk_text, 'length': len(chunk_text),
+ })
+
+ return TextChunkerOutput(chunks=chunks, chunk_count=len(chunks))
diff --git a/server/nodes/document/vector_store.py b/server/nodes/document/vector_store.py
index b968ab2c..4156261b 100644
--- a/server/nodes/document/vector_store.py
+++ b/server/nodes/document/vector_store.py
@@ -1,35 +1,223 @@
-"""Vector Store — Wave 11.C migration."""
+"""Vector Store — Wave 11.D.7 inlined.
+
+Backends: ChromaDB (local), Qdrant (self-hosted), Pinecone (cloud).
+Each backend helper is a pure function that reads the raw param dict;
+the dispatcher routes by ``backend`` + ``operation``.
+"""
from __future__ import annotations
-from typing import Any, Literal, Optional
+import asyncio
+import uuid
+from typing import Any, Dict, Literal, Optional
from pydantic import BaseModel, ConfigDict, Field
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
-from ._helpers import delegate
-
class VectorStoreParams(BaseModel):
operation: Literal["store", "query", "delete"] = "store"
- backend: Literal["chromadb", "qdrant", "pinecone"] = "chromadb"
- collection: str = Field(default="default")
+ backend: Literal["chroma", "chromadb", "qdrant", "pinecone"] = "chroma"
+ collection_name: str = Field(default="documents", alias="collectionName")
embeddings: Optional[list] = None
- query: str = Field(default="")
+ chunks: Optional[list] = None
+ query_embedding: Optional[list] = Field(default=None, alias="queryEmbedding")
top_k: int = Field(default=5, alias="topK", ge=1, le=100)
+ ids: Optional[list] = None
+ persist_dir: str = Field(default="./data/vectors", alias="persistDir")
+ qdrant_url: str = Field(default="http://localhost:6333", alias="qdrantUrl")
+ pinecone_api_key: str = Field(default="", alias="pineconeApiKey")
model_config = ConfigDict(populate_by_name=True, extra="allow")
class VectorStoreOutput(BaseModel):
operation: Optional[str] = None
+ backend: Optional[str] = None
+ collection_name: Optional[str] = None
matches: Optional[list] = None
stored_count: Optional[int] = None
+ collection_count: Optional[int] = None
+ deleted: Optional[bool] = None
+ count: Optional[int] = None
model_config = ConfigDict(extra="allow")
+async def _chroma_op(operation: str, params: Dict[str, Any], collection: str) -> Dict[str, Any]:
+ try:
+ import chromadb
+ except ImportError:
+ raise RuntimeError("ChromaDB unavailable. pip install chromadb")
+
+ persist_dir = params.get('persistDir', './data/vectors')
+ client = chromadb.PersistentClient(path=persist_dir)
+ coll = client.get_or_create_collection(name=collection)
+
+ if operation == 'store':
+ embeddings = params.get('embeddings', [])
+ chunks = params.get('chunks', [])
+ if not embeddings:
+ return {"stored_count": 0, "collection_count": coll.count()}
+ ids = [str(uuid.uuid4()) for _ in embeddings]
+ docs = [c.get('content', '') if isinstance(c, dict) else str(c) for c in chunks]
+ while len(docs) < len(embeddings):
+ docs.append('')
+ metas = [
+ {'source': c.get('source', 'unknown'), 'chunk_index': c.get('chunk_index', i)}
+ if isinstance(c, dict) else {'source': 'input', 'chunk_index': i}
+ for i, c in enumerate(chunks)
+ ]
+ while len(metas) < len(embeddings):
+ metas.append({'source': 'unknown', 'chunk_index': len(metas)})
+ await asyncio.to_thread(coll.add, ids=ids, embeddings=embeddings, documents=docs, metadatas=metas)
+ return {"stored_count": len(embeddings), "collection_count": coll.count()}
+
+ if operation == 'query':
+ query_emb = params.get('queryEmbedding', [])
+ top_k = int(params.get('topK', 5))
+ if not query_emb:
+ return {"matches": []}
+ results = coll.query(query_embeddings=[query_emb], n_results=top_k)
+ matches = []
+ if results['ids'] and results['ids'][0]:
+ for i in range(len(results['ids'][0])):
+ matches.append({
+ 'id': results['ids'][0][i],
+ 'document': results['documents'][0][i] if results['documents'] else '',
+ 'metadata': results['metadatas'][0][i] if results['metadatas'] else {},
+ 'distance': results['distances'][0][i] if results.get('distances') else None,
+ })
+ return {"matches": matches}
+
+ if operation == 'delete':
+ ids = params.get('ids', [])
+ if ids:
+ await asyncio.to_thread(coll.delete, ids=ids)
+ return {"deleted": True, "count": len(ids)}
+
+ return {}
+
+
+async def _qdrant_op(operation: str, params: Dict[str, Any], collection: str) -> Dict[str, Any]:
+ try:
+ from qdrant_client import QdrantClient
+ from qdrant_client.models import Distance, PointStruct, VectorParams
+ except ImportError:
+ raise RuntimeError("Qdrant client unavailable. pip install qdrant-client")
+
+ url = params.get('qdrantUrl', 'http://localhost:6333')
+ client = QdrantClient(url=url)
+
+ if operation == 'store':
+ embeddings = params.get('embeddings', [])
+ chunks = params.get('chunks', [])
+ if not embeddings:
+ return {"stored_count": 0}
+ vec_size = len(embeddings[0])
+ colls = client.get_collections().collections
+ if collection not in [c.name for c in colls]:
+ client.create_collection(
+ collection,
+ vectors_config=VectorParams(size=vec_size, distance=Distance.COSINE),
+ )
+ points = []
+ for i, emb in enumerate(embeddings):
+ payload: Dict[str, Any] = {}
+ if i < len(chunks):
+ c = chunks[i]
+ if isinstance(c, dict):
+ payload = {
+ 'content': c.get('content', ''),
+ 'source': c.get('source', 'unknown'),
+ 'chunk_index': c.get('chunk_index', i),
+ }
+ else:
+ payload = {'content': str(c), 'source': 'input', 'chunk_index': i}
+ points.append(PointStruct(id=str(uuid.uuid4()), vector=emb, payload=payload))
+ await asyncio.to_thread(client.upsert, collection_name=collection, points=points)
+ return {"stored_count": len(embeddings)}
+
+ if operation == 'query':
+ query_emb = params.get('queryEmbedding', [])
+ top_k = int(params.get('topK', 5))
+ if not query_emb:
+ return {"matches": []}
+ results = await asyncio.to_thread(
+ client.search, collection_name=collection, query_vector=query_emb, limit=top_k,
+ )
+ return {"matches": [
+ {'id': str(r.id), 'document': r.payload.get('content', ''),
+ 'metadata': r.payload, 'score': r.score}
+ for r in results
+ ]}
+
+ if operation == 'delete':
+ ids = params.get('ids', [])
+ if ids:
+ await asyncio.to_thread(client.delete, collection_name=collection, points_selector=ids)
+ return {"deleted": True, "count": len(ids)}
+
+ return {}
+
+
+async def _pinecone_op(operation: str, params: Dict[str, Any], collection: str) -> Dict[str, Any]:
+ from pinecone import Pinecone
+
+ api_key = params.get('pineconeApiKey', '')
+ if not api_key:
+ raise RuntimeError("Pinecone API key required")
+
+ pc = Pinecone(api_key=api_key)
+ index = pc.Index(collection)
+
+ if operation == 'store':
+ embeddings = params.get('embeddings', [])
+ chunks = params.get('chunks', [])
+ if not embeddings:
+ return {"stored_count": 0}
+ vectors = []
+ for i, emb in enumerate(embeddings):
+ meta: Dict[str, Any] = {}
+ if i < len(chunks):
+ c = chunks[i]
+ if isinstance(c, dict):
+ meta = {
+ 'content': c.get('content', ''),
+ 'source': c.get('source', 'unknown'),
+ 'chunk_index': c.get('chunk_index', i),
+ }
+ else:
+ meta = {'content': str(c), 'source': 'input', 'chunk_index': i}
+ vectors.append({'id': str(uuid.uuid4()), 'values': emb, 'metadata': meta})
+ await asyncio.to_thread(index.upsert, vectors=vectors)
+ return {"stored_count": len(embeddings)}
+
+ if operation == 'query':
+ query_emb = params.get('queryEmbedding', [])
+ top_k = int(params.get('topK', 5))
+ if not query_emb:
+ return {"matches": []}
+ results = await asyncio.to_thread(
+ index.query, vector=query_emb, top_k=top_k, include_metadata=True,
+ )
+ return {"matches": [
+ {'id': m.id,
+ 'document': m.metadata.get('content', '') if m.metadata else '',
+ 'metadata': m.metadata or {}, 'score': m.score}
+ for m in results.matches
+ ]}
+
+ if operation == 'delete':
+ ids = params.get('ids', [])
+ if ids:
+ await asyncio.to_thread(index.delete, ids=ids)
+ return {"deleted": True, "count": len(ids)}
+
+ return {}
+
+
class VectorStoreNode(ActionNode):
type = "vectorStore"
display_name = "Vector Store"
@@ -50,9 +238,23 @@ class VectorStoreNode(ActionNode):
Output = VectorStoreOutput
@Operation("dispatch")
- async def dispatch(self, ctx: NodeContext, params: VectorStoreParams) -> Any:
- from services.handlers.document import handle_vector_store
- return await delegate(
- handle_vector_store, node_type=self.type, node_id=ctx.node_id,
- payload=params.model_dump(by_alias=True), context=ctx.raw,
- )
+ async def dispatch(self, ctx: NodeContext, params: VectorStoreParams) -> VectorStoreOutput:
+ p = params.model_dump(by_alias=True)
+ operation = p.get('operation', 'store')
+ backend_raw = p.get('backend', 'chroma')
+ backend = 'chroma' if backend_raw in ('chroma', 'chromadb') else backend_raw
+ collection = p.get('collectionName', 'documents')
+
+ if backend == 'chroma':
+ result = await _chroma_op(operation, p, collection)
+ elif backend == 'qdrant':
+ result = await _qdrant_op(operation, p, collection)
+ elif backend == 'pinecone':
+ result = await _pinecone_op(operation, p, collection)
+ else:
+ raise RuntimeError(f"Unknown backend: {backend_raw}")
+
+ result['backend'] = backend
+ result['collection_name'] = collection
+ result['operation'] = operation
+ return VectorStoreOutput(**result)
diff --git a/server/services/handlers/__init__.py b/server/services/handlers/__init__.py
index b2e7de6d..cad2cdb3 100644
--- a/server/services/handlers/__init__.py
+++ b/server/services/handlers/__init__.py
@@ -87,15 +87,7 @@ from .social import (
handle_social_send,
)
-# Document processing handlers
-from .document import (
- handle_http_scraper,
- handle_file_downloader,
- handle_document_parser,
- handle_text_chunker,
- handle_embedding_generator,
- handle_vector_store,
-)
+# Document handlers inlined into nodes/document/*.py (Wave 11.D.7).
# Search handlers
from .search import (
@@ -179,13 +171,6 @@ __all__ = [
# Social
'handle_social_receive',
'handle_social_send',
- # Document processing
- 'handle_http_scraper',
- 'handle_file_downloader',
- 'handle_document_parser',
- 'handle_text_chunker',
- 'handle_embedding_generator',
- 'handle_vector_store',
# Search
'handle_brave_search',
'handle_serper_search',
diff --git a/server/services/handlers/document.py b/server/services/handlers/document.py
deleted file mode 100644
index c9e8333f..00000000
--- a/server/services/handlers/document.py
+++ /dev/null
@@ -1,625 +0,0 @@
-"""Document processing handlers for MachinaOs workflow nodes.
-
-Each handler is an independent async function that can be executed by
-the Temporal worker system for distributed processing.
-
-Handlers:
-- handle_http_scraper: Scrape links from web pages with pagination
-- handle_file_downloader: Download files in parallel
-- handle_document_parser: Parse documents (PyPDF, Marker, Unstructured)
-- handle_text_chunker: Split text into chunks (LangChain)
-- handle_embedding_generator: Generate embeddings (HF, OpenAI, Ollama)
-- handle_vector_store: Store/query vectors (ChromaDB, Qdrant, Pinecone)
-"""
-
-import asyncio
-import json
-import time
-from datetime import datetime, timedelta
-from pathlib import Path
-from typing import Dict, Any
-from urllib.parse import urljoin, urlparse, unquote
-
-from core.logging import get_logger
-
-logger = get_logger(__name__)
-
-
-# =============================================================================
-# HTTP Scraper
-# =============================================================================
-
-async def handle_http_scraper(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Scrape links from web pages with date/page pagination support."""
- import httpx
- from bs4 import BeautifulSoup
- start_time = time.time()
-
- try:
- url = parameters.get('url', '')
- iteration_mode = parameters.get('iterationMode', 'single')
- link_selector = parameters.get('linkSelector', 'a[href$=".pdf"]')
- headers_str = parameters.get('headers', '{}')
-
- if not url:
- raise ValueError("URL is required")
-
- headers = json.loads(headers_str) if headers_str else {}
- items, errors = [], []
- urls_to_fetch = []
-
- if iteration_mode == 'date':
- start_date = parameters.get('startDate', '')
- end_date = parameters.get('endDate', '')
- placeholder = parameters.get('datePlaceholder', '{date}')
- if not start_date or not end_date:
- raise ValueError("Start/end dates required for date mode")
- start = datetime.strptime(start_date, "%Y-%m-%d")
- end = datetime.strptime(end_date, "%Y-%m-%d")
- current = start
- while current <= end:
- urls_to_fetch.append((
- url.replace(placeholder, current.strftime("%Y-%m-%d")),
- {'date': current.isoformat()}
- ))
- current += timedelta(days=1)
- elif iteration_mode == 'page':
- start_page = int(parameters.get('startPage', 1))
- end_page = int(parameters.get('endPage', 10))
- for page in range(start_page, end_page + 1):
- urls_to_fetch.append((url.replace('{page}', str(page)), {'page': page}))
- else:
- urls_to_fetch.append((url, {}))
-
- # Transparent proxy injection
- proxy_url = None
- if parameters.get('useProxy', False):
- try:
- from services.proxy.service import get_proxy_service
- proxy_svc = get_proxy_service()
- if proxy_svc and proxy_svc.is_enabled():
- proxy_url = await proxy_svc.get_proxy_url(url, parameters)
- except Exception as e:
- logger.warning("[httpScraper] Proxy lookup failed, proceeding without proxy", error=str(e))
-
- logger.info("[httpScraper] Starting", node_id=node_id, urls=len(urls_to_fetch),
- proxy=bool(proxy_url))
-
- client_kwargs: dict = {"timeout": 30, "follow_redirects": True}
- if proxy_url:
- client_kwargs["proxy"] = proxy_url
-
- async with httpx.AsyncClient(**client_kwargs) as client:
- for fetch_url, meta in urls_to_fetch:
- try:
- response = await client.get(fetch_url, headers=headers)
- response.raise_for_status()
- soup = BeautifulSoup(response.text, 'html.parser')
- for el in soup.select(link_selector):
- href = el.get('href', '')
- if href:
- items.append({
- 'url': urljoin(fetch_url, href),
- 'text': el.get_text(strip=True),
- 'source_url': fetch_url,
- **meta
- })
- except Exception as e:
- errors.append(f"{fetch_url}: {str(e)}")
-
- return {
- "success": True,
- "node_id": node_id,
- "node_type": node_type,
- "result": {"items": items, "item_count": len(items), "errors": errors},
- "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
- except Exception as e:
- logger.error("[httpScraper] Failed", node_id=node_id, error=str(e))
- return {
- "success": False, "node_id": node_id, "node_type": node_type,
- "error": str(e), "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
-
-# =============================================================================
-# File Downloader
-# =============================================================================
-
-async def handle_file_downloader(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Download files from URLs in parallel using semaphore for concurrency."""
- import httpx
- start_time = time.time()
-
- try:
- items = parameters.get('items', [])
- workspace_dir = context.get('workspace_dir', '')
- default_dir = str(Path(workspace_dir) / 'downloads') if workspace_dir else 'downloads'
- output_dir = Path(parameters.get('outputDir') or default_dir)
- max_workers = int(parameters.get('maxWorkers', 8))
- skip_existing = parameters.get('skipExisting', True)
- timeout = float(parameters.get('timeout', 60))
-
- if not items:
- return {
- "success": True, "node_id": node_id, "node_type": node_type,
- "result": {"downloaded": 0, "skipped": 0, "failed": 0, "files": []},
- "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
- output_dir.mkdir(parents=True, exist_ok=True)
- downloaded, skipped, failed = [], [], []
- semaphore = asyncio.Semaphore(max_workers)
-
- async def download_file(item):
- async with semaphore:
- url = item.get('url', '') if isinstance(item, dict) else str(item)
- if not url:
- return {'status': 'failed', 'error': 'Empty URL'}
- filename = unquote(Path(urlparse(url).path).name or 'download')
- file_path = output_dir / filename
- if skip_existing and file_path.exists():
- return {'status': 'skipped', 'path': str(file_path), 'url': url}
- try:
- async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
- response = await client.get(url)
- response.raise_for_status()
- file_path.write_bytes(response.content)
- return {'status': 'downloaded', 'path': str(file_path), 'url': url,
- 'size': len(response.content), 'filename': filename}
- except Exception as e:
- return {'status': 'failed', 'url': url, 'error': str(e)}
-
- logger.info("[fileDownloader] Starting", node_id=node_id, items=len(items))
- results = await asyncio.gather(*[download_file(i) for i in items], return_exceptions=True)
-
- for r in results:
- if isinstance(r, Exception):
- failed.append({'error': str(r)})
- elif r.get('status') == 'downloaded':
- downloaded.append(r)
- elif r.get('status') == 'skipped':
- skipped.append(r)
- else:
- failed.append(r)
-
- return {
- "success": True, "node_id": node_id, "node_type": node_type,
- "result": {"downloaded": len(downloaded), "skipped": len(skipped),
- "failed": len(failed), "files": downloaded, "output_dir": str(output_dir)},
- "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
- except Exception as e:
- logger.error("[fileDownloader] Failed", node_id=node_id, error=str(e))
- return {
- "success": False, "node_id": node_id, "node_type": node_type,
- "error": str(e), "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
-
-# =============================================================================
-# Document Parser
-# =============================================================================
-
-def _parse_file_sync(path: Path, parser: str) -> str:
- """Synchronous file parsing - runs in thread pool."""
- if parser == 'pypdf':
- from pypdf import PdfReader
- return "\n\n".join(p.extract_text() or '' for p in PdfReader(str(path)).pages)
- elif parser == 'marker':
- from marker.converters.pdf import PdfConverter
- from marker.models import create_model_dict
- converter = PdfConverter(artifact_dict=create_model_dict())
- result = converter(str(path))
- return result.markdown if hasattr(result, 'markdown') else str(result)
- elif parser == 'unstructured':
- from unstructured.partition.auto import partition
- return "\n\n".join(str(el) for el in partition(str(path)))
- elif parser == 'beautifulsoup':
- from bs4 import BeautifulSoup
- soup = BeautifulSoup(path.read_text(errors='ignore'), 'html.parser')
- for s in soup(["script", "style"]):
- s.decompose()
- return soup.get_text(separator='\n')
- raise ValueError(f"Unknown parser: {parser}")
-
-
-async def handle_document_parser(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Parse documents to text using configurable parsers."""
- start_time = time.time()
-
- try:
- files = parameters.get('files', [])
- input_dir = parameters.get('inputDir', '')
- parser = parameters.get('parser', 'pypdf')
- file_pattern = parameters.get('filePattern', '*.pdf')
-
- paths = []
- for f in files:
- p = f.get('path', '') if isinstance(f, dict) else str(f)
- if p:
- paths.append(Path(p))
- if input_dir and Path(input_dir).exists():
- paths.extend(Path(input_dir).glob(file_pattern))
-
- if not paths:
- return {
- "success": True, "node_id": node_id, "node_type": node_type,
- "result": {"documents": [], "parsed_count": 0, "failed": []},
- "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
- logger.info("[documentParser] Starting", node_id=node_id, files=len(paths), parser=parser)
- documents, failed = [], []
-
- for path in paths:
- try:
- content = await asyncio.to_thread(_parse_file_sync, path, parser)
- documents.append({'source': str(path), 'filename': path.name,
- 'content': content, 'length': len(content), 'parser': parser})
- except Exception as e:
- failed.append({'file': str(path), 'error': str(e)})
-
- return {
- "success": True, "node_id": node_id, "node_type": node_type,
- "result": {"documents": documents, "parsed_count": len(documents), "failed": failed},
- "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
- except Exception as e:
- logger.error("[documentParser] Failed", node_id=node_id, error=str(e))
- return {
- "success": False, "node_id": node_id, "node_type": node_type,
- "error": str(e), "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
-
-# =============================================================================
-# Text Chunker
-# =============================================================================
-
-async def handle_text_chunker(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Split text into overlapping chunks using LangChain splitters."""
- start_time = time.time()
-
- try:
- documents = parameters.get('documents', [])
- chunk_size = int(parameters.get('chunkSize', 1024))
- chunk_overlap = int(parameters.get('chunkOverlap', 200))
- strategy = parameters.get('strategy', 'recursive')
-
- if not documents:
- return {
- "success": True, "node_id": node_id, "node_type": node_type,
- "result": {"chunks": [], "chunk_count": 0},
- "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
- from langchain_text_splitters import RecursiveCharacterTextSplitter, MarkdownTextSplitter
-
- if strategy == 'markdown':
- splitter = MarkdownTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
- else:
- splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
-
- logger.info("[textChunker] Starting", node_id=node_id, docs=len(documents))
- chunks = []
-
- for doc in documents:
- content = doc.get('content', '') if isinstance(doc, dict) else str(doc)
- source = doc.get('source', 'input') if isinstance(doc, dict) else 'input'
- if not content:
- continue
- for i, chunk_text in enumerate(splitter.split_text(content)):
- chunks.append({'source': source, 'chunk_index': i,
- 'content': chunk_text, 'length': len(chunk_text)})
-
- return {
- "success": True, "node_id": node_id, "node_type": node_type,
- "result": {"chunks": chunks, "chunk_count": len(chunks)},
- "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
- except Exception as e:
- logger.error("[textChunker] Failed", node_id=node_id, error=str(e))
- return {
- "success": False, "node_id": node_id, "node_type": node_type,
- "error": str(e), "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
-
-# =============================================================================
-# Embedding Generator
-# =============================================================================
-
-async def handle_embedding_generator(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Generate embeddings using HuggingFace, OpenAI, or Ollama."""
- start_time = time.time()
-
- try:
- chunks = parameters.get('chunks', [])
- provider = parameters.get('provider', 'huggingface')
- model = parameters.get('model', 'BAAI/bge-small-en-v1.5')
- api_key = parameters.get('apiKey', '')
-
- if not chunks:
- return {
- "success": True, "node_id": node_id, "node_type": node_type,
- "result": {"embeddings": [], "embedding_count": 0, "dimensions": 0, "chunks": []},
- "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
- texts = [c.get('content', '') if isinstance(c, dict) else str(c) for c in chunks]
- logger.info("[embeddingGenerator] Starting", node_id=node_id, texts=len(texts), provider=provider)
-
- if provider == 'huggingface':
- try:
- from langchain_huggingface import HuggingFaceEmbeddings
- except ImportError:
- raise ImportError("HuggingFace embeddings not available. Install with: pip install langchain-huggingface sentence-transformers")
- embedder = HuggingFaceEmbeddings(model_name=model)
- elif provider == 'openai':
- from langchain_openai import OpenAIEmbeddings
- embedder = OpenAIEmbeddings(model=model, api_key=api_key)
- elif provider == 'ollama':
- from langchain_ollama import OllamaEmbeddings
- embedder = OllamaEmbeddings(model=model)
- else:
- raise ValueError(f"Unknown provider: {provider}")
-
- embeddings = await asyncio.to_thread(embedder.embed_documents, texts)
- dimensions = len(embeddings[0]) if embeddings else 0
-
- return {
- "success": True, "node_id": node_id, "node_type": node_type,
- "result": {"embeddings": embeddings, "embedding_count": len(embeddings),
- "dimensions": dimensions, "chunks": chunks, "provider": provider, "model": model},
- "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
- except Exception as e:
- logger.error("[embeddingGenerator] Failed", node_id=node_id, error=str(e))
- return {
- "success": False, "node_id": node_id, "node_type": node_type,
- "error": str(e), "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
-
-# =============================================================================
-# Vector Store
-# =============================================================================
-
-async def handle_vector_store(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Store or query vectors using ChromaDB, Qdrant, or Pinecone."""
- start_time = time.time()
-
- try:
- operation = parameters.get('operation', 'store')
- backend = parameters.get('backend', 'chroma')
- collection_name = parameters.get('collectionName', 'documents')
-
- logger.info("[vectorStore] Starting", node_id=node_id, op=operation, backend=backend)
-
- if backend == 'chroma':
- result = await _chroma_op(operation, parameters, collection_name)
- elif backend == 'qdrant':
- result = await _qdrant_op(operation, parameters, collection_name)
- elif backend == 'pinecone':
- result = await _pinecone_op(operation, parameters, collection_name)
- else:
- raise ValueError(f"Unknown backend: {backend}")
-
- result['backend'] = backend
- result['collection_name'] = collection_name
-
- return {
- "success": True, "node_id": node_id, "node_type": node_type,
- "result": result, "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
- except Exception as e:
- logger.error("[vectorStore] Failed", node_id=node_id, error=str(e))
- return {
- "success": False, "node_id": node_id, "node_type": node_type,
- "error": str(e), "execution_time": time.time() - start_time,
- "timestamp": datetime.now().isoformat()
- }
-
-
-async def _chroma_op(operation: str, params: Dict, collection: str) -> Dict:
- """ChromaDB operations."""
- try:
- import chromadb
- except ImportError:
- raise ImportError("ChromaDB not available. Install with: pip install chromadb")
- import uuid
-
- persist_dir = params.get('persistDir', './data/vectors')
- client = chromadb.PersistentClient(path=persist_dir)
- coll = client.get_or_create_collection(name=collection)
-
- if operation == 'store':
- embeddings = params.get('embeddings', [])
- chunks = params.get('chunks', [])
- if not embeddings:
- return {"stored_count": 0, "collection_count": coll.count()}
- ids = [str(uuid.uuid4()) for _ in embeddings]
- docs = [c.get('content', '') if isinstance(c, dict) else str(c) for c in chunks]
- while len(docs) < len(embeddings):
- docs.append('')
- metas = [{'source': c.get('source', 'unknown'), 'chunk_index': c.get('chunk_index', i)}
- if isinstance(c, dict) else {'source': 'input', 'chunk_index': i}
- for i, c in enumerate(chunks)]
- while len(metas) < len(embeddings):
- metas.append({'source': 'unknown', 'chunk_index': len(metas)})
- await asyncio.to_thread(coll.add, ids=ids, embeddings=embeddings, documents=docs, metadatas=metas)
- return {"stored_count": len(embeddings), "collection_count": coll.count()}
-
- elif operation == 'query':
- query_emb = params.get('queryEmbedding', [])
- top_k = int(params.get('topK', 5))
- if not query_emb:
- return {"matches": []}
- results = coll.query(query_embeddings=[query_emb], n_results=top_k)
- matches = []
- if results['ids'] and results['ids'][0]:
- for i in range(len(results['ids'][0])):
- matches.append({
- 'id': results['ids'][0][i],
- 'document': results['documents'][0][i] if results['documents'] else '',
- 'metadata': results['metadatas'][0][i] if results['metadatas'] else {},
- 'distance': results['distances'][0][i] if results.get('distances') else None
- })
- return {"matches": matches}
-
- elif operation == 'delete':
- ids = params.get('ids', [])
- if ids:
- await asyncio.to_thread(coll.delete, ids=ids)
- return {"deleted": True, "count": len(ids)}
-
- return {}
-
-
-async def _qdrant_op(operation: str, params: Dict, collection: str) -> Dict:
- """Qdrant operations."""
- try:
- from qdrant_client import QdrantClient
- from qdrant_client.models import VectorParams, Distance, PointStruct
- except ImportError:
- raise ImportError("Qdrant client not available. Install with: pip install qdrant-client")
- import uuid
-
- url = params.get('qdrantUrl', 'http://localhost:6333')
- client = QdrantClient(url=url)
-
- if operation == 'store':
- embeddings = params.get('embeddings', [])
- chunks = params.get('chunks', [])
- if not embeddings:
- return {"stored_count": 0}
- vec_size = len(embeddings[0])
- colls = client.get_collections().collections
- if collection not in [c.name for c in colls]:
- client.create_collection(collection, vectors_config=VectorParams(size=vec_size, distance=Distance.COSINE))
- points = []
- for i, emb in enumerate(embeddings):
- payload = {}
- if i < len(chunks):
- c = chunks[i]
- if isinstance(c, dict):
- payload = {'content': c.get('content', ''), 'source': c.get('source', 'unknown'),
- 'chunk_index': c.get('chunk_index', i)}
- else:
- payload = {'content': str(c), 'source': 'input', 'chunk_index': i}
- points.append(PointStruct(id=str(uuid.uuid4()), vector=emb, payload=payload))
- await asyncio.to_thread(client.upsert, collection_name=collection, points=points)
- return {"stored_count": len(embeddings)}
-
- elif operation == 'query':
- query_emb = params.get('queryEmbedding', [])
- top_k = int(params.get('topK', 5))
- if not query_emb:
- return {"matches": []}
- results = await asyncio.to_thread(client.search, collection_name=collection,
- query_vector=query_emb, limit=top_k)
- return {"matches": [{'id': str(r.id), 'document': r.payload.get('content', ''),
- 'metadata': r.payload, 'score': r.score} for r in results]}
-
- elif operation == 'delete':
- ids = params.get('ids', [])
- if ids:
- await asyncio.to_thread(client.delete, collection_name=collection, points_selector=ids)
- return {"deleted": True, "count": len(ids)}
-
- return {}
-
-
-async def _pinecone_op(operation: str, params: Dict, collection: str) -> Dict:
- """Pinecone operations."""
- from pinecone import Pinecone
- import uuid
-
- api_key = params.get('pineconeApiKey', '')
- if not api_key:
- raise ValueError("Pinecone API key required")
-
- pc = Pinecone(api_key=api_key)
- index = pc.Index(collection)
-
- if operation == 'store':
- embeddings = params.get('embeddings', [])
- chunks = params.get('chunks', [])
- if not embeddings:
- return {"stored_count": 0}
- vectors = []
- for i, emb in enumerate(embeddings):
- meta = {}
- if i < len(chunks):
- c = chunks[i]
- if isinstance(c, dict):
- meta = {'content': c.get('content', ''), 'source': c.get('source', 'unknown'),
- 'chunk_index': c.get('chunk_index', i)}
- else:
- meta = {'content': str(c), 'source': 'input', 'chunk_index': i}
- vectors.append({'id': str(uuid.uuid4()), 'values': emb, 'metadata': meta})
- await asyncio.to_thread(index.upsert, vectors=vectors)
- return {"stored_count": len(embeddings)}
-
- elif operation == 'query':
- query_emb = params.get('queryEmbedding', [])
- top_k = int(params.get('topK', 5))
- if not query_emb:
- return {"matches": []}
- results = await asyncio.to_thread(index.query, vector=query_emb, top_k=top_k, include_metadata=True)
- return {"matches": [{'id': m.id, 'document': m.metadata.get('content', '') if m.metadata else '',
- 'metadata': m.metadata or {}, 'score': m.score} for m in results.matches]}
-
- elif operation == 'delete':
- ids = params.get('ids', [])
- if ids:
- await asyncio.to_thread(index.delete, ids=ids)
- return {"deleted": True, "count": len(ids)}
-
- return {}