Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
refactor(plugin): Wave 11.D.7 — document pipeline inlined
trohitgc21739325d40
diff --git a/server/nodes/document/_helpers.py b/server/nodes/document/_helpers.pydeleted file mode 100644index 97783246..00000000--- a/server/nodes/document/_helpers.py+++ /dev/null@@ -1,22 +0,0 @@-"""Helpers for thin-delegating document plugins to their handlers."""--from __future__ import annotations--from typing import Any, Awaitable, Callable, Dict---async def delegate(- handler: Callable[..., Awaitable[Dict[str, Any]]],- *,- node_type: str,- node_id: str,- payload: Dict[str, Any],- context: Dict[str, Any],-) -> Any:- response = await handler(- node_id=node_id, node_type=node_type,- parameters=payload, context=context,- )- if response.get("success"):- return response.get("result") or response- raise RuntimeError(response.get("error") or f"{node_type} failed")diff --git a/server/nodes/document/document_parser.py b/server/nodes/document/document_parser.pyindex bce35f02..50edc7ce 100644--- a/server/nodes/document/document_parser.py+++ b/server/nodes/document/document_parser.py@@ -1,26 +1,53 @@-"""Document Parser — Wave 11.C migration."""+"""Document Parser — Wave 11.D.7 inlined."""from __future__ import annotations-from typing import Any, Literal, Optional+import asyncio+from pathlib import Path+from typing import Literal, Optionalfrom pydantic import BaseModel, ConfigDict, Field+from core.logging import get_loggerfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue-from ._helpers import delegate+logger = get_logger(__name__)+++def _parse_file_sync(path: Path, parser: str) -> str:+ """Synchronous file parsing — runs in thread pool."""+ if parser == 'pypdf':+ from pypdf import PdfReader+ return "\n\n".join(p.extract_text() or '' for p in PdfReader(str(path)).pages)+ if parser == 'marker':+ from marker.converters.pdf import PdfConverter+ from marker.models import create_model_dict+ converter = PdfConverter(artifact_dict=create_model_dict())+ result = converter(str(path))+ return result.markdown if hasattr(result, 'markdown') else str(result)+ if parser == 'unstructured':+ from unstructured.partition.auto import partition+ return "\n\n".join(str(el) for el in partition(str(path)))+ if parser == 'beautifulsoup':+ from bs4 import BeautifulSoup+ soup = BeautifulSoup(path.read_text(errors='ignore'), 'html.parser')+ for s in soup(["script", "style"]):+ s.decompose()+ return soup.get_text(separator='\n')+ raise ValueError(f"Unknown parser: {parser}")class DocumentParserParams(BaseModel):- file_path: str = Field(..., alias="filePath")+ file_path: str = Field(default="", alias="filePath")parser: Literal["pypdf", "marker", "unstructured", "beautifulsoup"] = "pypdf"model_config = ConfigDict(populate_by_name=True, extra="allow")class DocumentParserOutput(BaseModel):- text: Optional[str] = None- pages: Optional[int] = None+ documents: Optional[list] = None+ parsed_count: Optional[int] = None+ failed: Optional[list] = Nonemodel_config = ConfigDict(extra="allow")@@ -45,9 +72,38 @@ class DocumentParserNode(ActionNode):Output = DocumentParserOutput@Operation("parse")- async def parse(self, ctx: NodeContext, params: DocumentParserParams) -> Any:- from services.handlers.document import handle_document_parser- return await delegate(- handle_document_parser, node_type=self.type, node_id=ctx.node_id,- payload=params.model_dump(by_alias=True), context=ctx.raw,+ async def parse(self, ctx: NodeContext, params: DocumentParserParams) -> DocumentParserOutput:+ p = params.model_dump(by_alias=True)+ files = p.get('files', [])+ input_dir = p.get('inputDir', '')+ parser = p.get('parser', 'pypdf')+ file_pattern = p.get('filePattern', '*.pdf')+ explicit_path = p.get('filePath') or p.get('file_path')++ paths: list[Path] = []+ if explicit_path:+ paths.append(Path(explicit_path))+ for f in files:+ path = f.get('path', '') if isinstance(f, dict) else str(f)+ if path:+ paths.append(Path(path))+ if input_dir and Path(input_dir).exists():+ paths.extend(Path(input_dir).glob(file_pattern))++ if not paths:+ return DocumentParserOutput(documents=[], parsed_count=0, failed=[])++ documents, failed = [], []+ for path in paths:+ try:+ content = await asyncio.to_thread(_parse_file_sync, path, parser)+ documents.append({+ 'source': str(path), 'filename': path.name,+ 'content': content, 'length': len(content), 'parser': parser,+ })+ except Exception as e:+ failed.append({'file': str(path), 'error': str(e)})++ return DocumentParserOutput(+ documents=documents, parsed_count=len(documents), failed=failed,)diff --git a/server/nodes/document/embedding_generator.py b/server/nodes/document/embedding_generator.pyindex 946a7ded..a99a9f7f 100644--- a/server/nodes/document/embedding_generator.py+++ b/server/nodes/document/embedding_generator.py@@ -1,27 +1,31 @@-"""Embedding Generator — Wave 11.C migration."""+"""Embedding Generator — Wave 11.D.7 inlined."""from __future__ import annotations-from typing import Any, List, Literal, Optional+import asyncio+from typing import List, Literal, Optionalfrom pydantic import BaseModel, ConfigDict, Fieldfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue-from ._helpers import delegate-class EmbeddingGeneratorParams(BaseModel):- chunks: List[str] = Field(default_factory=list)+ chunks: List[Optional[dict]] = Field(default_factory=list)provider: Literal["huggingface", "openai", "ollama"] = "huggingface"model: str = Field(default="BAAI/bge-small-en-v1.5")+ api_key: str = Field(default="", alias="apiKey")- model_config = ConfigDict(extra="allow")+ model_config = ConfigDict(populate_by_name=True, extra="allow")class EmbeddingGeneratorOutput(BaseModel):embeddings: Optional[list] = None- dimension: Optional[int] = None+ embedding_count: Optional[int] = None+ dimensions: Optional[int] = None+ chunks: Optional[list] = None+ provider: Optional[str] = None+ model: Optional[str] = Nonemodel_config = ConfigDict(extra="allow")@@ -46,9 +50,47 @@ class EmbeddingGeneratorNode(ActionNode):Output = EmbeddingGeneratorOutput@Operation("embed")- async def embed(self, ctx: NodeContext, params: EmbeddingGeneratorParams) -> Any:- from services.handlers.document import handle_embedding_generator- return await delegate(- handle_embedding_generator, node_type=self.type, node_id=ctx.node_id,- payload=params.model_dump(by_alias=True), context=ctx.raw,+ async def embed(self, ctx: NodeContext, params: EmbeddingGeneratorParams) -> EmbeddingGeneratorOutput:+ p = params.model_dump(by_alias=True)+ chunks = p.get('chunks', [])+ provider = p.get('provider', 'huggingface')+ model = p.get('model', 'BAAI/bge-small-en-v1.5')+ api_key = p.get('apiKey', '')++ if not chunks:+ return EmbeddingGeneratorOutput(+ embeddings=[], embedding_count=0, dimensions=0, chunks=[],+ provider=provider, model=model,+ )++ texts = [c.get('content', '') if isinstance(c, dict) else str(c) for c in chunks]++ if provider == 'huggingface':+ try:+ from langchain_huggingface import HuggingFaceEmbeddings+ except ImportError:+ raise RuntimeError(+ "HuggingFace embeddings unavailable. "+ "pip install langchain-huggingface sentence-transformers",+ )+ embedder = HuggingFaceEmbeddings(model_name=model)+ elif provider == 'openai':+ from langchain_openai import OpenAIEmbeddings+ embedder = OpenAIEmbeddings(model=model, api_key=api_key)+ elif provider == 'ollama':+ from langchain_ollama import OllamaEmbeddings+ embedder = OllamaEmbeddings(model=model)+ else:+ raise RuntimeError(f"Unknown provider: {provider}")++ embeddings = await asyncio.to_thread(embedder.embed_documents, texts)+ dimensions = len(embeddings[0]) if embeddings else 0++ return EmbeddingGeneratorOutput(+ embeddings=embeddings,+ embedding_count=len(embeddings),+ dimensions=dimensions,+ chunks=chunks,+ provider=provider,+ model=model,)diff --git a/server/nodes/document/file_downloader.py b/server/nodes/document/file_downloader.pyindex a429198c..988449b8 100644--- a/server/nodes/document/file_downloader.py+++ b/server/nodes/document/file_downloader.py@@ -1,14 +1,19 @@-"""File Downloader — Wave 11.C migration."""+"""File Downloader — Wave 11.D.7 inlined."""from __future__ import annotations-from typing import Any, List, Optional+import asyncio+from pathlib import Path+from typing import List, Optional+from urllib.parse import unquote, urlparse+import httpxfrom pydantic import BaseModel, ConfigDict, Field+from core.logging import get_loggerfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue-from ._helpers import delegate+logger = get_logger(__name__)class FileDownloaderParams(BaseModel):@@ -22,9 +27,11 @@ class FileDownloaderParams(BaseModel):class FileDownloaderOutput(BaseModel):- downloaded: Optional[list] = None- skipped: Optional[list] = None- failed: Optional[list] = None+ downloaded: Optional[int] = None+ skipped: Optional[int] = None+ failed: Optional[int] = None+ files: Optional[list] = None+ output_dir: Optional[str] = Nonemodel_config = ConfigDict(extra="allow")@@ -49,9 +56,58 @@ class FileDownloaderNode(ActionNode):Output = FileDownloaderOutput@Operation("download")- async def download(self, ctx: NodeContext, params: FileDownloaderParams) -> Any:- from services.handlers.document import handle_file_downloader- return await delegate(- handle_file_downloader, node_type=self.type, node_id=ctx.node_id,- payload=params.model_dump(by_alias=True), context=ctx.raw,+ async def download(self, ctx: NodeContext, params: FileDownloaderParams) -> FileDownloaderOutput:+ p = params.model_dump(by_alias=True)+ items = p.get('items') or [{'url': u} for u in p.get('urls', [])]+ workspace_dir = ctx.raw.get('workspace_dir', '')+ default_dir = str(Path(workspace_dir) / 'downloads') if workspace_dir else p.get('outputDir', 'downloads')+ output_dir = Path(p.get('outputDir') or default_dir)+ max_workers = int(p.get('maxWorkers', 8))+ skip_existing = p.get('skipExisting', True)+ timeout = float(p.get('timeout', 60))++ if not items:+ return FileDownloaderOutput(+ downloaded=0, skipped=0, failed=0, files=[], output_dir=str(output_dir),+ )++ output_dir.mkdir(parents=True, exist_ok=True)+ semaphore = asyncio.Semaphore(max_workers)++ async def dl(item):+ async with semaphore:+ url = item.get('url', '') if isinstance(item, dict) else str(item)+ if not url:+ return {'status': 'failed', 'error': 'Empty URL'}+ filename = unquote(Path(urlparse(url).path).name or 'download')+ file_path = output_dir / filename+ if skip_existing and file_path.exists():+ return {'status': 'skipped', 'path': str(file_path), 'url': url}+ try:+ async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:+ resp = await client.get(url)+ resp.raise_for_status()+ file_path.write_bytes(resp.content)+ return {+ 'status': 'downloaded', 'path': str(file_path),+ 'url': url, 'size': len(resp.content), 'filename': filename,+ }+ except Exception as e:+ return {'status': 'failed', 'url': url, 'error': str(e)}++ results = await asyncio.gather(*[dl(i) for i in items], return_exceptions=True)+ downloaded, skipped, failed = [], [], []+ for r in results:+ if isinstance(r, Exception):+ failed.append({'error': str(r)})+ elif r.get('status') == 'downloaded':+ downloaded.append(r)+ elif r.get('status') == 'skipped':+ skipped.append(r)+ else:+ failed.append(r)++ return FileDownloaderOutput(+ downloaded=len(downloaded), skipped=len(skipped), failed=len(failed),+ files=downloaded, output_dir=str(output_dir),)diff --git a/server/nodes/document/http_scraper.py b/server/nodes/document/http_scraper.pyindex 263cf323..ddc0215b 100644--- a/server/nodes/document/http_scraper.py+++ b/server/nodes/document/http_scraper.py@@ -1,14 +1,23 @@-"""HTTP Scraper — Wave 11.C migration."""+"""HTTP Scraper — Wave 11.D.7 inlined.++Scrapes links from web pages with optional date/page pagination.+"""from __future__ import annotations+import json+from datetime import datetime, timedeltafrom typing import Any, Optional+from urllib.parse import urljoin+import httpx+from bs4 import BeautifulSoupfrom pydantic import BaseModel, ConfigDict, Field+from core.logging import get_loggerfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue-from ._helpers import delegate+logger = get_logger(__name__)class HttpScraperParams(BaseModel):@@ -22,7 +31,8 @@ class HttpScraperParams(BaseModel):class HttpScraperOutput(BaseModel):items: Optional[list] = None- count: Optional[int] = None+ item_count: Optional[int] = None+ errors: Optional[list] = Nonemodel_config = ConfigDict(extra="allow")@@ -47,9 +57,72 @@ class HttpScraperNode(ActionNode):Output = HttpScraperOutput@Operation("scrape")- async def scrape(self, ctx: NodeContext, params: HttpScraperParams) -> Any:- from services.handlers.document import handle_http_scraper- return await delegate(- handle_http_scraper, node_type=self.type, node_id=ctx.node_id,- payload=params.model_dump(by_alias=True), context=ctx.raw,- )+ async def scrape(self, ctx: NodeContext, params: HttpScraperParams) -> HttpScraperOutput:+ p = params.model_dump(by_alias=True)+ url = p.get('url', '')+ if not url:+ raise RuntimeError("URL is required")++ iteration_mode = p.get('iterationMode') or p.get('paginationMode', 'single')+ link_selector = p.get('linkSelector', 'a[href$=".pdf"]')+ headers_str = p.get('headers', '{}')+ headers = json.loads(headers_str) if isinstance(headers_str, str) and headers_str else {}++ urls_to_fetch = []+ if iteration_mode == 'date':+ start_date = p.get('startDate', '')+ end_date = p.get('endDate', '')+ placeholder = p.get('datePlaceholder', '{date}')+ if not start_date or not end_date:+ raise RuntimeError("Start/end dates required for date mode")+ start = datetime.strptime(start_date, "%Y-%m-%d")+ end = datetime.strptime(end_date, "%Y-%m-%d")+ current = start+ while current <= end:+ urls_to_fetch.append((+ url.replace(placeholder, current.strftime("%Y-%m-%d")),+ {'date': current.isoformat()},+ ))+ current += timedelta(days=1)+ elif iteration_mode == 'page':+ start_page = int(p.get('startPage', 1))+ end_page = int(p.get('endPage', 10))+ for page in range(start_page, end_page + 1):+ urls_to_fetch.append((url.replace('{page}', str(page)), {'page': page}))+ else:+ urls_to_fetch.append((url, {}))++ proxy_url = None+ if p.get('useProxy', False):+ try:+ from services.proxy.service import get_proxy_service+ proxy_svc = get_proxy_service()+ if proxy_svc and proxy_svc.is_enabled():+ proxy_url = await proxy_svc.get_proxy_url(url, p)+ except Exception as e:+ logger.warning("[httpScraper] Proxy lookup failed", error=str(e))++ items, errors = [], []+ client_kwargs: dict = {"timeout": 30, "follow_redirects": True}+ if proxy_url:+ client_kwargs["proxy"] = proxy_url++ async with httpx.AsyncClient(**client_kwargs) as client:+ for fetch_url, meta in urls_to_fetch:+ try:+ response = await client.get(fetch_url, headers=headers)+ response.raise_for_status()+ soup = BeautifulSoup(response.text, 'html.parser')+ for el in soup.select(link_selector):+ href = el.get('href', '')+ if href:+ items.append({+ 'url': urljoin(fetch_url, href),+ 'text': el.get_text(strip=True),+ 'source_url': fetch_url,+ **meta,+ })+ except Exception as e:+ errors.append(f"{fetch_url}: {e}")++ return HttpScraperOutput(items=items, item_count=len(items), errors=errors)diff --git a/server/nodes/document/text_chunker.py b/server/nodes/document/text_chunker.pyindex d683b60c..511b58af 100644--- a/server/nodes/document/text_chunker.py+++ b/server/nodes/document/text_chunker.py@@ -1,28 +1,26 @@-"""Text Chunker — Wave 11.C migration."""+"""Text Chunker — Wave 11.D.7 inlined."""from __future__ import annotations-from typing import Any, Literal, Optional+from typing import Literal, Optionalfrom pydantic import BaseModel, ConfigDict, Fieldfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue-from ._helpers import delegate-class TextChunkerParams(BaseModel):text: str = Field(default="")strategy: Literal["recursive", "markdown", "token"] = "recursive"chunk_size: int = Field(default=1000, alias="chunkSize", ge=100, le=8000)- overlap: int = Field(default=200, ge=0, le=1000)+ overlap: int = Field(default=200, alias="chunkOverlap", ge=0, le=1000)model_config = ConfigDict(populate_by_name=True, extra="allow")class TextChunkerOutput(BaseModel):chunks: Optional[list] = None- count: Optional[int] = None+ chunk_count: Optional[int] = Nonemodel_config = ConfigDict(extra="allow")@@ -47,9 +45,35 @@ class TextChunkerNode(ActionNode):Output = TextChunkerOutput@Operation("chunk")- async def chunk(self, ctx: NodeContext, params: TextChunkerParams) -> Any:- from services.handlers.document import handle_text_chunker- return await delegate(- handle_text_chunker, node_type=self.type, node_id=ctx.node_id,- payload=params.model_dump(by_alias=True), context=ctx.raw,+ async def chunk(self, ctx: NodeContext, params: TextChunkerParams) -> TextChunkerOutput:+ from langchain_text_splitters import (+ MarkdownTextSplitter, RecursiveCharacterTextSplitter,)++ p = params.model_dump(by_alias=True)+ documents = p.get('documents') or ([{'content': p.get('text', '')}] if p.get('text') else [])+ chunk_size = int(p.get('chunkSize', 1000))+ chunk_overlap = int(p.get('chunkOverlap') or p.get('overlap', 200))+ strategy = p.get('strategy', 'recursive')++ if not documents:+ return TextChunkerOutput(chunks=[], chunk_count=0)++ if strategy == 'markdown':+ splitter = MarkdownTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)+ else:+ splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)++ chunks = []+ for doc in documents:+ content = doc.get('content', '') if isinstance(doc, dict) else str(doc)+ source = doc.get('source', 'input') if isinstance(doc, dict) else 'input'+ if not content:+ continue+ for i, chunk_text in enumerate(splitter.split_text(content)):+ chunks.append({+ 'source': source, 'chunk_index': i,+ 'content': chunk_text, 'length': len(chunk_text),+ })++ return TextChunkerOutput(chunks=chunks, chunk_count=len(chunks))diff --git a/server/nodes/document/vector_store.py b/server/nodes/document/vector_store.pyindex b968ab2c..4156261b 100644--- a/server/nodes/document/vector_store.py+++ b/server/nodes/document/vector_store.py@@ -1,35 +1,223 @@-"""Vector Store — Wave 11.C migration."""+"""Vector Store — Wave 11.D.7 inlined.++Backends: ChromaDB (local), Qdrant (self-hosted), Pinecone (cloud).+Each backend helper is a pure function that reads the raw param dict;+the dispatcher routes by ``backend`` + ``operation``.+"""from __future__ import annotations-from typing import Any, Literal, Optional+import asyncio+import uuid+from typing import Any, Dict, Literal, Optionalfrom pydantic import BaseModel, ConfigDict, Fieldfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue-from ._helpers import delegate-class VectorStoreParams(BaseModel):operation: Literal["store", "query", "delete"] = "store"- backend: Literal["chromadb", "qdrant", "pinecone"] = "chromadb"- collection: str = Field(default="default")+ backend: Literal["chroma", "chromadb", "qdrant", "pinecone"] = "chroma"+ collection_name: str = Field(default="documents", alias="collectionName")embeddings: Optional[list] = None- query: str = Field(default="")+ chunks: Optional[list] = None+ query_embedding: Optional[list] = Field(default=None, alias="queryEmbedding")top_k: int = Field(default=5, alias="topK", ge=1, le=100)+ ids: Optional[list] = None+ persist_dir: str = Field(default="./data/vectors", alias="persistDir")+ qdrant_url: str = Field(default="http://localhost:6333", alias="qdrantUrl")+ pinecone_api_key: str = Field(default="", alias="pineconeApiKey")model_config = ConfigDict(populate_by_name=True, extra="allow")class VectorStoreOutput(BaseModel):operation: Optional[str] = None+ backend: Optional[str] = None+ collection_name: Optional[str] = Nonematches: Optional[list] = Nonestored_count: Optional[int] = None+ collection_count: Optional[int] = None+ deleted: Optional[bool] = None+ count: Optional[int] = Nonemodel_config = ConfigDict(extra="allow")+async def _chroma_op(operation: str, params: Dict[str, Any], collection: str) -> Dict[str, Any]:+ try:+ import chromadb+ except ImportError:+ raise RuntimeError("ChromaDB unavailable. pip install chromadb")++ persist_dir = params.get('persistDir', './data/vectors')+ client = chromadb.PersistentClient(path=persist_dir)+ coll = client.get_or_create_collection(name=collection)++ if operation == 'store':+ embeddings = params.get('embeddings', [])+ chunks = params.get('chunks', [])+ if not embeddings:+ return {"stored_count": 0, "collection_count": coll.count()}+ ids = [str(uuid.uuid4()) for _ in embeddings]+ docs = [c.get('content', '') if isinstance(c, dict) else str(c) for c in chunks]+ while len(docs) < len(embeddings):+ docs.append('')+ metas = [+ {'source': c.get('source', 'unknown'), 'chunk_index': c.get('chunk_index', i)}+ if isinstance(c, dict) else {'source': 'input', 'chunk_index': i}+ for i, c in enumerate(chunks)+ ]+ while len(metas) < len(embeddings):+ metas.append({'source': 'unknown', 'chunk_index': len(metas)})+ await asyncio.to_thread(coll.add, ids=ids, embeddings=embeddings, documents=docs, metadatas=metas)+ return {"stored_count": len(embeddings), "collection_count": coll.count()}++ if operation == 'query':+ query_emb = params.get('queryEmbedding', [])+ top_k = int(params.get('topK', 5))+ if not query_emb:+ return {"matches": []}+ results = coll.query(query_embeddings=[query_emb], n_results=top_k)+ matches = []+ if results['ids'] and results['ids'][0]:+ for i in range(len(results['ids'][0])):+ matches.append({+ 'id': results['ids'][0][i],+ 'document': results['documents'][0][i] if results['documents'] else '',+ 'metadata': results['metadatas'][0][i] if results['metadatas'] else {},+ 'distance': results['distances'][0][i] if results.get('distances') else None,+ })+ return {"matches": matches}++ if operation == 'delete':+ ids = params.get('ids', [])+ if ids:+ await asyncio.to_thread(coll.delete, ids=ids)+ return {"deleted": True, "count": len(ids)}++ return {}+++async def _qdrant_op(operation: str, params: Dict[str, Any], collection: str) -> Dict[str, Any]:+ try:+ from qdrant_client import QdrantClient+ from qdrant_client.models import Distance, PointStruct, VectorParams+ except ImportError:+ raise RuntimeError("Qdrant client unavailable. pip install qdrant-client")++ url = params.get('qdrantUrl', 'http://localhost:6333')+ client = QdrantClient(url=url)++ if operation == 'store':+ embeddings = params.get('embeddings', [])+ chunks = params.get('chunks', [])+ if not embeddings:+ return {"stored_count": 0}+ vec_size = len(embeddings[0])+ colls = client.get_collections().collections+ if collection not in [c.name for c in colls]:+ client.create_collection(+ collection,+ vectors_config=VectorParams(size=vec_size, distance=Distance.COSINE),+ )+ points = []+ for i, emb in enumerate(embeddings):+ payload: Dict[str, Any] = {}+ if i < len(chunks):+ c = chunks[i]+ if isinstance(c, dict):+ payload = {+ 'content': c.get('content', ''),+ 'source': c.get('source', 'unknown'),+ 'chunk_index': c.get('chunk_index', i),+ }+ else:+ payload = {'content': str(c), 'source': 'input', 'chunk_index': i}+ points.append(PointStruct(id=str(uuid.uuid4()), vector=emb, payload=payload))+ await asyncio.to_thread(client.upsert, collection_name=collection, points=points)+ return {"stored_count": len(embeddings)}++ if operation == 'query':+ query_emb = params.get('queryEmbedding', [])+ top_k = int(params.get('topK', 5))+ if not query_emb:+ return {"matches": []}+ results = await asyncio.to_thread(+ client.search, collection_name=collection, query_vector=query_emb, limit=top_k,+ )+ return {"matches": [+ {'id': str(r.id), 'document': r.payload.get('content', ''),+ 'metadata': r.payload, 'score': r.score}+ for r in results+ ]}++ if operation == 'delete':+ ids = params.get('ids', [])+ if ids:+ await asyncio.to_thread(client.delete, collection_name=collection, points_selector=ids)+ return {"deleted": True, "count": len(ids)}++ return {}+++async def _pinecone_op(operation: str, params: Dict[str, Any], collection: str) -> Dict[str, Any]:+ from pinecone import Pinecone++ api_key = params.get('pineconeApiKey', '')+ if not api_key:+ raise RuntimeError("Pinecone API key required")++ pc = Pinecone(api_key=api_key)+ index = pc.Index(collection)++ if operation == 'store':+ embeddings = params.get('embeddings', [])+ chunks = params.get('chunks', [])+ if not embeddings:+ return {"stored_count": 0}+ vectors = []+ for i, emb in enumerate(embeddings):+ meta: Dict[str, Any] = {}+ if i < len(chunks):+ c = chunks[i]+ if isinstance(c, dict):+ meta = {+ 'content': c.get('content', ''),+ 'source': c.get('source', 'unknown'),+ 'chunk_index': c.get('chunk_index', i),+ }+ else:+ meta = {'content': str(c), 'source': 'input', 'chunk_index': i}+ vectors.append({'id': str(uuid.uuid4()), 'values': emb, 'metadata': meta})+ await asyncio.to_thread(index.upsert, vectors=vectors)+ return {"stored_count": len(embeddings)}++ if operation == 'query':+ query_emb = params.get('queryEmbedding', [])+ top_k = int(params.get('topK', 5))+ if not query_emb:+ return {"matches": []}+ results = await asyncio.to_thread(+ index.query, vector=query_emb, top_k=top_k, include_metadata=True,+ )+ return {"matches": [+ {'id': m.id,+ 'document': m.metadata.get('content', '') if m.metadata else '',+ 'metadata': m.metadata or {}, 'score': m.score}+ for m in results.matches+ ]}++ if operation == 'delete':+ ids = params.get('ids', [])+ if ids:+ await asyncio.to_thread(index.delete, ids=ids)+ return {"deleted": True, "count": len(ids)}++ return {}++class VectorStoreNode(ActionNode):type = "vectorStore"display_name = "Vector Store"@@ -50,9 +238,23 @@ class VectorStoreNode(ActionNode):Output = VectorStoreOutput@Operation("dispatch")- async def dispatch(self, ctx: NodeContext, params: VectorStoreParams) -> Any:- from services.handlers.document import handle_vector_store- return await delegate(- handle_vector_store, node_type=self.type, node_id=ctx.node_id,- payload=params.model_dump(by_alias=True), context=ctx.raw,- )+ async def dispatch(self, ctx: NodeContext, params: VectorStoreParams) -> VectorStoreOutput:+ p = params.model_dump(by_alias=True)+ operation = p.get('operation', 'store')+ backend_raw = p.get('backend', 'chroma')+ backend = 'chroma' if backend_raw in ('chroma', 'chromadb') else backend_raw+ collection = p.get('collectionName', 'documents')++ if backend == 'chroma':+ result = await _chroma_op(operation, p, collection)+ elif backend == 'qdrant':+ result = await _qdrant_op(operation, p, collection)+ elif backend == 'pinecone':+ result = await _pinecone_op(operation, p, collection)+ else:+ raise RuntimeError(f"Unknown backend: {backend_raw}")++ result['backend'] = backend+ result['collection_name'] = collection+ result['operation'] = operation+ return VectorStoreOutput(**result)diff --git a/server/services/handlers/__init__.py b/server/services/handlers/__init__.pyindex b2e7de6d..cad2cdb3 100644--- a/server/services/handlers/__init__.py+++ b/server/services/handlers/__init__.py@@ -87,15 +87,7 @@ from .social import (handle_social_send,)-# Document processing handlers-from .document import (- handle_http_scraper,- handle_file_downloader,- handle_document_parser,- handle_text_chunker,- handle_embedding_generator,- handle_vector_store,-)+# Document handlers inlined into nodes/document/*.py (Wave 11.D.7).# Search handlersfrom .search import (@@ -179,13 +171,6 @@ __all__ = [# Social'handle_social_receive','handle_social_send',- # Document processing- 'handle_http_scraper',- 'handle_file_downloader',- 'handle_document_parser',- 'handle_text_chunker',- 'handle_embedding_generator',- 'handle_vector_store',# Search'handle_brave_search','handle_serper_search',diff --git a/server/services/handlers/document.py b/server/services/handlers/document.pydeleted file mode 100644index c9e8333f..00000000--- a/server/services/handlers/document.py+++ /dev/null@@ -1,625 +0,0 @@-"""Document processing handlers for MachinaOs workflow nodes.--Each handler is an independent async function that can be executed by-the Temporal worker system for distributed processing.--Handlers:-- handle_http_scraper: Scrape links from web pages with pagination-- handle_file_downloader: Download files in parallel-- handle_document_parser: Parse documents (PyPDF, Marker, Unstructured)-- handle_text_chunker: Split text into chunks (LangChain)-- handle_embedding_generator: Generate embeddings (HF, OpenAI, Ollama)-- handle_vector_store: Store/query vectors (ChromaDB, Qdrant, Pinecone)-"""--import asyncio-import json-import time-from datetime import datetime, timedelta-from pathlib import Path-from typing import Dict, Any-from urllib.parse import urljoin, urlparse, unquote--from core.logging import get_logger--logger = get_logger(__name__)---# =============================================================================-# HTTP Scraper-# =============================================================================--async def handle_http_scraper(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Scrape links from web pages with date/page pagination support."""- import httpx- from bs4 import BeautifulSoup- start_time = time.time()-- try:- url = parameters.get('url', '')- iteration_mode = parameters.get('iterationMode', 'single')- link_selector = parameters.get('linkSelector', 'a[href$=".pdf"]')- headers_str = parameters.get('headers', '{}')-- if not url:- raise ValueError("URL is required")-- headers = json.loads(headers_str) if headers_str else {}- items, errors = [], []- urls_to_fetch = []-- if iteration_mode == 'date':- start_date = parameters.get('startDate', '')- end_date = parameters.get('endDate', '')- placeholder = parameters.get('datePlaceholder', '{date}')- if not start_date or not end_date:- raise ValueError("Start/end dates required for date mode")- start = datetime.strptime(start_date, "%Y-%m-%d")- end = datetime.strptime(end_date, "%Y-%m-%d")- current = start- while current <= end:- urls_to_fetch.append((- url.replace(placeholder, current.strftime("%Y-%m-%d")),- {'date': current.isoformat()}- ))- current += timedelta(days=1)- elif iteration_mode == 'page':- start_page = int(parameters.get('startPage', 1))- end_page = int(parameters.get('endPage', 10))- for page in range(start_page, end_page + 1):- urls_to_fetch.append((url.replace('{page}', str(page)), {'page': page}))- else:- urls_to_fetch.append((url, {}))-- # Transparent proxy injection- proxy_url = None- if parameters.get('useProxy', False):- try:- from services.proxy.service import get_proxy_service- proxy_svc = get_proxy_service()- if proxy_svc and proxy_svc.is_enabled():- proxy_url = await proxy_svc.get_proxy_url(url, parameters)- except Exception as e:- logger.warning("[httpScraper] Proxy lookup failed, proceeding without proxy", error=str(e))-- logger.info("[httpScraper] Starting", node_id=node_id, urls=len(urls_to_fetch),- proxy=bool(proxy_url))-- client_kwargs: dict = {"timeout": 30, "follow_redirects": True}- if proxy_url:- client_kwargs["proxy"] = proxy_url-- async with httpx.AsyncClient(**client_kwargs) as client:- for fetch_url, meta in urls_to_fetch:- try:- response = await client.get(fetch_url, headers=headers)- response.raise_for_status()- soup = BeautifulSoup(response.text, 'html.parser')- for el in soup.select(link_selector):- href = el.get('href', '')- if href:- items.append({- 'url': urljoin(fetch_url, href),- 'text': el.get_text(strip=True),- 'source_url': fetch_url,- **meta- })- except Exception as e:- errors.append(f"{fetch_url}: {str(e)}")-- return {- "success": True,- "node_id": node_id,- "node_type": node_type,- "result": {"items": items, "item_count": len(items), "errors": errors},- "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }- except Exception as e:- logger.error("[httpScraper] Failed", node_id=node_id, error=str(e))- return {- "success": False, "node_id": node_id, "node_type": node_type,- "error": str(e), "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }---# =============================================================================-# File Downloader-# =============================================================================--async def handle_file_downloader(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Download files from URLs in parallel using semaphore for concurrency."""- import httpx- start_time = time.time()-- try:- items = parameters.get('items', [])- workspace_dir = context.get('workspace_dir', '')- default_dir = str(Path(workspace_dir) / 'downloads') if workspace_dir else 'downloads'- output_dir = Path(parameters.get('outputDir') or default_dir)- max_workers = int(parameters.get('maxWorkers', 8))- skip_existing = parameters.get('skipExisting', True)- timeout = float(parameters.get('timeout', 60))-- if not items:- return {- "success": True, "node_id": node_id, "node_type": node_type,- "result": {"downloaded": 0, "skipped": 0, "failed": 0, "files": []},- "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }-- output_dir.mkdir(parents=True, exist_ok=True)- downloaded, skipped, failed = [], [], []- semaphore = asyncio.Semaphore(max_workers)-- async def download_file(item):- async with semaphore:- url = item.get('url', '') if isinstance(item, dict) else str(item)- if not url:- return {'status': 'failed', 'error': 'Empty URL'}- filename = unquote(Path(urlparse(url).path).name or 'download')- file_path = output_dir / filename- if skip_existing and file_path.exists():- return {'status': 'skipped', 'path': str(file_path), 'url': url}- try:- async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:- response = await client.get(url)- response.raise_for_status()- file_path.write_bytes(response.content)- return {'status': 'downloaded', 'path': str(file_path), 'url': url,- 'size': len(response.content), 'filename': filename}- except Exception as e:- return {'status': 'failed', 'url': url, 'error': str(e)}-- logger.info("[fileDownloader] Starting", node_id=node_id, items=len(items))- results = await asyncio.gather(*[download_file(i) for i in items], return_exceptions=True)-- for r in results:- if isinstance(r, Exception):- failed.append({'error': str(r)})- elif r.get('status') == 'downloaded':- downloaded.append(r)- elif r.get('status') == 'skipped':- skipped.append(r)- else:- failed.append(r)-- return {- "success": True, "node_id": node_id, "node_type": node_type,- "result": {"downloaded": len(downloaded), "skipped": len(skipped),- "failed": len(failed), "files": downloaded, "output_dir": str(output_dir)},- "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }- except Exception as e:- logger.error("[fileDownloader] Failed", node_id=node_id, error=str(e))- return {- "success": False, "node_id": node_id, "node_type": node_type,- "error": str(e), "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }---# =============================================================================-# Document Parser-# =============================================================================--def _parse_file_sync(path: Path, parser: str) -> str:- """Synchronous file parsing - runs in thread pool."""- if parser == 'pypdf':- from pypdf import PdfReader- return "\n\n".join(p.extract_text() or '' for p in PdfReader(str(path)).pages)- elif parser == 'marker':- from marker.converters.pdf import PdfConverter- from marker.models import create_model_dict- converter = PdfConverter(artifact_dict=create_model_dict())- result = converter(str(path))- return result.markdown if hasattr(result, 'markdown') else str(result)- elif parser == 'unstructured':- from unstructured.partition.auto import partition- return "\n\n".join(str(el) for el in partition(str(path)))- elif parser == 'beautifulsoup':- from bs4 import BeautifulSoup- soup = BeautifulSoup(path.read_text(errors='ignore'), 'html.parser')- for s in soup(["script", "style"]):- s.decompose()- return soup.get_text(separator='\n')- raise ValueError(f"Unknown parser: {parser}")---async def handle_document_parser(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Parse documents to text using configurable parsers."""- start_time = time.time()-- try:- files = parameters.get('files', [])- input_dir = parameters.get('inputDir', '')- parser = parameters.get('parser', 'pypdf')- file_pattern = parameters.get('filePattern', '*.pdf')-- paths = []- for f in files:- p = f.get('path', '') if isinstance(f, dict) else str(f)- if p:- paths.append(Path(p))- if input_dir and Path(input_dir).exists():- paths.extend(Path(input_dir).glob(file_pattern))-- if not paths:- return {- "success": True, "node_id": node_id, "node_type": node_type,- "result": {"documents": [], "parsed_count": 0, "failed": []},- "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }-- logger.info("[documentParser] Starting", node_id=node_id, files=len(paths), parser=parser)- documents, failed = [], []-- for path in paths:- try:- content = await asyncio.to_thread(_parse_file_sync, path, parser)- documents.append({'source': str(path), 'filename': path.name,- 'content': content, 'length': len(content), 'parser': parser})- except Exception as e:- failed.append({'file': str(path), 'error': str(e)})-- return {- "success": True, "node_id": node_id, "node_type": node_type,- "result": {"documents": documents, "parsed_count": len(documents), "failed": failed},- "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }- except Exception as e:- logger.error("[documentParser] Failed", node_id=node_id, error=str(e))- return {- "success": False, "node_id": node_id, "node_type": node_type,- "error": str(e), "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }---# =============================================================================-# Text Chunker-# =============================================================================--async def handle_text_chunker(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Split text into overlapping chunks using LangChain splitters."""- start_time = time.time()-- try:- documents = parameters.get('documents', [])- chunk_size = int(parameters.get('chunkSize', 1024))- chunk_overlap = int(parameters.get('chunkOverlap', 200))- strategy = parameters.get('strategy', 'recursive')-- if not documents:- return {- "success": True, "node_id": node_id, "node_type": node_type,- "result": {"chunks": [], "chunk_count": 0},- "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }-- from langchain_text_splitters import RecursiveCharacterTextSplitter, MarkdownTextSplitter-- if strategy == 'markdown':- splitter = MarkdownTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)- else:- splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)-- logger.info("[textChunker] Starting", node_id=node_id, docs=len(documents))- chunks = []-- for doc in documents:- content = doc.get('content', '') if isinstance(doc, dict) else str(doc)- source = doc.get('source', 'input') if isinstance(doc, dict) else 'input'- if not content:- continue- for i, chunk_text in enumerate(splitter.split_text(content)):- chunks.append({'source': source, 'chunk_index': i,- 'content': chunk_text, 'length': len(chunk_text)})-- return {- "success": True, "node_id": node_id, "node_type": node_type,- "result": {"chunks": chunks, "chunk_count": len(chunks)},- "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }- except Exception as e:- logger.error("[textChunker] Failed", node_id=node_id, error=str(e))- return {- "success": False, "node_id": node_id, "node_type": node_type,- "error": str(e), "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }---# =============================================================================-# Embedding Generator-# =============================================================================--async def handle_embedding_generator(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Generate embeddings using HuggingFace, OpenAI, or Ollama."""- start_time = time.time()-- try:- chunks = parameters.get('chunks', [])- provider = parameters.get('provider', 'huggingface')- model = parameters.get('model', 'BAAI/bge-small-en-v1.5')- api_key = parameters.get('apiKey', '')-- if not chunks:- return {- "success": True, "node_id": node_id, "node_type": node_type,- "result": {"embeddings": [], "embedding_count": 0, "dimensions": 0, "chunks": []},- "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }-- texts = [c.get('content', '') if isinstance(c, dict) else str(c) for c in chunks]- logger.info("[embeddingGenerator] Starting", node_id=node_id, texts=len(texts), provider=provider)-- if provider == 'huggingface':- try:- from langchain_huggingface import HuggingFaceEmbeddings- except ImportError:- raise ImportError("HuggingFace embeddings not available. Install with: pip install langchain-huggingface sentence-transformers")- embedder = HuggingFaceEmbeddings(model_name=model)- elif provider == 'openai':- from langchain_openai import OpenAIEmbeddings- embedder = OpenAIEmbeddings(model=model, api_key=api_key)- elif provider == 'ollama':- from langchain_ollama import OllamaEmbeddings- embedder = OllamaEmbeddings(model=model)- else:- raise ValueError(f"Unknown provider: {provider}")-- embeddings = await asyncio.to_thread(embedder.embed_documents, texts)- dimensions = len(embeddings[0]) if embeddings else 0-- return {- "success": True, "node_id": node_id, "node_type": node_type,- "result": {"embeddings": embeddings, "embedding_count": len(embeddings),- "dimensions": dimensions, "chunks": chunks, "provider": provider, "model": model},- "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }- except Exception as e:- logger.error("[embeddingGenerator] Failed", node_id=node_id, error=str(e))- return {- "success": False, "node_id": node_id, "node_type": node_type,- "error": str(e), "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }---# =============================================================================-# Vector Store-# =============================================================================--async def handle_vector_store(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Store or query vectors using ChromaDB, Qdrant, or Pinecone."""- start_time = time.time()-- try:- operation = parameters.get('operation', 'store')- backend = parameters.get('backend', 'chroma')- collection_name = parameters.get('collectionName', 'documents')-- logger.info("[vectorStore] Starting", node_id=node_id, op=operation, backend=backend)-- if backend == 'chroma':- result = await _chroma_op(operation, parameters, collection_name)- elif backend == 'qdrant':- result = await _qdrant_op(operation, parameters, collection_name)- elif backend == 'pinecone':- result = await _pinecone_op(operation, parameters, collection_name)- else:- raise ValueError(f"Unknown backend: {backend}")-- result['backend'] = backend- result['collection_name'] = collection_name-- return {- "success": True, "node_id": node_id, "node_type": node_type,- "result": result, "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }- except Exception as e:- logger.error("[vectorStore] Failed", node_id=node_id, error=str(e))- return {- "success": False, "node_id": node_id, "node_type": node_type,- "error": str(e), "execution_time": time.time() - start_time,- "timestamp": datetime.now().isoformat()- }---async def _chroma_op(operation: str, params: Dict, collection: str) -> Dict:- """ChromaDB operations."""- try:- import chromadb- except ImportError:- raise ImportError("ChromaDB not available. Install with: pip install chromadb")- import uuid-- persist_dir = params.get('persistDir', './data/vectors')- client = chromadb.PersistentClient(path=persist_dir)- coll = client.get_or_create_collection(name=collection)-- if operation == 'store':- embeddings = params.get('embeddings', [])- chunks = params.get('chunks', [])- if not embeddings:- return {"stored_count": 0, "collection_count": coll.count()}- ids = [str(uuid.uuid4()) for _ in embeddings]- docs = [c.get('content', '') if isinstance(c, dict) else str(c) for c in chunks]- while len(docs) < len(embeddings):- docs.append('')- metas = [{'source': c.get('source', 'unknown'), 'chunk_index': c.get('chunk_index', i)}- if isinstance(c, dict) else {'source': 'input', 'chunk_index': i}- for i, c in enumerate(chunks)]- while len(metas) < len(embeddings):- metas.append({'source': 'unknown', 'chunk_index': len(metas)})- await asyncio.to_thread(coll.add, ids=ids, embeddings=embeddings, documents=docs, metadatas=metas)- return {"stored_count": len(embeddings), "collection_count": coll.count()}-- elif operation == 'query':- query_emb = params.get('queryEmbedding', [])- top_k = int(params.get('topK', 5))- if not query_emb:- return {"matches": []}- results = coll.query(query_embeddings=[query_emb], n_results=top_k)- matches = []- if results['ids'] and results['ids'][0]:- for i in range(len(results['ids'][0])):- matches.append({- 'id': results['ids'][0][i],- 'document': results['documents'][0][i] if results['documents'] else '',- 'metadata': results['metadatas'][0][i] if results['metadatas'] else {},- 'distance': results['distances'][0][i] if results.get('distances') else None- })- return {"matches": matches}-- elif operation == 'delete':- ids = params.get('ids', [])- if ids:- await asyncio.to_thread(coll.delete, ids=ids)- return {"deleted": True, "count": len(ids)}-- return {}---async def _qdrant_op(operation: str, params: Dict, collection: str) -> Dict:- """Qdrant operations."""- try:- from qdrant_client import QdrantClient- from qdrant_client.models import VectorParams, Distance, PointStruct- except ImportError:- raise ImportError("Qdrant client not available. Install with: pip install qdrant-client")- import uuid-- url = params.get('qdrantUrl', 'http://localhost:6333')- client = QdrantClient(url=url)-- if operation == 'store':- embeddings = params.get('embeddings', [])- chunks = params.get('chunks', [])- if not embeddings:- return {"stored_count": 0}- vec_size = len(embeddings[0])- colls = client.get_collections().collections- if collection not in [c.name for c in colls]:- client.create_collection(collection, vectors_config=VectorParams(size=vec_size, distance=Distance.COSINE))- points = []- for i, emb in enumerate(embeddings):- payload = {}- if i < len(chunks):- c = chunks[i]- if isinstance(c, dict):- payload = {'content': c.get('content', ''), 'source': c.get('source', 'unknown'),- 'chunk_index': c.get('chunk_index', i)}- else:- payload = {'content': str(c), 'source': 'input', 'chunk_index': i}- points.append(PointStruct(id=str(uuid.uuid4()), vector=emb, payload=payload))- await asyncio.to_thread(client.upsert, collection_name=collection, points=points)- return {"stored_count": len(embeddings)}-- elif operation == 'query':- query_emb = params.get('queryEmbedding', [])- top_k = int(params.get('topK', 5))- if not query_emb:- return {"matches": []}- results = await asyncio.to_thread(client.search, collection_name=collection,- query_vector=query_emb, limit=top_k)- return {"matches": [{'id': str(r.id), 'document': r.payload.get('content', ''),- 'metadata': r.payload, 'score': r.score} for r in results]}-- elif operation == 'delete':- ids = params.get('ids', [])- if ids:- await asyncio.to_thread(client.delete, collection_name=collection, points_selector=ids)- return {"deleted": True, "count": len(ids)}-- return {}---async def _pinecone_op(operation: str, params: Dict, collection: str) -> Dict:- """Pinecone operations."""- from pinecone import Pinecone- import uuid-- api_key = params.get('pineconeApiKey', '')- if not api_key:- raise ValueError("Pinecone API key required")-- pc = Pinecone(api_key=api_key)- index = pc.Index(collection)-- if operation == 'store':- embeddings = params.get('embeddings', [])- chunks = params.get('chunks', [])- if not embeddings:- return {"stored_count": 0}- vectors = []- for i, emb in enumerate(embeddings):- meta = {}- if i < len(chunks):- c = chunks[i]- if isinstance(c, dict):- meta = {'content': c.get('content', ''), 'source': c.get('source', 'unknown'),- 'chunk_index': c.get('chunk_index', i)}- else:- meta = {'content': str(c), 'source': 'input', 'chunk_index': i}- vectors.append({'id': str(uuid.uuid4()), 'values': emb, 'metadata': meta})- await asyncio.to_thread(index.upsert, vectors=vectors)- return {"stored_count": len(embeddings)}-- elif operation == 'query':- query_emb = params.get('queryEmbedding', [])- top_k = int(params.get('topK', 5))- if not query_emb:- return {"matches": []}- results = await asyncio.to_thread(index.query, vector=query_emb, top_k=top_k, include_metadata=True)- return {"matches": [{'id': m.id, 'document': m.metadata.get('content', '') if m.metadata else '',- 'metadata': m.metadata or {}, 'score': m.score} for m in results.matches]}-- elif operation == 'delete':- ids = params.get('ids', [])- if ids:- await asyncio.to_thread(index.delete, ids=ids)- return {"deleted": True, "count": len(ids)}-- return {}