LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix/Removing document indexer (#182)
Athul8fedf07c7977
diff --git a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json b/unstract/tool-registry/src/unstract/tool_registry/public_tools.jsonindex b2df6bb6d..83e199fea 100644--- a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json+++ b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json@@ -143,125 +143,6 @@"image_name": "unstract/tool-classifier","image_tag": "0.0.4"},- "document_indexer": {- "tool_uid": "document_indexer",- "properties": {- "schemaVersion": "0.0.1",- "displayName": "Document Indexer",- "functionName": "document_indexer",- "toolVersion": "0.0.2",- "description": "Used to index documents in the specified vector DB / embedding.",- "input": {- "description": "Document that needs to be indexed"- },- "output": {- "description": "No files written"- },- "result": {- "type": "TXT",- "description": "Result of the indexing process"- },- "adapter": {- "languageModels": [- {- "isEnabled": false- }- ],- "embeddingServices": [- {- "adapterId": "embeddingAdapterId",- "isEnabled": true,- "title": "Embedding to use",- "isRequired": true,- "description": "Embedding service to use for indexing"- }- ],- "vectorStores": [- {- "adapterId": "vectorDbAdapterId",- "isEnabled": true,- "title": "Vector DB to use",- "isRequired": true,- "description": "Vector DB to use for indexing"- }- ],- "textExtractors": [- {- "adapterId": "x2TextAdapterId",- "isEnabled": true,- "title": "Text extractor to use",- "isRequired": true,- "description": "Text extractor to use for indexing"- }- ]- },- "ioCompatibility": {- "api": {- "sourceSupport": true,- "destinationSupport": false,- "additionalArgs": {- "sync": true- }- },- "file": {- "sourceSupport": true,- "destinationSupport": false,- "additionalArgs": {}- },- "db": {- "destinationSupport": false,- "additionalArgs": {}- }- },- "restrictions": {- "maxFileSize": "50 MB",- "allowedFileTypes": [- "txt",- "pdf",- "doc",- "docx",- "odt"- ]- }- },- "spec": {- "title": "Document Indexer",- "description": "Index documents based on their semantic content",- "type": "object",- "required": [],- "properties": {- "chunkSize": {- "type": "number",- "title": "Chunk size",- "default": 1024,- "description": "Size of chunks to be considered for the embedding. Smaller chunk size means embeddings are more precise, while larger chunk size means that the embeddings may be more general, but can miss fine-grained details."- },- "chunkOverlap": {- "type": "number",- "title": "Chunk overlap",- "default": 128,- "description": "Overlap to be considered for the embedding."- },- "reIndex": {- "type": "boolean",- "title": "Re-Index",- "default": false,- "description": "Re-index files every time (useful during development)"- }- }- },- "variables": {- "title": "Runtime Variables",- "description": "Runtime Variables for Document Indexer",- "type": "object",- "required": [],- "properties": {}- },- "icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 24 24\"\n height=\"48\"\n viewBox=\"0 0 24 24\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg552\"\n sodipodi:docname=\"summarize_black_48dp.svg\"\n inkscape:export-filename=\"summarize_black_48dp.png\"\n inkscape:export-xdpi=\"96\"\n inkscape:export-ydpi=\"96\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs556\" />\n <sodipodi:namedview\n id=\"namedview554\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g544\">\n <path\n d=\"M 0,0 H 24 V 24 H 0 Z\"\n fill=\"none\"\n id=\"path542\" />\n </g>\n <g\n id=\"g550\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g548\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 15,3 H 5 C 3.9,3 3.01,3.9 3.01,5 L 3,19 c 0,1.1 0.89,2 1.99,2 H 19 c 1.1,0 2,-0.9 2,-2 V 9 Z M 5,19 V 5 h 9 v 5 h 5 v 9 z M 9,8 C 9,8.55 8.55,9 8,9 7.45,9 7,8.55 7,8 7,7.45 7.45,7 8,7 8.55,7 9,7.45 9,8 Z m 0,4 c 0,0.55 -0.45,1 -1,1 -0.55,0 -1,-0.45 -1,-1 0,-0.55 0.45,-1 1,-1 0.55,0 1,0.45 1,1 z m 0,4 c 0,0.55 -0.45,1 -1,1 -0.55,0 -1,-0.45 -1,-1 0,-0.55 0.45,-1 1,-1 0.55,0 1,0.45 1,1 z\"\n id=\"path546\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",- "image_url": "docker:unstract/tool-indexer:0.0.2",- "image_name": "unstract/tool-indexer",- "image_tag": "0.0.2"- },"text_extractor": {"tool_uid": "text_extractor","properties": {