Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Redis Sentinel support for tool containers and sidecars (#1858)

Chandrasekharan Mbf56976d74de
diff --git a/backend/sample.env b/backend/sample.env
index 41246c704..79ca75886 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -100,9 +100,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
# Structure Tool Image (Runs prompt studio exported tools)
# https://hub.docker.com/r/unstract/tool-structure
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.97"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.98"
STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.97"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.98"
# Feature Flags
EVALUATION_SERVER_IP=unstract-flipt
diff --git a/runner/src/unstract/runner/constants.py b/runner/src/unstract/runner/constants.py
index 34ceee25b..b46971d61 100644
--- a/runner/src/unstract/runner/constants.py
+++ b/runner/src/unstract/runner/constants.py
@@ -39,6 +39,8 @@ class Env:
REDIS_PORT = "REDIS_PORT"
REDIS_USER = "REDIS_USER"
REDIS_PASSWORD = "REDIS_PASSWORD"
+ REDIS_SENTINEL_MODE = "REDIS_SENTINEL_MODE"
+ REDIS_SENTINEL_MASTER_NAME = "REDIS_SENTINEL_MASTER_NAME"
CELERY_BROKER_BASE_URL = "CELERY_BROKER_BASE_URL"
CELERY_BROKER_USER = "CELERY_BROKER_USER"
CELERY_BROKER_PASS = "CELERY_BROKER_PASS"
diff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.py
index 03ed6f341..1125bf69e 100644
--- a/runner/src/unstract/runner/runner.py
+++ b/runner/src/unstract/runner/runner.py
@@ -232,6 +232,10 @@ class UnstractRunner:
"REDIS_PORT": os.getenv(Env.REDIS_PORT),
"REDIS_USER": os.getenv(Env.REDIS_USER),
"REDIS_PASSWORD": os.getenv(Env.REDIS_PASSWORD),
+ "REDIS_SENTINEL_MODE": os.getenv(Env.REDIS_SENTINEL_MODE, "False"),
+ "REDIS_SENTINEL_MASTER_NAME": os.getenv(
+ Env.REDIS_SENTINEL_MASTER_NAME, "mymaster"
+ ),
"TOOL_INSTANCE_ID": tool_instance_id,
"EXECUTION_ID": execution_id,
"ORGANIZATION_ID": organization_id,
diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.json
index 4685282de..48cb20eda 100644
--- a/tools/classifier/src/config/properties.json
+++ b/tools/classifier/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "File Classifier",
"functionName": "classify",
- "toolVersion": "0.0.76",
+ "toolVersion": "0.0.77",
"description": "Classifies a file into a bin based on its contents",
"input": {
"description": "File to be classified"
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json
index 8144c8b64..c0fcc5782 100644
--- a/tools/structure/src/config/properties.json
+++ b/tools/structure/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Structure Tool",
"functionName": "structure_tool",
- "toolVersion": "0.0.97",
+ "toolVersion": "0.0.98",
"description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio",
"input": {
"description": "File that needs to be indexed and parsed for answers"
diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.json
index 1c2c4dafc..8cbdbb911 100644
--- a/tools/text_extractor/src/config/properties.json
+++ b/tools/text_extractor/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Text Extractor",
"functionName": "text_extractor",
- "toolVersion": "0.0.72",
+ "toolVersion": "0.0.73",
"description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents",
"input": {
"description": "Document"
diff --git a/unstract/tool-registry/tool_registry_config/public_tools.json b/unstract/tool-registry/tool_registry_config/public_tools.json
index fbb927513..0174d17f0 100644
--- a/unstract/tool-registry/tool_registry_config/public_tools.json
+++ b/unstract/tool-registry/tool_registry_config/public_tools.json
@@ -5,7 +5,7 @@
"schemaVersion": "0.0.1",
"displayName": "File Classifier",
"functionName": "classify",
- "toolVersion": "0.0.76",
+ "toolVersion": "0.0.77",
"description": "Classifies a file into a bin based on its contents",
"input": {
"description": "File to be classified"
@@ -106,9 +106,9 @@
"properties": {}
},
"icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 20 20\"\n height=\"48\"\n viewBox=\"0 0 20 20\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg8109\"\n sodipodi:docname=\"folder_copy_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs8113\" />\n <sodipodi:namedview\n id=\"namedview8111\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g8099\">\n <rect\n fill=\"none\"\n height=\"20\"\n width=\"20\"\n x=\"0\"\n id=\"rect8097\"\n y=\"0\" />\n </g>\n <g\n id=\"g8107\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g8105\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 2.5,5 H 1 V 15.5 C 1,16.33 1.67,17 2.5,17 H 15.68 V 15.5 H 2.5 Z\"\n id=\"path8101\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 16.5,4 H 11 L 9,2 H 5.5 C 4.67,2 4,2.67 4,3.5 v 9 C 4,13.33 4.67,14 5.5,14 h 11 c 0.83,0 1.5,-0.67 1.5,-1.5 v -7 C 18,4.67 17.33,4 16.5,4 Z m 0,8.5 h -11 v -9 h 2.88 l 2,2 h 6.12 z\"\n id=\"path8103\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",
- "image_url": "docker:unstract/tool-classifier:0.0.76",
+ "image_url": "docker:unstract/tool-classifier:0.0.77",
"image_name": "unstract/tool-classifier",
- "image_tag": "0.0.76"
+ "image_tag": "0.0.77"
},
"text_extractor": {
"tool_uid": "text_extractor",
@@ -116,7 +116,7 @@
"schemaVersion": "0.0.1",
"displayName": "Text Extractor",
"functionName": "text_extractor",
- "toolVersion": "0.0.72",
+ "toolVersion": "0.0.73",
"description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents",
"input": {
"description": "Document"
@@ -191,8 +191,8 @@
}
},
"icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 20 20\"\n height=\"48\"\n viewBox=\"0 0 20 20\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg8109\"\n sodipodi:docname=\"folder_copy_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs8113\" />\n <sodipodi:namedview\n id=\"namedview8111\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g8099\">\n <rect\n fill=\"none\"\n height=\"20\"\n width=\"20\"\n x=\"0\"\n id=\"rect8097\"\n y=\"0\" />\n </g>\n <g\n id=\"g8107\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g8105\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 2.5,5 H 1 V 15.5 C 1,16.33 1.67,17 2.5,17 H 15.68 V 15.5 H 2.5 Z\"\n id=\"path8101\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 16.5,4 H 11 L 9,2 H 5.5 C 4.67,2 4,2.67 4,3.5 v 9 C 4,13.33 4.67,14 5.5,14 h 11 c 0.83,0 1.5,-0.67 1.5,-1.5 v -7 C 18,4.67 17.33,4 16.5,4 Z m 0,8.5 h -11 v -9 h 2.88 l 2,2 h 6.12 z\"\n id=\"path8103\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",
- "image_url": "docker:unstract/tool-text-extractor:0.0.72",
+ "image_url": "docker:unstract/tool-text-extractor:0.0.73",
"image_name": "unstract/tool-text-extractor",
- "image_tag": "0.0.72"
+ "image_tag": "0.0.73"
}
}
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
index 53f1c8b1b..e7b9d4616 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
@@ -28,6 +28,8 @@ class ToolRuntimeVariable:
REDIS_PORT = "REDIS_PORT"
REDIS_USER = "REDIS_USER"
REDIS_PASSWORD = "REDIS_PASSWORD"
+ REDIS_SENTINEL_MODE = "REDIS_SENTINEL_MODE"
+ REDIS_SENTINEL_MASTER_NAME = "REDIS_SENTINEL_MASTER_NAME"
class WorkflowFileType:
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
index 3530994ea..6778eafe1 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
@@ -63,6 +63,12 @@ class ToolsUtils:
self.redis_password = ToolsUtils.get_env(
ToolRV.REDIS_PASSWORD, raise_exception=True
)
+ self.redis_sentinel_mode = ToolsUtils.get_env(
+ ToolRV.REDIS_SENTINEL_MODE, raise_exception=False
+ )
+ self.redis_sentinel_master_name = ToolsUtils.get_env(
+ ToolRV.REDIS_SENTINEL_MASTER_NAME, raise_exception=False
+ )
def set_messaging_channel(self, messaging_channel: str) -> None:
self.messaging_channel = messaging_channel
@@ -235,6 +241,9 @@ class ToolsUtils:
ToolRV.REDIS_PORT: self.redis_port,
ToolRV.REDIS_USER: self.redis_user,
ToolRV.REDIS_PASSWORD: self.redis_password,
+ ToolRV.REDIS_SENTINEL_MODE: self.redis_sentinel_mode or "False",
+ ToolRV.REDIS_SENTINEL_MASTER_NAME: self.redis_sentinel_master_name
+ or "mymaster",
}
# For async LLM Whisperer extraction
if self.llmw_poll_interval: