Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Refactored worker to avoid some duplication of code (#367)

Ritwik Gdee3663785f2
diff --git a/worker/src/unstract/worker/constants.py b/worker/src/unstract/worker/constants.py
index eeb3bc472..9a6e02810 100644
--- a/worker/src/unstract/worker/constants.py
+++ b/worker/src/unstract/worker/constants.py
@@ -1,10 +1,3 @@
-class ToolCommandKey:
- PROPERTIES = "properties"
- SPEC = "spec"
- VARIABLES = "variables"
- ICON = "icon"
-
-
class LogType:
LOG = "LOG"
UPDATE = "UPDATE"
diff --git a/worker/src/unstract/worker/main.py b/worker/src/unstract/worker/main.py
index fa29277a7..36858b714 100644
--- a/worker/src/unstract/worker/main.py
+++ b/worker/src/unstract/worker/main.py
@@ -1,8 +1,7 @@
from typing import Any, Optional
-from flask import Blueprint, Flask, Response, jsonify, request
+from flask import Blueprint, Flask, Response, abort, jsonify, request
from unstract.worker import UnstractWorker
-from unstract.worker.constants import ToolCommandKey
from unstract.worker.utils import Utils
app = Flask(__name__)
@@ -45,41 +44,24 @@ def run_container() -> Optional[Any]:
return result
[email protected]("container/spec", methods=["GET"])
-def get_spec() -> Optional[Any]:
- return get_resource(ToolCommandKey.SPEC)
[email protected]("container/<command>", methods=["GET"])
+def run_command(command: str) -> Optional[Any]:
+ """Endpoint which will can execute any of the below commands.
+ Args:
+ command (str): AnyOf("properties","spec","variables","icon")
[email protected]("container/properties", methods=["GET"])
-def get_properties() -> Optional[Any]:
- return get_resource(ToolCommandKey.PROPERTIES)
-
-
[email protected]("container/icon", methods=["GET"])
-def get_icon() -> Optional[Any]:
- return get_resource(ToolCommandKey.ICON)
-
-
[email protected]("container/variables", methods=["GET"])
-def get_variables() -> Optional[Any]:
- return get_resource(ToolCommandKey.VARIABLES)
-
-
-def get_resource(endpoint: str) -> Optional[Any]:
+ Returns:
+ Optional[Any]: Response from container for the specific command.
+ Returns None in case of error.
+ """
+ if command not in {"properties", "spec", "variables", "icon"}:
+ abort(404)
image_name = request.args.get("image_name")
image_tag = request.args.get("image_tag")
worker = UnstractWorker(image_name, image_tag)
- if endpoint == ToolCommandKey.PROPERTIES:
- return worker.get_properties()
- elif endpoint == ToolCommandKey.ICON:
- return worker.get_icon()
- elif endpoint == ToolCommandKey.VARIABLES:
- return worker.get_variables()
- elif endpoint == ToolCommandKey.SPEC:
- return worker.get_spec()
- else:
- return None
+ return worker.run_command(command)
# Register the Blueprint with the Flask app
diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.py
index b79f95913..fed45146d 100644
--- a/worker/src/unstract/worker/worker.py
+++ b/worker/src/unstract/worker/worker.py
@@ -217,90 +217,21 @@ class UnstractWorker:
return log_type
return None
- def get_spec(self) -> Optional[Any]:
- self.container_name = self.normalize_container_name(self.image_name)
- container_config = {
- "name": self.container_name,
- "image": self.image,
- "command": ["--command", "SPEC"],
- "detach": True,
- "stream": True,
- "auto_remove": True,
- "stderr": True,
- "stdout": True,
- }
- self.logger.info(f"Docker config: {container_config}")
-
- # Run the Docker container
- try:
- container = self.client.containers.run(**container_config)
- for line in container.logs(stream=True):
- text = line.decode().strip()
- self.logger.info(text)
- if '"type": "SPEC"' in text:
- return json.loads(text)
- except Exception as e:
- self.logger.error(f"Failed to run docker container: {e}")
- return None
-
- def get_properties(self) -> Optional[Any]:
- self.container_name = self.normalize_container_name(self.image_name)
- container_config = {
- "name": self.container_name,
- "image": self.image,
- "command": ["--command", "PROPERTIES"],
- "detach": True,
- "stream": True,
- "auto_remove": True,
- "stderr": True,
- "stdout": True,
- }
- self.logger.info(f"Docker config: {container_config}")
-
- # Run the Docker container
- try:
- container = self.client.containers.run(**container_config)
- for line in container.logs(stream=True):
- text = line.decode().strip()
- self.logger.info(text)
- if '"type": "PROPERTIES"' in text:
- return json.loads(text)
- except Exception as e:
- self.logger.error(f"Failed to run docker container: {e}")
- return None
+ def run_command(self, command: str) -> Optional[Any]:
+ """Runs any given command on the container.
- def get_icon(self) -> Optional[Any]:
- self.container_name = self.normalize_container_name(self.image_name)
- container_config = {
- "name": self.container_name,
- "image": self.image,
- "command": ["--command", "ICON"],
- "detach": True,
- "stream": True,
- "auto_remove": True,
- "stderr": True,
- "stdout": True,
- }
- self.logger.info(f"Docker config: {container_config}")
-
- # Run the Docker container
- try:
- container = self.client.containers.run(**container_config)
- for line in container.logs(stream=True):
- text = line.decode().strip()
- self.logger.info(text)
- if '"type": "ICON"' in text:
- return json.loads(text)
- except Exception as e:
- self.logger.error(f"Failed to run docker container: {e}")
- return None
+ Args:
+ command (str): Command to be executed.
- def get_variables(self) -> Optional[Any]:
+ Returns:
+ Optional[Any]: Response from container or None if error occures.
+ """
+ command = command.upper()
self.container_name = self.normalize_container_name(self.image_name)
container_config = {
"name": self.container_name,
"image": self.image,
- "command": ["--command", "VARIABLES"],
+ "command": ["--command", command],
"detach": True,
"stream": True,
"auto_remove": True,
@@ -315,7 +246,7 @@ class UnstractWorker:
for line in container.logs(stream=True):
text = line.decode().strip()
self.logger.info(text)
- if '"type": "VARIABLES"' in text:
+ if f'"type": "${command}"' in text:
return json.loads(text)
except Exception as e:
self.logger.error(f"Failed to run docker container: {e}")