LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Refactored worker to avoid some duplication of code (#367)
Ritwik Gdee3663785f2
diff --git a/worker/src/unstract/worker/constants.py b/worker/src/unstract/worker/constants.pyindex eeb3bc472..9a6e02810 100644--- a/worker/src/unstract/worker/constants.py+++ b/worker/src/unstract/worker/constants.py@@ -1,10 +1,3 @@-class ToolCommandKey:- PROPERTIES = "properties"- SPEC = "spec"- VARIABLES = "variables"- ICON = "icon"--class LogType:LOG = "LOG"UPDATE = "UPDATE"diff --git a/worker/src/unstract/worker/main.py b/worker/src/unstract/worker/main.pyindex fa29277a7..36858b714 100644--- a/worker/src/unstract/worker/main.py+++ b/worker/src/unstract/worker/main.py@@ -1,8 +1,7 @@from typing import Any, Optional-from flask import Blueprint, Flask, Response, jsonify, request+from flask import Blueprint, Flask, Response, abort, jsonify, requestfrom unstract.worker import UnstractWorker-from unstract.worker.constants import ToolCommandKeyfrom unstract.worker.utils import Utilsapp = Flask(__name__)@@ -45,41 +44,24 @@ def run_container() -> Optional[Any]:return result[email protected]("container/spec", methods=["GET"])-def get_spec() -> Optional[Any]:- return get_resource(ToolCommandKey.SPEC)[email protected]("container/<command>", methods=["GET"])+def run_command(command: str) -> Optional[Any]:+ """Endpoint which will can execute any of the below commands.+ Args:+ command (str): AnyOf("properties","spec","variables","icon")[email protected]("container/properties", methods=["GET"])-def get_properties() -> Optional[Any]:- return get_resource(ToolCommandKey.PROPERTIES)--[email protected]("container/icon", methods=["GET"])-def get_icon() -> Optional[Any]:- return get_resource(ToolCommandKey.ICON)--[email protected]("container/variables", methods=["GET"])-def get_variables() -> Optional[Any]:- return get_resource(ToolCommandKey.VARIABLES)---def get_resource(endpoint: str) -> Optional[Any]:+ Returns:+ Optional[Any]: Response from container for the specific command.+ Returns None in case of error.+ """+ if command not in {"properties", "spec", "variables", "icon"}:+ abort(404)image_name = request.args.get("image_name")image_tag = request.args.get("image_tag")worker = UnstractWorker(image_name, image_tag)- if endpoint == ToolCommandKey.PROPERTIES:- return worker.get_properties()- elif endpoint == ToolCommandKey.ICON:- return worker.get_icon()- elif endpoint == ToolCommandKey.VARIABLES:- return worker.get_variables()- elif endpoint == ToolCommandKey.SPEC:- return worker.get_spec()- else:- return None+ return worker.run_command(command)# Register the Blueprint with the Flask appdiff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.pyindex b79f95913..fed45146d 100644--- a/worker/src/unstract/worker/worker.py+++ b/worker/src/unstract/worker/worker.py@@ -217,90 +217,21 @@ class UnstractWorker:return log_typereturn None- def get_spec(self) -> Optional[Any]:- self.container_name = self.normalize_container_name(self.image_name)- container_config = {- "name": self.container_name,- "image": self.image,- "command": ["--command", "SPEC"],- "detach": True,- "stream": True,- "auto_remove": True,- "stderr": True,- "stdout": True,- }- self.logger.info(f"Docker config: {container_config}")-- # Run the Docker container- try:- container = self.client.containers.run(**container_config)- for line in container.logs(stream=True):- text = line.decode().strip()- self.logger.info(text)- if '"type": "SPEC"' in text:- return json.loads(text)- except Exception as e:- self.logger.error(f"Failed to run docker container: {e}")- return None-- def get_properties(self) -> Optional[Any]:- self.container_name = self.normalize_container_name(self.image_name)- container_config = {- "name": self.container_name,- "image": self.image,- "command": ["--command", "PROPERTIES"],- "detach": True,- "stream": True,- "auto_remove": True,- "stderr": True,- "stdout": True,- }- self.logger.info(f"Docker config: {container_config}")-- # Run the Docker container- try:- container = self.client.containers.run(**container_config)- for line in container.logs(stream=True):- text = line.decode().strip()- self.logger.info(text)- if '"type": "PROPERTIES"' in text:- return json.loads(text)- except Exception as e:- self.logger.error(f"Failed to run docker container: {e}")- return None+ def run_command(self, command: str) -> Optional[Any]:+ """Runs any given command on the container.- def get_icon(self) -> Optional[Any]:- self.container_name = self.normalize_container_name(self.image_name)- container_config = {- "name": self.container_name,- "image": self.image,- "command": ["--command", "ICON"],- "detach": True,- "stream": True,- "auto_remove": True,- "stderr": True,- "stdout": True,- }- self.logger.info(f"Docker config: {container_config}")-- # Run the Docker container- try:- container = self.client.containers.run(**container_config)- for line in container.logs(stream=True):- text = line.decode().strip()- self.logger.info(text)- if '"type": "ICON"' in text:- return json.loads(text)- except Exception as e:- self.logger.error(f"Failed to run docker container: {e}")- return None+ Args:+ command (str): Command to be executed.- def get_variables(self) -> Optional[Any]:+ Returns:+ Optional[Any]: Response from container or None if error occures.+ """+ command = command.upper()self.container_name = self.normalize_container_name(self.image_name)container_config = {"name": self.container_name,"image": self.image,- "command": ["--command", "VARIABLES"],+ "command": ["--command", command],"detach": True,"stream": True,"auto_remove": True,@@ -315,7 +246,7 @@ class UnstractWorker:for line in container.logs(stream=True):text = line.decode().strip()self.logger.info(text)- if '"type": "VARIABLES"' in text:+ if f'"type": "${command}"' in text:return json.loads(text)except Exception as e:self.logger.error(f"Failed to run docker container: {e}")