LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feat/enhancing worker logs (#302)
ali671f8ccbd874
diff --git a/worker/README.md b/worker/README.mdindex 0e81486aa..f29ff471e 100644--- a/worker/README.md+++ b/worker/README.md@@ -34,3 +34,18 @@ export DOCKER_HOST=unix:///$HOME/.docker/desktop/docker.sock```sudo ln -s "$HOME/.docker/run/docker.sock" /var/run/docker.sock```++## Required Environment Variables++| Variable | Description |+| -------------------------- | ---------------------------------------------------------------------------------------|+| `REDIS_HOST` | Host address of the Redis server. |+| `REDIS_PORT` | Port number for the Redis server. |+| `REDIS_PASSWORD` | Password for accessing the Redis server. (Leave empty if not required) |+| `REDIS_USER` | User for accessing the Redis server. (If applicable) |+| `TOOL_CONTAINER_NETWORK` | Network used for running tool containers. |+| `TOOL_CONTAINER_LABELS` | Labels applied to tool containers for observability [Optional]. |+| `WORKFLOW_DATA_DIR` | Source mount bind directory for tool containers to access input files. |+| `TOOL_DATA_DIR` | Target mount directory within tool containers. (Default: "/data") |+| `LOG_LEVEL` | Log level for worker (Options: INFO, WARNING, ERROR, DEBUG, etc.) |+| `REMOVE_CONTAINER_ON_EXIT`| Flag to decide whether to clean up/ remove the tool container after execution. (Default: True) |diff --git a/worker/sample.env b/worker/sample.envindex fdc3da2cb..15fcfb728 100644--- a/worker/sample.env+++ b/worker/sample.env@@ -11,3 +11,10 @@ TOOL_DATA_DIR="/data"PRIVATE_REGISTRY_CREDENTIAL_PATH=PRIVATE_REGISTRY_USERNAME=PRIVATE_REGISTRY_URL=++# Log level for worker (Options: INFO, WARNING, ERROR, DEBUG, etc.)+LOG_LEVEL = "ERROR"++# Flag to decide whether to clean up/ remove the tool container after execution.+# (Default: True)+REMOVE_CONTAINER_ON_EXIT = Truediff --git a/worker/src/unstract/worker/constants.py b/worker/src/unstract/worker/constants.pyindex beacc2ef1..eeb3bc472 100644--- a/worker/src/unstract/worker/constants.py+++ b/worker/src/unstract/worker/constants.py@@ -25,3 +25,5 @@ class Env:PRIVATE_REGISTRY_CREDENTIAL_PATH = "PRIVATE_REGISTRY_CREDENTIAL_PATH"PRIVATE_REGISTRY_USERNAME = "PRIVATE_REGISTRY_USERNAME"PRIVATE_REGISTRY_URL = "PRIVATE_REGISTRY_URL"+ LOG_LEVEL = "LOG_LEVEL"+ REMOVE_CONTAINER_ON_EXIT = "REMOVE_CONTAINER_ON_EXIT"diff --git a/worker/src/unstract/worker/enum.py b/worker/src/unstract/worker/enum.pynew file mode 100644index 000000000..a771937dd--- /dev/null+++ b/worker/src/unstract/worker/enum.py@@ -0,0 +1,10 @@+import logging+from enum import Enum+++class LogLevel(Enum):+ DEBUG = logging.DEBUG+ INFO = logging.INFO+ WARNING = logging.WARNING+ ERROR = logging.ERROR+ CRITICAL = logging.CRITICALdiff --git a/worker/src/unstract/worker/main.py b/worker/src/unstract/worker/main.pyindex cb618f56f..fa29277a7 100644--- a/worker/src/unstract/worker/main.py+++ b/worker/src/unstract/worker/main.py@@ -3,9 +3,13 @@ from typing import Any, Optionalfrom flask import Blueprint, Flask, Response, jsonify, requestfrom unstract.worker import UnstractWorkerfrom unstract.worker.constants import ToolCommandKey+from unstract.worker.utils import Utilsapp = Flask(__name__)+log_level = Utils.get_log_level()+app.logger.setLevel(log_level.value)+# Define a Blueprint with a root URL pathbp = Blueprint("v1", __name__, url_prefix="/v1/api")@@ -29,7 +33,7 @@ def run_container() -> Optional[Any]:envs = data["envs"]messaging_channel = data["messaging_channel"]- worker = UnstractWorker(image_name, image_tag)+ worker = UnstractWorker(image_name, image_tag, app=app)result = worker.run_container(organization_id=organization_id,workflow_id=workflow_id,diff --git a/worker/src/unstract/worker/utils.py b/worker/src/unstract/worker/utils.pynew file mode 100644index 000000000..f68f76b62--- /dev/null+++ b/worker/src/unstract/worker/utils.py@@ -0,0 +1,48 @@+import os++from dotenv import load_dotenv+from unstract.worker.constants import Env+from unstract.worker.enum import LogLevel++load_dotenv()+++class Utils:+ @staticmethod+ def str_to_bool(string: str) -> bool:+ """Convert string to boolean.++ Args:+ string (str): String to convert++ Returns:+ bool+ """+ if string.lower() == "true":+ return True+ elif string.lower() == "false":+ return False+ else:+ raise ValueError("Invalid boolean string")++ @staticmethod+ def get_log_level() -> LogLevel:+ """Get log level from environment variable.++ Returns:+ LogLevel+ """+ try:+ log_level_str = os.environ.get(Env.LOG_LEVEL, "").upper()+ return LogLevel[log_level_str.upper()]+ except KeyError:+ return LogLevel.INFO++ @staticmethod+ def remove_container_on_exit() -> bool:+ """Get remove container on exit from environment variable.++ Returns:+ bool+ """+ return Utils.str_to_bool(os.environ.get(Env.REMOVE_CONTAINER_ON_EXIT, "true"))diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.pyindex b92eba7ce..66e5fce05 100644--- a/worker/src/unstract/worker/worker.py+++ b/worker/src/unstract/worker/worker.py@@ -1,13 +1,14 @@import astimport json-import loggingimport osimport uuidfrom typing import Any, Optionalfrom docker.errors import APIErrorfrom dotenv import load_dotenv+from flask import Flaskfrom unstract.worker.constants import Env, LogType, ToolKey+from unstract.worker.utils import Utilsimport dockerfrom docker import DockerClient # type: ignore[attr-defined]@@ -15,14 +16,13 @@ from unstract.core.pubsub_helper import LogPublisherload_dotenv()-logger = logging.getLogger(__name__)-class UnstractWorker:- def __init__(self, image_name: str, image_tag: str) -> None:+ def __init__(self, image_name: str, image_tag: str, app: Flask) -> None:self.image_name = image_name# If no image_tag is provided will assume the `latest` tagself.image_tag = image_tag or "latest"+ self.logger = app.logger# Create a Docker client that communicates with# the Docker daemon in the host environmentself.client: DockerClient = docker.from_env() # type: ignore[attr-defined] # noqa: E501@@ -70,7 +70,7 @@ class UnstractWorker:if self._image_exists(image_name_with_tag):return image_name_with_tag- logger.info("Pulling the container: %s", image_name_with_tag)+ self.logger.info("Pulling the container: %s", image_name_with_tag)resp = self.client.api.pull(repository=self.image_name,tag=self.image_tag,@@ -85,13 +85,13 @@ class UnstractWorker:counter += 1continuecounter = 0- logger.info(+ self.logger.info("CONTAINER PULL STATUS: %s - %s : %s",line.get("status"),line.get("id"),line.get("progress"),)- logger.info("Finished pulling the container: %s", image_name_with_tag)+ self.logger.info("Finished pulling the container: %s", image_name_with_tag)return image_name_with_tag@@ -108,13 +108,17 @@ class UnstractWorker:try:# Attempt to get the image informationself.client.images.get(image_name_with_tag)- logger.info(f"Image '{image_name_with_tag}' found in the local system.")+ self.logger.info(+ f"Image '{image_name_with_tag}' found in the local system."+ )return Trueexcept docker.errors.ImageNotFound: # type: ignore[attr-defined]- logger.info(f"Image '{image_name_with_tag}' not found in the local system.")+ self.logger.info(+ f"Image '{image_name_with_tag}' not found in the local system."+ )return Falseexcept docker.errors.APIError as e: # type: ignore[attr-defined]- logger.error(f"An API error occurred: {e}")+ self.logger.error(f"An API error occurred: {e}")return Falsedef normalize_container_name(self, name: str) -> str:@@ -129,27 +133,28 @@ class UnstractWorker:) -> None:for line in container.logs(stream=True, follow=True):log_message = line.decode().strip()- logger.info(f"Log message: {log_message}")+ self.logger.info(f"[{self.container_name}] - {log_message}")self.process_log_message(log_message=log_message,tool_instance_id=tool_instance_id,channel=channel,)- def get_valid_log_message(- self, log_message: str, channel: Optional[str] = None- ) -> Optional[dict[str, Any]]:+ def get_valid_log_message(self, log_message: str) -> Optional[dict[str, Any]]:+ """Get a valid log message from the log message.++ Args:+ log_message (str): str++ Returns:+ Optional[dict[str, Any]]: json message+ """try:log_dict = json.loads(log_message)- if not isinstance(log_dict, dict):- logger.error(f"Received invalid log: {log_dict}")- else:+ if isinstance(log_dict, dict):return log_dict- except json.JSONDecodeError as e:- logger.warn(- f"Received invalid JSON log message: {log_message} \n " f"error: {e}"- )- return None+ except json.JSONDecodeError:+ return Nonedef process_log_message(self,@@ -157,12 +162,12 @@ class UnstractWorker:tool_instance_id: str,channel: Optional[str] = None,) -> Optional[dict[str, Any]]:- log_dict = self.get_valid_log_message(log_message, channel)+ log_dict = self.get_valid_log_message(log_message)if not log_dict:return Nonelog_type = self.get_log_type(log_dict)if not self.is_valid_log_type(log_type):- logger.warning(+ self.logger.warning(f"Received invalid logType: {log_type} with log message: " f"{log_dict}")return None@@ -204,18 +209,18 @@ class UnstractWorker:"stderr": True,"stdout": True,}- logger.info(f"Docker config: {container_config}")+ self.logger.info(f"Docker config: {container_config}")# Run the Docker containertry:container = self.client.containers.run(**container_config)for line in container.logs(stream=True):text = line.decode().strip()- logger.info(text)+ self.logger.info(text)if '"type": "SPEC"' in text:return json.loads(text)except Exception as e:- logger.error(f"Failed to run docker container: {e}")+ self.logger.error(f"Failed to run docker container: {e}")return Nonedef get_properties(self) -> Optional[Any]:@@ -230,18 +235,18 @@ class UnstractWorker:"stderr": True,"stdout": True,}- logger.info(f"Docker config: {container_config}")+ self.logger.info(f"Docker config: {container_config}")# Run the Docker containertry:container = self.client.containers.run(**container_config)for line in container.logs(stream=True):text = line.decode().strip()- logger.info(text)+ self.logger.info(text)if '"type": "PROPERTIES"' in text:return json.loads(text)except Exception as e:- logger.error(f"Failed to run docker container: {e}")+ self.logger.error(f"Failed to run docker container: {e}")return Nonedef get_icon(self) -> Optional[Any]:@@ -256,18 +261,18 @@ class UnstractWorker:"stderr": True,"stdout": True,}- logger.info(f"Docker config: {container_config}")+ self.logger.info(f"Docker config: {container_config}")# Run the Docker containertry:container = self.client.containers.run(**container_config)for line in container.logs(stream=True):text = line.decode().strip()- logger.info(text)+ self.logger.info(text)if '"type": "ICON"' in text:return json.loads(text)except Exception as e:- logger.error(f"Failed to run docker container: {e}")+ self.logger.error(f"Failed to run docker container: {e}")return Nonedef get_variables(self) -> Optional[Any]:@@ -282,18 +287,18 @@ class UnstractWorker:"stderr": True,"stdout": True,}- logger.info(f"Docker config: {container_config}")+ self.logger.info(f"Docker config: {container_config}")# Run the Docker containertry:container = self.client.containers.run(**container_config)for line in container.logs(stream=True):text = line.decode().strip()- logger.info(text)+ self.logger.info(text)if '"type": "VARIABLES"' in text:return json.loads(text)except Exception as e:- logger.error(f"Failed to run docker container: {e}")+ self.logger.error(f"Failed to run docker container: {e}")return Nonedef run_container(@@ -330,21 +335,23 @@ class UnstractWorker:self.normalize_container_name(self.image_name) + "-" + self.workflow_id)container_config = self.get_container_run_config()+ remove_container_on_exit = Utils.remove_container_on_exit()# Add labels to container for logging with Loki.# This only required for observability.try:labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, ""))container_config["labels"] = labelsexcept Exception as e:- logger.info(f"Invalid labels for logging: {e}")+ self.logger.info(f"Invalid labels for logging: {e}")- logger.info(f"Docker config: {container_config}")+ self.logger.info(f"Docker config: {container_config}")# Run the Docker containercontainer = Noneresult = {"type": "RESULT", "result": None}try:container = self.client.containers.run(**container_config)+ self.logger.info(f"Running Docker container: {self.container_name}")tool_instance_id = str(self.settings.get(ToolKey.TOOL_INSTANCE_ID))# Stream logsself.stream_logs(@@ -353,9 +360,9 @@ class UnstractWorker:channel=self.messaging_channel,)except Exception as e:- logger.error(f"Failed to run docker container: {e}")+ self.logger.error(f"Failed to run docker container: {e}")result = {"type": "RESULT", "result": None, "error": str(e)}- if container:+ if container and remove_container_on_exit:self._cleanup_container(container)return result@@ -396,4 +403,4 @@ class UnstractWorker:try:container.remove(force=True)except Exception as remove_error:- logger.error(f"Failed to remove docker container: {remove_error}")+ self.logger.error(f"Failed to remove docker container: {remove_error}")