Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Feat/enhancing worker logs (#302)

ali671f8ccbd874
diff --git a/worker/README.md b/worker/README.md
index 0e81486aa..f29ff471e 100644
--- a/worker/README.md
+++ b/worker/README.md
@@ -34,3 +34,18 @@ export DOCKER_HOST=unix:///$HOME/.docker/desktop/docker.sock
```
sudo ln -s "$HOME/.docker/run/docker.sock" /var/run/docker.sock
```
+
+## Required Environment Variables
+
+| Variable | Description |
+| -------------------------- | ---------------------------------------------------------------------------------------|
+| `REDIS_HOST` | Host address of the Redis server. |
+| `REDIS_PORT` | Port number for the Redis server. |
+| `REDIS_PASSWORD` | Password for accessing the Redis server. (Leave empty if not required) |
+| `REDIS_USER` | User for accessing the Redis server. (If applicable) |
+| `TOOL_CONTAINER_NETWORK` | Network used for running tool containers. |
+| `TOOL_CONTAINER_LABELS` | Labels applied to tool containers for observability [Optional]. |
+| `WORKFLOW_DATA_DIR` | Source mount bind directory for tool containers to access input files. |
+| `TOOL_DATA_DIR` | Target mount directory within tool containers. (Default: "/data") |
+| `LOG_LEVEL` | Log level for worker (Options: INFO, WARNING, ERROR, DEBUG, etc.) |
+| `REMOVE_CONTAINER_ON_EXIT`| Flag to decide whether to clean up/ remove the tool container after execution. (Default: True) |
diff --git a/worker/sample.env b/worker/sample.env
index fdc3da2cb..15fcfb728 100644
--- a/worker/sample.env
+++ b/worker/sample.env
@@ -11,3 +11,10 @@ TOOL_DATA_DIR="/data"
PRIVATE_REGISTRY_CREDENTIAL_PATH=
PRIVATE_REGISTRY_USERNAME=
PRIVATE_REGISTRY_URL=
+
+# Log level for worker (Options: INFO, WARNING, ERROR, DEBUG, etc.)
+LOG_LEVEL = "ERROR"
+
+# Flag to decide whether to clean up/ remove the tool container after execution.
+# (Default: True)
+REMOVE_CONTAINER_ON_EXIT = True
diff --git a/worker/src/unstract/worker/constants.py b/worker/src/unstract/worker/constants.py
index beacc2ef1..eeb3bc472 100644
--- a/worker/src/unstract/worker/constants.py
+++ b/worker/src/unstract/worker/constants.py
@@ -25,3 +25,5 @@ class Env:
PRIVATE_REGISTRY_CREDENTIAL_PATH = "PRIVATE_REGISTRY_CREDENTIAL_PATH"
PRIVATE_REGISTRY_USERNAME = "PRIVATE_REGISTRY_USERNAME"
PRIVATE_REGISTRY_URL = "PRIVATE_REGISTRY_URL"
+ LOG_LEVEL = "LOG_LEVEL"
+ REMOVE_CONTAINER_ON_EXIT = "REMOVE_CONTAINER_ON_EXIT"
diff --git a/worker/src/unstract/worker/enum.py b/worker/src/unstract/worker/enum.py
new file mode 100644
index 000000000..a771937dd
--- /dev/null
+++ b/worker/src/unstract/worker/enum.py
@@ -0,0 +1,10 @@
+import logging
+from enum import Enum
+
+
+class LogLevel(Enum):
+ DEBUG = logging.DEBUG
+ INFO = logging.INFO
+ WARNING = logging.WARNING
+ ERROR = logging.ERROR
+ CRITICAL = logging.CRITICAL
diff --git a/worker/src/unstract/worker/main.py b/worker/src/unstract/worker/main.py
index cb618f56f..fa29277a7 100644
--- a/worker/src/unstract/worker/main.py
+++ b/worker/src/unstract/worker/main.py
@@ -3,9 +3,13 @@ from typing import Any, Optional
from flask import Blueprint, Flask, Response, jsonify, request
from unstract.worker import UnstractWorker
from unstract.worker.constants import ToolCommandKey
+from unstract.worker.utils import Utils
app = Flask(__name__)
+log_level = Utils.get_log_level()
+app.logger.setLevel(log_level.value)
+
# Define a Blueprint with a root URL path
bp = Blueprint("v1", __name__, url_prefix="/v1/api")
@@ -29,7 +33,7 @@ def run_container() -> Optional[Any]:
envs = data["envs"]
messaging_channel = data["messaging_channel"]
- worker = UnstractWorker(image_name, image_tag)
+ worker = UnstractWorker(image_name, image_tag, app=app)
result = worker.run_container(
organization_id=organization_id,
workflow_id=workflow_id,
diff --git a/worker/src/unstract/worker/utils.py b/worker/src/unstract/worker/utils.py
new file mode 100644
index 000000000..f68f76b62
--- /dev/null
+++ b/worker/src/unstract/worker/utils.py
@@ -0,0 +1,48 @@
+import os
+
+from dotenv import load_dotenv
+from unstract.worker.constants import Env
+from unstract.worker.enum import LogLevel
+
+load_dotenv()
+
+
+class Utils:
+ @staticmethod
+ def str_to_bool(string: str) -> bool:
+ """Convert string to boolean.
+
+ Args:
+ string (str): String to convert
+
+ Returns:
+ bool
+ """
+ if string.lower() == "true":
+ return True
+ elif string.lower() == "false":
+ return False
+ else:
+ raise ValueError("Invalid boolean string")
+
+ @staticmethod
+ def get_log_level() -> LogLevel:
+ """Get log level from environment variable.
+
+ Returns:
+ LogLevel
+ """
+ try:
+ log_level_str = os.environ.get(Env.LOG_LEVEL, "").upper()
+ return LogLevel[log_level_str.upper()]
+ except KeyError:
+ return LogLevel.INFO
+
+ @staticmethod
+ def remove_container_on_exit() -> bool:
+ """Get remove container on exit from environment variable.
+
+ Returns:
+ bool
+ """
+ return Utils.str_to_bool(os.environ.get(Env.REMOVE_CONTAINER_ON_EXIT, "true"))
diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.py
index b92eba7ce..66e5fce05 100644
--- a/worker/src/unstract/worker/worker.py
+++ b/worker/src/unstract/worker/worker.py
@@ -1,13 +1,14 @@
import ast
import json
-import logging
import os
import uuid
from typing import Any, Optional
from docker.errors import APIError
from dotenv import load_dotenv
+from flask import Flask
from unstract.worker.constants import Env, LogType, ToolKey
+from unstract.worker.utils import Utils
import docker
from docker import DockerClient # type: ignore[attr-defined]
@@ -15,14 +16,13 @@ from unstract.core.pubsub_helper import LogPublisher
load_dotenv()
-logger = logging.getLogger(__name__)
-
class UnstractWorker:
- def __init__(self, image_name: str, image_tag: str) -> None:
+ def __init__(self, image_name: str, image_tag: str, app: Flask) -> None:
self.image_name = image_name
# If no image_tag is provided will assume the `latest` tag
self.image_tag = image_tag or "latest"
+ self.logger = app.logger
# Create a Docker client that communicates with
# the Docker daemon in the host environment
self.client: DockerClient = docker.from_env() # type: ignore[attr-defined] # noqa: E501
@@ -70,7 +70,7 @@ class UnstractWorker:
if self._image_exists(image_name_with_tag):
return image_name_with_tag
- logger.info("Pulling the container: %s", image_name_with_tag)
+ self.logger.info("Pulling the container: %s", image_name_with_tag)
resp = self.client.api.pull(
repository=self.image_name,
tag=self.image_tag,
@@ -85,13 +85,13 @@ class UnstractWorker:
counter += 1
continue
counter = 0
- logger.info(
+ self.logger.info(
"CONTAINER PULL STATUS: %s - %s : %s",
line.get("status"),
line.get("id"),
line.get("progress"),
)
- logger.info("Finished pulling the container: %s", image_name_with_tag)
+ self.logger.info("Finished pulling the container: %s", image_name_with_tag)
return image_name_with_tag
@@ -108,13 +108,17 @@ class UnstractWorker:
try:
# Attempt to get the image information
self.client.images.get(image_name_with_tag)
- logger.info(f"Image '{image_name_with_tag}' found in the local system.")
+ self.logger.info(
+ f"Image '{image_name_with_tag}' found in the local system."
+ )
return True
except docker.errors.ImageNotFound: # type: ignore[attr-defined]
- logger.info(f"Image '{image_name_with_tag}' not found in the local system.")
+ self.logger.info(
+ f"Image '{image_name_with_tag}' not found in the local system."
+ )
return False
except docker.errors.APIError as e: # type: ignore[attr-defined]
- logger.error(f"An API error occurred: {e}")
+ self.logger.error(f"An API error occurred: {e}")
return False
def normalize_container_name(self, name: str) -> str:
@@ -129,27 +133,28 @@ class UnstractWorker:
) -> None:
for line in container.logs(stream=True, follow=True):
log_message = line.decode().strip()
- logger.info(f"Log message: {log_message}")
+ self.logger.info(f"[{self.container_name}] - {log_message}")
self.process_log_message(
log_message=log_message,
tool_instance_id=tool_instance_id,
channel=channel,
)
- def get_valid_log_message(
- self, log_message: str, channel: Optional[str] = None
- ) -> Optional[dict[str, Any]]:
+ def get_valid_log_message(self, log_message: str) -> Optional[dict[str, Any]]:
+ """Get a valid log message from the log message.
+
+ Args:
+ log_message (str): str
+
+ Returns:
+ Optional[dict[str, Any]]: json message
+ """
try:
log_dict = json.loads(log_message)
- if not isinstance(log_dict, dict):
- logger.error(f"Received invalid log: {log_dict}")
- else:
+ if isinstance(log_dict, dict):
return log_dict
- except json.JSONDecodeError as e:
- logger.warn(
- f"Received invalid JSON log message: {log_message} \n " f"error: {e}"
- )
- return None
+ except json.JSONDecodeError:
+ return None
def process_log_message(
self,
@@ -157,12 +162,12 @@ class UnstractWorker:
tool_instance_id: str,
channel: Optional[str] = None,
) -> Optional[dict[str, Any]]:
- log_dict = self.get_valid_log_message(log_message, channel)
+ log_dict = self.get_valid_log_message(log_message)
if not log_dict:
return None
log_type = self.get_log_type(log_dict)
if not self.is_valid_log_type(log_type):
- logger.warning(
+ self.logger.warning(
f"Received invalid logType: {log_type} with log message: " f"{log_dict}"
)
return None
@@ -204,18 +209,18 @@ class UnstractWorker:
"stderr": True,
"stdout": True,
}
- logger.info(f"Docker config: {container_config}")
+ self.logger.info(f"Docker config: {container_config}")
# Run the Docker container
try:
container = self.client.containers.run(**container_config)
for line in container.logs(stream=True):
text = line.decode().strip()
- logger.info(text)
+ self.logger.info(text)
if '"type": "SPEC"' in text:
return json.loads(text)
except Exception as e:
- logger.error(f"Failed to run docker container: {e}")
+ self.logger.error(f"Failed to run docker container: {e}")
return None
def get_properties(self) -> Optional[Any]:
@@ -230,18 +235,18 @@ class UnstractWorker:
"stderr": True,
"stdout": True,
}
- logger.info(f"Docker config: {container_config}")
+ self.logger.info(f"Docker config: {container_config}")
# Run the Docker container
try:
container = self.client.containers.run(**container_config)
for line in container.logs(stream=True):
text = line.decode().strip()
- logger.info(text)
+ self.logger.info(text)
if '"type": "PROPERTIES"' in text:
return json.loads(text)
except Exception as e:
- logger.error(f"Failed to run docker container: {e}")
+ self.logger.error(f"Failed to run docker container: {e}")
return None
def get_icon(self) -> Optional[Any]:
@@ -256,18 +261,18 @@ class UnstractWorker:
"stderr": True,
"stdout": True,
}
- logger.info(f"Docker config: {container_config}")
+ self.logger.info(f"Docker config: {container_config}")
# Run the Docker container
try:
container = self.client.containers.run(**container_config)
for line in container.logs(stream=True):
text = line.decode().strip()
- logger.info(text)
+ self.logger.info(text)
if '"type": "ICON"' in text:
return json.loads(text)
except Exception as e:
- logger.error(f"Failed to run docker container: {e}")
+ self.logger.error(f"Failed to run docker container: {e}")
return None
def get_variables(self) -> Optional[Any]:
@@ -282,18 +287,18 @@ class UnstractWorker:
"stderr": True,
"stdout": True,
}
- logger.info(f"Docker config: {container_config}")
+ self.logger.info(f"Docker config: {container_config}")
# Run the Docker container
try:
container = self.client.containers.run(**container_config)
for line in container.logs(stream=True):
text = line.decode().strip()
- logger.info(text)
+ self.logger.info(text)
if '"type": "VARIABLES"' in text:
return json.loads(text)
except Exception as e:
- logger.error(f"Failed to run docker container: {e}")
+ self.logger.error(f"Failed to run docker container: {e}")
return None
def run_container(
@@ -330,21 +335,23 @@ class UnstractWorker:
self.normalize_container_name(self.image_name) + "-" + self.workflow_id
)
container_config = self.get_container_run_config()
+ remove_container_on_exit = Utils.remove_container_on_exit()
# Add labels to container for logging with Loki.
# This only required for observability.
try:
labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, ""))
container_config["labels"] = labels
except Exception as e:
- logger.info(f"Invalid labels for logging: {e}")
+ self.logger.info(f"Invalid labels for logging: {e}")
- logger.info(f"Docker config: {container_config}")
+ self.logger.info(f"Docker config: {container_config}")
# Run the Docker container
container = None
result = {"type": "RESULT", "result": None}
try:
container = self.client.containers.run(**container_config)
+ self.logger.info(f"Running Docker container: {self.container_name}")
tool_instance_id = str(self.settings.get(ToolKey.TOOL_INSTANCE_ID))
# Stream logs
self.stream_logs(
@@ -353,9 +360,9 @@ class UnstractWorker:
channel=self.messaging_channel,
)
except Exception as e:
- logger.error(f"Failed to run docker container: {e}")
+ self.logger.error(f"Failed to run docker container: {e}")
result = {"type": "RESULT", "result": None, "error": str(e)}
- if container:
+ if container and remove_container_on_exit:
self._cleanup_container(container)
return result
@@ -396,4 +403,4 @@ class UnstractWorker:
try:
container.remove(force=True)
except Exception as remove_error:
- logger.error(f"Failed to remove docker container: {remove_error}")
+ self.logger.error(f"Failed to remove docker container: {remove_error}")