Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Refactored container config and some fixes in run command API (#389)

Ritwik G2e1c1a783783
diff --git a/docker/dockerfiles/worker.Dockerfile b/docker/dockerfiles/worker.Dockerfile
index 2727e669d..58f86454a 100644
--- a/docker/dockerfiles/worker.Dockerfile
+++ b/docker/dockerfiles/worker.Dockerfile
@@ -34,6 +34,8 @@ RUN set -e; \
\
pdm sync --prod --no-editable; \
\
+ [ -f requirements.txt ] && pip install -r requirements.txt || { echo "requirements.txt does not exist";}; \
+ \
# REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenv
pip install --no-cache-dir gunicorn gevent;
diff --git a/worker/.gitignore b/worker/.gitignore
index 4b3ab3003..84321f311 100644
--- a/worker/.gitignore
+++ b/worker/.gitignore
@@ -160,3 +160,9 @@ cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
+
+src/unstract/worker/clients/
+!src/unstract/worker/clients/__init__.py
+!src/unstract/worker/clients/interface.py
+!src/unstract/worker/clients/helper.py
+!src/unstract/worker/clients/docker.py
diff --git a/worker/src/unstract/worker/clients/docker.py b/worker/src/unstract/worker/clients/docker.py
index c6fd447b6..c741be947 100644
--- a/worker/src/unstract/worker/clients/docker.py
+++ b/worker/src/unstract/worker/clients/docker.py
@@ -1,7 +1,7 @@
import logging
import os
from collections.abc import Iterator
-from typing import Any
+from typing import Any, Optional
from docker.errors import APIError, ImageNotFound
from docker.models.containers import Container
@@ -10,6 +10,7 @@ from unstract.worker.utils import Utils
from docker import DockerClient, from_env
+from .helper import normalize_container_name
from .interface import ContainerClientInterface, ContainerInterface
@@ -17,8 +18,13 @@ class DockerContainer(ContainerInterface):
def __init__(self, container: Container) -> None:
self.container: Container = container
- def logs(self, follow=False) -> Iterator[str]:
- return self.container.logs(stream=True, follow=follow)
+ @property
+ def name(self):
+ return self.container.name
+
+ def logs(self, follow=True) -> Iterator[str]:
+ for line in self.container.logs(stream=True, follow=follow):
+ yield line.decode().strip()
def cleanup(self) -> None:
if not self.container or not Utils.remove_container_on_exit():
@@ -147,5 +153,45 @@ class Client(ContainerClientInterface):
return image_name_with_tag
+ def get_container_run_config(
+ self,
+ command: list[str],
+ organization_id: str,
+ workflow_id: str,
+ execution_id: str,
+ envs: Optional[dict[str, Any]] = None,
+ auto_remove: bool = False,
+ ) -> dict[str, Any]:
+ if envs is None:
+ envs = {}
+ mounts = []
+ if organization_id and workflow_id and execution_id:
+ mounts.append(
+ {
+ "type": "bind",
+ "source": os.path.join(
+ os.getenv(Env.WORKFLOW_DATA_DIR, ""),
+ organization_id,
+ workflow_id,
+ execution_id,
+ ),
+ "target": os.getenv(Env.TOOL_DATA_DIR, "/data"),
+ }
+ )
+ return {
+ "name": normalize_container_name(self.image_name),
+ "image": self.get_image(),
+ "command": command,
+ "detach": True,
+ "stream": True,
+ "auto_remove": auto_remove,
+ "environment": envs,
+ "stderr": True,
+ "stdout": True,
+ "network": os.getenv(Env.TOOL_CONTAINER_NETWORK, ""),
+ "mounts": mounts,
+ }
+
def run_container(self, config: dict[Any, Any]) -> Any:
+ self.logger.info(f"Docker config: {config}")
return DockerContainer(self.client.containers.run(**config))
diff --git a/worker/src/unstract/worker/clients/helper.py b/worker/src/unstract/worker/clients/helper.py
index db7bf8bd9..87b6d46a5 100644
--- a/worker/src/unstract/worker/clients/helper.py
+++ b/worker/src/unstract/worker/clients/helper.py
@@ -1,4 +1,5 @@
import os
+import uuid
from importlib import import_module
from .interface import ContainerClientInterface
@@ -6,4 +7,9 @@ from .interface import ContainerClientInterface
def get_container_client() -> ContainerClientInterface:
client_path = os.getenv("CONTAINER_CLIENT_PATH", "unstract.worker.clients.docker")
+ print("Loading the container client from path:", client_path)
return import_module(client_path).Client
+
+
+def normalize_container_name(name: str) -> str:
+ return name.replace("/", "-") + "-" + str(uuid.uuid4())
diff --git a/worker/src/unstract/worker/clients/interface.py b/worker/src/unstract/worker/clients/interface.py
index 9c4a19d68..fa731e247 100644
--- a/worker/src/unstract/worker/clients/interface.py
+++ b/worker/src/unstract/worker/clients/interface.py
@@ -1,13 +1,18 @@
import logging
from abc import ABC, abstractmethod
from collections.abc import Iterator
-from typing import Any
+from typing import Any, Optional
class ContainerInterface(ABC):
+ @property
+ @abstractmethod
+ def name(self):
+ """The name of the container."""
+ pass
@abstractmethod
- def logs(self, follow=False) -> Iterator[str]:
+ def logs(self, follow=True) -> Iterator[str]:
"""Returns an iterator object of logs.
Args:
@@ -52,3 +57,20 @@ class ContainerClientInterface(ABC):
str: full image name with tag.
"""
pass
+
+ @abstractmethod
+ def get_container_run_config(
+ self,
+ command: list[str],
+ organization_id: str,
+ workflow_id: str,
+ execution_id: str,
+ envs: Optional[dict[str, Any]] = None,
+ auto_remove: bool = False,
+ ) -> dict[str, Any]:
+ """Generate the configuration dictionary to run the container.
+
+ Returns:
+ dict[str, Any]: Configuration for running the container.
+ """
+ pass
diff --git a/worker/src/unstract/worker/main.py b/worker/src/unstract/worker/main.py
index 36858b714..f2e2915d4 100644
--- a/worker/src/unstract/worker/main.py
+++ b/worker/src/unstract/worker/main.py
@@ -32,7 +32,7 @@ def run_container() -> Optional[Any]:
envs = data["envs"]
messaging_channel = data["messaging_channel"]
- worker = UnstractWorker(image_name, image_tag, app=app)
+ worker = UnstractWorker(image_name, image_tag, app)
result = worker.run_container(
organization_id=organization_id,
workflow_id=workflow_id,
@@ -59,7 +59,7 @@ def run_command(command: str) -> Optional[Any]:
abort(404)
image_name = request.args.get("image_name")
image_tag = request.args.get("image_tag")
- worker = UnstractWorker(image_name, image_tag)
+ worker = UnstractWorker(image_name, image_tag, app)
return worker.run_command(command)
diff --git a/worker/src/unstract/worker/utils.py b/worker/src/unstract/worker/utils.py
index f68f76b62..267a59bd0 100644
--- a/worker/src/unstract/worker/utils.py
+++ b/worker/src/unstract/worker/utils.py
@@ -33,7 +33,7 @@ class Utils:
LogLevel
"""
try:
- log_level_str = os.environ.get(Env.LOG_LEVEL, "").upper()
+ log_level_str = os.getenv(Env.LOG_LEVEL, "").upper()
return LogLevel[log_level_str.upper()]
except KeyError:
return LogLevel.INFO
@@ -45,4 +45,4 @@ class Utils:
Returns:
bool
"""
- return Utils.str_to_bool(os.environ.get(Env.REMOVE_CONTAINER_ON_EXIT, "true"))
+ return Utils.str_to_bool(os.getenv(Env.REMOVE_CONTAINER_ON_EXIT, "true"))
diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.py
index 6b302eec2..5dd46a658 100644
--- a/worker/src/unstract/worker/worker.py
+++ b/worker/src/unstract/worker/worker.py
@@ -1,7 +1,6 @@
import ast
import json
import os
-import uuid
from datetime import datetime, timezone
from typing import Any, Optional
@@ -18,6 +17,8 @@ from unstract.core.constants import LogFieldName
from unstract.core.pubsub_helper import LogPublisher
load_dotenv()
+# Loads the container clinet class.
+client_class = get_container_client()
class UnstractWorker:
@@ -26,14 +27,9 @@ class UnstractWorker:
# If no image_tag is provided will assume the `latest` tag
self.image_tag = image_tag or "latest"
self.logger = app.logger
- client_class = get_container_client()
self.client: ContainerClientInterface = client_class(
self.image_name, self.image_tag, self.logger
)
- self.image = self.client.get_image()
-
- def normalize_container_name(self, name: str) -> str:
- return name.replace("/", "-") + "-" + str(uuid.uuid4())
# Function to stream logs
def stream_logs(
@@ -46,7 +42,7 @@ class UnstractWorker:
) -> None:
for line in container.logs(follow=True):
log_message = line.decode().strip()
- self.logger.debug(f"[{self.container_name}] - {log_message}")
+ self.logger.debug(f"[{container.name}] - {log_message}")
self.process_log_message(
log_message=log_message,
tool_instance_id=tool_instance_id,
@@ -128,29 +124,26 @@ class UnstractWorker:
Optional[Any]: Response from container or None if error occures.
"""
command = command.upper()
- self.container_name = self.normalize_container_name(self.image_name)
- container_config = {
- "name": self.container_name,
- "image": self.image,
- "command": ["--command", command],
- "detach": True,
- "stream": True,
- "auto_remove": True,
- "stderr": True,
- "stdout": True,
- }
- self.logger.info(f"Docker config: {container_config}")
+ container_config = self.client.get_container_run_config(
+ command=["--command", command],
+ organization_id="",
+ workflow_id="",
+ execution_id="",
+ auto_remove=True,
+ )
+ container = None
# Run the Docker container
try:
container: ContainerInterface = self.client.run_container(container_config)
- for line in container.logs():
- text = line.decode().strip()
- self.logger.info(text)
- if f'"type": "${command}"' in text:
+ for text in container.logs(follow=True):
+ self.logger.info(f"[{container.name}] - {text}")
+ if f'"type": "{command}"' in text:
return json.loads(text)
except Exception as e:
self.logger.error(f"Failed to run docker container: {e}")
+ if container:
+ container.cleanup()
return None
def run_container(
@@ -174,41 +167,44 @@ class UnstractWorker:
Returns:
Optional[Any]: _description_
"""
- tool_data_dir = os.environ.get(Env.TOOL_DATA_DIR, "/data")
+ tool_data_dir = os.getenv(Env.TOOL_DATA_DIR, "/data")
envs[Env.TOOL_DATA_DIR] = tool_data_dir
- self.organization_id = organization_id
- self.workflow_id = workflow_id
- self.execution_id = execution_id
- self.params: dict[str, Any] = {}
- self.settings = settings
- self.envs = envs
- self.messaging_channel = messaging_channel
- self.container_name = (
- self.normalize_container_name(self.image_name) + "-" + self.workflow_id
+ container_config = self.client.get_container_run_config(
+ command=[
+ "--command",
+ "RUN",
+ "--settings",
+ json.dumps(settings),
+ "--log-level",
+ "DEBUG",
+ ],
+ organization_id=organization_id,
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ envs=envs,
)
- container_config = self.get_container_run_config()
# Add labels to container for logging with Loki.
# This only required for observability.
try:
- labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, "[]"))
+ labels = ast.literal_eval(os.getenv(Env.TOOL_CONTAINER_LABELS, "[]"))
container_config["labels"] = labels
except Exception as e:
self.logger.info(f"Invalid labels for logging: {e}")
- self.logger.info(f"Docker config: {container_config}")
-
# Run the Docker container
container = None
result = {"type": "RESULT", "result": None}
try:
container: ContainerInterface = self.client.run_container(container_config)
- self.logger.info(f"Running Docker container: {self.container_name}")
- tool_instance_id = str(self.settings.get(ToolKey.TOOL_INSTANCE_ID))
+ self.logger.info(
+ f"Running Docker container: {container_config.get('name')}"
+ )
+ tool_instance_id = str(settings.get(ToolKey.TOOL_INSTANCE_ID))
# Stream logs
self.stream_logs(
container=container,
tool_instance_id=tool_instance_id,
- channel=self.messaging_channel,
+ channel=messaging_channel,
execution_id=execution_id,
organization_id=organization_id,
)
@@ -218,36 +214,3 @@ class UnstractWorker:
if container:
container.cleanup()
return result
-
- def get_container_run_config(self) -> dict[str, Any]:
- return {
- "name": self.container_name,
- "image": self.image,
- "command": [
- "--command",
- "RUN",
- "--settings",
- json.dumps(self.settings),
- "--log-level",
- "DEBUG",
- ],
- "detach": True,
- "stream": True,
- "auto_remove": False,
- "environment": self.envs,
- "stderr": True,
- "stdout": True,
- "network": os.environ.get(Env.TOOL_CONTAINER_NETWORK, ""),
- "mounts": [
- {
- "type": "bind",
- "source": os.path.join(
- os.environ.get(Env.WORKFLOW_DATA_DIR, ""),
- self.organization_id,
- self.workflow_id,
- self.execution_id,
- ),
- "target": os.environ.get(Env.TOOL_DATA_DIR, "/data"),
- }
- ],
- }