Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Abstracted container object using a new interface in worker (#386)

Ritwik G97ecc87257ca
diff --git a/worker/sample.env b/worker/sample.env
index bd51b25bb..c43173fcf 100644
--- a/worker/sample.env
+++ b/worker/sample.env
@@ -4,7 +4,7 @@ REDIS_PASSWORD=""
REDIS_USER=default
TOOL_CONTAINER_NETWORK="unstract-network"
-TOOL_CONTAINER_LABELS=""
+TOOL_CONTAINER_LABELS="[]"
WORKFLOW_DATA_DIR="${PWD}/workflow_data/execution"
TOOL_DATA_DIR="/data"
diff --git a/worker/src/unstract/worker/clients/docker.py b/worker/src/unstract/worker/clients/docker.py
index cf757ecef..c6fd447b6 100644
--- a/worker/src/unstract/worker/clients/docker.py
+++ b/worker/src/unstract/worker/clients/docker.py
@@ -1,14 +1,32 @@
import logging
import os
+from collections.abc import Iterator
from typing import Any
from docker.errors import APIError, ImageNotFound
from docker.models.containers import Container
from unstract.worker.constants import Env
+from unstract.worker.utils import Utils
from docker import DockerClient, from_env
-from .interface import ContainerClientInterface
+from .interface import ContainerClientInterface, ContainerInterface
+
+
+class DockerContainer(ContainerInterface):
+ def __init__(self, container: Container) -> None:
+ self.container: Container = container
+
+ def logs(self, follow=False) -> Iterator[str]:
+ return self.container.logs(stream=True, follow=follow)
+
+ def cleanup(self) -> None:
+ if not self.container or not Utils.remove_container_on_exit():
+ return
+ try:
+ self.container.remove(force=True)
+ except Exception as remove_error:
+ self.logger.error(f"Failed to remove docker container: {remove_error}")
class Client(ContainerClientInterface):
@@ -67,6 +85,32 @@ class Client(ContainerClientInterface):
f"Internal service error occured while authentication: {exc}"
)
+ def __image_exists(self, image_name_with_tag: str) -> bool:
+ """Check if the container image exists in system.
+
+ Args:
+ image_name_with_tag (str): The image name with tag.
+
+ Returns:
+ bool: True if the image exists, False otherwise.
+ """
+
+ try:
+ # Attempt to get the image information
+ self.client.images.get(image_name_with_tag)
+ self.logger.info(
+ f"Image '{image_name_with_tag}' found in the local system."
+ )
+ return True
+ except ImageNotFound: # type: ignore[attr-defined]
+ self.logger.info(
+ f"Image '{image_name_with_tag}' not found in the local system."
+ )
+ return False
+ except APIError as e: # type: ignore[attr-defined]
+ self.logger.error(f"An API error occurred: {e}")
+ return False
+
def get_image(self) -> str:
"""Will check if image exists locally and pulls the image using
`self.image_name` and `self.image_tag` if necessary.
@@ -103,31 +147,5 @@ class Client(ContainerClientInterface):
return image_name_with_tag
- def __image_exists(self, image_name_with_tag: str) -> bool:
- """Check if the container image exists in system.
-
- Args:
- image_name_with_tag (str): The image name with tag.
-
- Returns:
- bool: True if the image exists, False otherwise.
- """
-
- try:
- # Attempt to get the image information
- self.client.images.get(image_name_with_tag)
- self.logger.info(
- f"Image '{image_name_with_tag}' found in the local system."
- )
- return True
- except ImageNotFound: # type: ignore[attr-defined]
- self.logger.info(
- f"Image '{image_name_with_tag}' not found in the local system."
- )
- return False
- except APIError as e: # type: ignore[attr-defined]
- self.logger.error(f"An API error occurred: {e}")
- return False
-
- def run_container(self, config: dict[Any, Any]) -> Container:
- return self.client.containers.run(**config)
+ def run_container(self, config: dict[Any, Any]) -> Any:
+ return DockerContainer(self.client.containers.run(**config))
diff --git a/worker/src/unstract/worker/clients/interface.py b/worker/src/unstract/worker/clients/interface.py
index bea1f1b0a..9c4a19d68 100644
--- a/worker/src/unstract/worker/clients/interface.py
+++ b/worker/src/unstract/worker/clients/interface.py
@@ -1,8 +1,27 @@
import logging
from abc import ABC, abstractmethod
+from collections.abc import Iterator
from typing import Any
-from docker.models.containers import Container
+
+class ContainerInterface(ABC):
+
+ @abstractmethod
+ def logs(self, follow=False) -> Iterator[str]:
+ """Returns an iterator object of logs.
+
+ Args:
+ follow (bool, optional): Should the logs be followed. Defaults to False.
+
+ Yields:
+ Iterator[str]: Yields logs line by line.
+ """
+ pass
+
+ @abstractmethod
+ def cleanup(self) -> None:
+ """Stops and removes the running container."""
+ pass
class ContainerClientInterface(ABC):
@@ -12,8 +31,9 @@ class ContainerClientInterface(ABC):
pass
@abstractmethod
- def run_container(self, config: dict[Any, Any]) -> Container:
- """Method to run a container with provided config.
+ def run_container(self, config: dict[Any, Any]) -> ContainerInterface:
+ """Method to run a container with provided config. This method will run
+ the container.
Args:
config (dict[Any, Any]): Configuration for container.
diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.py
index e94230e31..6b302eec2 100644
--- a/worker/src/unstract/worker/worker.py
+++ b/worker/src/unstract/worker/worker.py
@@ -8,9 +8,11 @@ from typing import Any, Optional
from dotenv import load_dotenv
from flask import Flask
from unstract.worker.clients.helper import get_container_client
-from unstract.worker.clients.interface import ContainerClientInterface
+from unstract.worker.clients.interface import (
+ ContainerClientInterface,
+ ContainerInterface,
+)
from unstract.worker.constants import Env, LogType, ToolKey
-from unstract.worker.utils import Utils
from unstract.core.constants import LogFieldName
from unstract.core.pubsub_helper import LogPublisher
@@ -36,13 +38,13 @@ class UnstractWorker:
# Function to stream logs
def stream_logs(
self,
- container: Any,
+ container: ContainerInterface,
tool_instance_id: str,
execution_id: str,
organization_id: str,
channel: Optional[str] = None,
) -> None:
- for line in container.logs(stream=True, follow=True):
+ for line in container.logs(follow=True):
log_message = line.decode().strip()
self.logger.debug(f"[{self.container_name}] - {log_message}")
self.process_log_message(
@@ -141,8 +143,8 @@ class UnstractWorker:
# Run the Docker container
try:
- container = self.client.run_container(container_config)
- for line in container.logs(stream=True):
+ container: ContainerInterface = self.client.run_container(container_config)
+ for line in container.logs():
text = line.decode().strip()
self.logger.info(text)
if f'"type": "${command}"' in text:
@@ -188,7 +190,7 @@ class UnstractWorker:
# Add labels to container for logging with Loki.
# This only required for observability.
try:
- labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, ""))
+ labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, "[]"))
container_config["labels"] = labels
except Exception as e:
self.logger.info(f"Invalid labels for logging: {e}")
@@ -199,7 +201,7 @@ class UnstractWorker:
container = None
result = {"type": "RESULT", "result": None}
try:
- container = self.client.run_container(container_config)
+ container: ContainerInterface = self.client.run_container(container_config)
self.logger.info(f"Running Docker container: {self.container_name}")
tool_instance_id = str(self.settings.get(ToolKey.TOOL_INSTANCE_ID))
# Stream logs
@@ -213,7 +215,8 @@ class UnstractWorker:
except Exception as e:
self.logger.error(f"Failed to run docker container: {e}")
result = {"type": "RESULT", "result": None, "error": str(e)}
- self._cleanup_container(container)
+ if container:
+ container.cleanup()
return result
def get_container_run_config(self) -> dict[str, Any]:
@@ -248,12 +251,3 @@ class UnstractWorker:
}
],
}
-
- def _cleanup_container(self, container: Any) -> None:
- # NOTE: This method might need to be
- if not container or not Utils.remove_container_on_exit():
- return
- try:
- container.remove(force=True)
- except Exception as remove_error:
- self.logger.error(f"Failed to remove docker container: {remove_error}")