LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Abstracted container object using a new interface in worker (#386)
Ritwik G97ecc87257ca
diff --git a/worker/sample.env b/worker/sample.envindex bd51b25bb..c43173fcf 100644--- a/worker/sample.env+++ b/worker/sample.env@@ -4,7 +4,7 @@ REDIS_PASSWORD=""REDIS_USER=defaultTOOL_CONTAINER_NETWORK="unstract-network"-TOOL_CONTAINER_LABELS=""+TOOL_CONTAINER_LABELS="[]"WORKFLOW_DATA_DIR="${PWD}/workflow_data/execution"TOOL_DATA_DIR="/data"diff --git a/worker/src/unstract/worker/clients/docker.py b/worker/src/unstract/worker/clients/docker.pyindex cf757ecef..c6fd447b6 100644--- a/worker/src/unstract/worker/clients/docker.py+++ b/worker/src/unstract/worker/clients/docker.py@@ -1,14 +1,32 @@import loggingimport os+from collections.abc import Iteratorfrom typing import Anyfrom docker.errors import APIError, ImageNotFoundfrom docker.models.containers import Containerfrom unstract.worker.constants import Env+from unstract.worker.utils import Utilsfrom docker import DockerClient, from_env-from .interface import ContainerClientInterface+from .interface import ContainerClientInterface, ContainerInterface+++class DockerContainer(ContainerInterface):+ def __init__(self, container: Container) -> None:+ self.container: Container = container++ def logs(self, follow=False) -> Iterator[str]:+ return self.container.logs(stream=True, follow=follow)++ def cleanup(self) -> None:+ if not self.container or not Utils.remove_container_on_exit():+ return+ try:+ self.container.remove(force=True)+ except Exception as remove_error:+ self.logger.error(f"Failed to remove docker container: {remove_error}")class Client(ContainerClientInterface):@@ -67,6 +85,32 @@ class Client(ContainerClientInterface):f"Internal service error occured while authentication: {exc}")+ def __image_exists(self, image_name_with_tag: str) -> bool:+ """Check if the container image exists in system.++ Args:+ image_name_with_tag (str): The image name with tag.++ Returns:+ bool: True if the image exists, False otherwise.+ """++ try:+ # Attempt to get the image information+ self.client.images.get(image_name_with_tag)+ self.logger.info(+ f"Image '{image_name_with_tag}' found in the local system."+ )+ return True+ except ImageNotFound: # type: ignore[attr-defined]+ self.logger.info(+ f"Image '{image_name_with_tag}' not found in the local system."+ )+ return False+ except APIError as e: # type: ignore[attr-defined]+ self.logger.error(f"An API error occurred: {e}")+ return False+def get_image(self) -> str:"""Will check if image exists locally and pulls the image using`self.image_name` and `self.image_tag` if necessary.@@ -103,31 +147,5 @@ class Client(ContainerClientInterface):return image_name_with_tag- def __image_exists(self, image_name_with_tag: str) -> bool:- """Check if the container image exists in system.-- Args:- image_name_with_tag (str): The image name with tag.-- Returns:- bool: True if the image exists, False otherwise.- """-- try:- # Attempt to get the image information- self.client.images.get(image_name_with_tag)- self.logger.info(- f"Image '{image_name_with_tag}' found in the local system."- )- return True- except ImageNotFound: # type: ignore[attr-defined]- self.logger.info(- f"Image '{image_name_with_tag}' not found in the local system."- )- return False- except APIError as e: # type: ignore[attr-defined]- self.logger.error(f"An API error occurred: {e}")- return False-- def run_container(self, config: dict[Any, Any]) -> Container:- return self.client.containers.run(**config)+ def run_container(self, config: dict[Any, Any]) -> Any:+ return DockerContainer(self.client.containers.run(**config))diff --git a/worker/src/unstract/worker/clients/interface.py b/worker/src/unstract/worker/clients/interface.pyindex bea1f1b0a..9c4a19d68 100644--- a/worker/src/unstract/worker/clients/interface.py+++ b/worker/src/unstract/worker/clients/interface.py@@ -1,8 +1,27 @@import loggingfrom abc import ABC, abstractmethod+from collections.abc import Iteratorfrom typing import Any-from docker.models.containers import Container++class ContainerInterface(ABC):++ @abstractmethod+ def logs(self, follow=False) -> Iterator[str]:+ """Returns an iterator object of logs.++ Args:+ follow (bool, optional): Should the logs be followed. Defaults to False.++ Yields:+ Iterator[str]: Yields logs line by line.+ """+ pass++ @abstractmethod+ def cleanup(self) -> None:+ """Stops and removes the running container."""+ passclass ContainerClientInterface(ABC):@@ -12,8 +31,9 @@ class ContainerClientInterface(ABC):pass@abstractmethod- def run_container(self, config: dict[Any, Any]) -> Container:- """Method to run a container with provided config.+ def run_container(self, config: dict[Any, Any]) -> ContainerInterface:+ """Method to run a container with provided config. This method will run+ the container.Args:config (dict[Any, Any]): Configuration for container.diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.pyindex e94230e31..6b302eec2 100644--- a/worker/src/unstract/worker/worker.py+++ b/worker/src/unstract/worker/worker.py@@ -8,9 +8,11 @@ from typing import Any, Optionalfrom dotenv import load_dotenvfrom flask import Flaskfrom unstract.worker.clients.helper import get_container_client-from unstract.worker.clients.interface import ContainerClientInterface+from unstract.worker.clients.interface import (+ ContainerClientInterface,+ ContainerInterface,+)from unstract.worker.constants import Env, LogType, ToolKey-from unstract.worker.utils import Utilsfrom unstract.core.constants import LogFieldNamefrom unstract.core.pubsub_helper import LogPublisher@@ -36,13 +38,13 @@ class UnstractWorker:# Function to stream logsdef stream_logs(self,- container: Any,+ container: ContainerInterface,tool_instance_id: str,execution_id: str,organization_id: str,channel: Optional[str] = None,) -> None:- for line in container.logs(stream=True, follow=True):+ for line in container.logs(follow=True):log_message = line.decode().strip()self.logger.debug(f"[{self.container_name}] - {log_message}")self.process_log_message(@@ -141,8 +143,8 @@ class UnstractWorker:# Run the Docker containertry:- container = self.client.run_container(container_config)- for line in container.logs(stream=True):+ container: ContainerInterface = self.client.run_container(container_config)+ for line in container.logs():text = line.decode().strip()self.logger.info(text)if f'"type": "${command}"' in text:@@ -188,7 +190,7 @@ class UnstractWorker:# Add labels to container for logging with Loki.# This only required for observability.try:- labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, ""))+ labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, "[]"))container_config["labels"] = labelsexcept Exception as e:self.logger.info(f"Invalid labels for logging: {e}")@@ -199,7 +201,7 @@ class UnstractWorker:container = Noneresult = {"type": "RESULT", "result": None}try:- container = self.client.run_container(container_config)+ container: ContainerInterface = self.client.run_container(container_config)self.logger.info(f"Running Docker container: {self.container_name}")tool_instance_id = str(self.settings.get(ToolKey.TOOL_INSTANCE_ID))# Stream logs@@ -213,7 +215,8 @@ class UnstractWorker:except Exception as e:self.logger.error(f"Failed to run docker container: {e}")result = {"type": "RESULT", "result": None, "error": str(e)}- self._cleanup_container(container)+ if container:+ container.cleanup()return resultdef get_container_run_config(self) -> dict[str, Any]:@@ -248,12 +251,3 @@ class UnstractWorker:}],}-- def _cleanup_container(self, container: Any) -> None:- # NOTE: This method might need to be- if not container or not Utils.remove_container_on_exit():- return- try:- container.remove(force=True)- except Exception as remove_error:- self.logger.error(f"Failed to remove docker container: {remove_error}")