LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Refactored container config and some fixes in run command API (#389)
Ritwik G2e1c1a783783
diff --git a/docker/dockerfiles/worker.Dockerfile b/docker/dockerfiles/worker.Dockerfileindex 2727e669d..58f86454a 100644--- a/docker/dockerfiles/worker.Dockerfile+++ b/docker/dockerfiles/worker.Dockerfile@@ -34,6 +34,8 @@ RUN set -e; \\pdm sync --prod --no-editable; \\+ [ -f requirements.txt ] && pip install -r requirements.txt || { echo "requirements.txt does not exist";}; \+ \# REF: https://docs.gunicorn.org/en/stable/deploy.html#using-virtualenvpip install --no-cache-dir gunicorn gevent;diff --git a/worker/.gitignore b/worker/.gitignoreindex 4b3ab3003..84321f311 100644--- a/worker/.gitignore+++ b/worker/.gitignore@@ -160,3 +160,9 @@ cython_debug/# and can be added to the global gitignore or merged into this file. For a more nuclear# option (not recommended) you can uncomment the following to ignore the entire idea folder.#.idea/++src/unstract/worker/clients/+!src/unstract/worker/clients/__init__.py+!src/unstract/worker/clients/interface.py+!src/unstract/worker/clients/helper.py+!src/unstract/worker/clients/docker.pydiff --git a/worker/src/unstract/worker/clients/docker.py b/worker/src/unstract/worker/clients/docker.pyindex c6fd447b6..c741be947 100644--- a/worker/src/unstract/worker/clients/docker.py+++ b/worker/src/unstract/worker/clients/docker.py@@ -1,7 +1,7 @@import loggingimport osfrom collections.abc import Iterator-from typing import Any+from typing import Any, Optionalfrom docker.errors import APIError, ImageNotFoundfrom docker.models.containers import Container@@ -10,6 +10,7 @@ from unstract.worker.utils import Utilsfrom docker import DockerClient, from_env+from .helper import normalize_container_namefrom .interface import ContainerClientInterface, ContainerInterface@@ -17,8 +18,13 @@ class DockerContainer(ContainerInterface):def __init__(self, container: Container) -> None:self.container: Container = container- def logs(self, follow=False) -> Iterator[str]:- return self.container.logs(stream=True, follow=follow)+ @property+ def name(self):+ return self.container.name++ def logs(self, follow=True) -> Iterator[str]:+ for line in self.container.logs(stream=True, follow=follow):+ yield line.decode().strip()def cleanup(self) -> None:if not self.container or not Utils.remove_container_on_exit():@@ -147,5 +153,45 @@ class Client(ContainerClientInterface):return image_name_with_tag+ def get_container_run_config(+ self,+ command: list[str],+ organization_id: str,+ workflow_id: str,+ execution_id: str,+ envs: Optional[dict[str, Any]] = None,+ auto_remove: bool = False,+ ) -> dict[str, Any]:+ if envs is None:+ envs = {}+ mounts = []+ if organization_id and workflow_id and execution_id:+ mounts.append(+ {+ "type": "bind",+ "source": os.path.join(+ os.getenv(Env.WORKFLOW_DATA_DIR, ""),+ organization_id,+ workflow_id,+ execution_id,+ ),+ "target": os.getenv(Env.TOOL_DATA_DIR, "/data"),+ }+ )+ return {+ "name": normalize_container_name(self.image_name),+ "image": self.get_image(),+ "command": command,+ "detach": True,+ "stream": True,+ "auto_remove": auto_remove,+ "environment": envs,+ "stderr": True,+ "stdout": True,+ "network": os.getenv(Env.TOOL_CONTAINER_NETWORK, ""),+ "mounts": mounts,+ }+def run_container(self, config: dict[Any, Any]) -> Any:+ self.logger.info(f"Docker config: {config}")return DockerContainer(self.client.containers.run(**config))diff --git a/worker/src/unstract/worker/clients/helper.py b/worker/src/unstract/worker/clients/helper.pyindex db7bf8bd9..87b6d46a5 100644--- a/worker/src/unstract/worker/clients/helper.py+++ b/worker/src/unstract/worker/clients/helper.py@@ -1,4 +1,5 @@import os+import uuidfrom importlib import import_modulefrom .interface import ContainerClientInterface@@ -6,4 +7,9 @@ from .interface import ContainerClientInterfacedef get_container_client() -> ContainerClientInterface:client_path = os.getenv("CONTAINER_CLIENT_PATH", "unstract.worker.clients.docker")+ print("Loading the container client from path:", client_path)return import_module(client_path).Client+++def normalize_container_name(name: str) -> str:+ return name.replace("/", "-") + "-" + str(uuid.uuid4())diff --git a/worker/src/unstract/worker/clients/interface.py b/worker/src/unstract/worker/clients/interface.pyindex 9c4a19d68..fa731e247 100644--- a/worker/src/unstract/worker/clients/interface.py+++ b/worker/src/unstract/worker/clients/interface.py@@ -1,13 +1,18 @@import loggingfrom abc import ABC, abstractmethodfrom collections.abc import Iterator-from typing import Any+from typing import Any, Optionalclass ContainerInterface(ABC):+ @property+ @abstractmethod+ def name(self):+ """The name of the container."""+ pass@abstractmethod- def logs(self, follow=False) -> Iterator[str]:+ def logs(self, follow=True) -> Iterator[str]:"""Returns an iterator object of logs.Args:@@ -52,3 +57,20 @@ class ContainerClientInterface(ABC):str: full image name with tag."""pass++ @abstractmethod+ def get_container_run_config(+ self,+ command: list[str],+ organization_id: str,+ workflow_id: str,+ execution_id: str,+ envs: Optional[dict[str, Any]] = None,+ auto_remove: bool = False,+ ) -> dict[str, Any]:+ """Generate the configuration dictionary to run the container.++ Returns:+ dict[str, Any]: Configuration for running the container.+ """+ passdiff --git a/worker/src/unstract/worker/main.py b/worker/src/unstract/worker/main.pyindex 36858b714..f2e2915d4 100644--- a/worker/src/unstract/worker/main.py+++ b/worker/src/unstract/worker/main.py@@ -32,7 +32,7 @@ def run_container() -> Optional[Any]:envs = data["envs"]messaging_channel = data["messaging_channel"]- worker = UnstractWorker(image_name, image_tag, app=app)+ worker = UnstractWorker(image_name, image_tag, app)result = worker.run_container(organization_id=organization_id,workflow_id=workflow_id,@@ -59,7 +59,7 @@ def run_command(command: str) -> Optional[Any]:abort(404)image_name = request.args.get("image_name")image_tag = request.args.get("image_tag")- worker = UnstractWorker(image_name, image_tag)+ worker = UnstractWorker(image_name, image_tag, app)return worker.run_command(command)diff --git a/worker/src/unstract/worker/utils.py b/worker/src/unstract/worker/utils.pyindex f68f76b62..267a59bd0 100644--- a/worker/src/unstract/worker/utils.py+++ b/worker/src/unstract/worker/utils.py@@ -33,7 +33,7 @@ class Utils:LogLevel"""try:- log_level_str = os.environ.get(Env.LOG_LEVEL, "").upper()+ log_level_str = os.getenv(Env.LOG_LEVEL, "").upper()return LogLevel[log_level_str.upper()]except KeyError:return LogLevel.INFO@@ -45,4 +45,4 @@ class Utils:Returns:bool"""- return Utils.str_to_bool(os.environ.get(Env.REMOVE_CONTAINER_ON_EXIT, "true"))+ return Utils.str_to_bool(os.getenv(Env.REMOVE_CONTAINER_ON_EXIT, "true"))diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.pyindex 6b302eec2..5dd46a658 100644--- a/worker/src/unstract/worker/worker.py+++ b/worker/src/unstract/worker/worker.py@@ -1,7 +1,6 @@import astimport jsonimport os-import uuidfrom datetime import datetime, timezonefrom typing import Any, Optional@@ -18,6 +17,8 @@ from unstract.core.constants import LogFieldNamefrom unstract.core.pubsub_helper import LogPublisherload_dotenv()+# Loads the container clinet class.+client_class = get_container_client()class UnstractWorker:@@ -26,14 +27,9 @@ class UnstractWorker:# If no image_tag is provided will assume the `latest` tagself.image_tag = image_tag or "latest"self.logger = app.logger- client_class = get_container_client()self.client: ContainerClientInterface = client_class(self.image_name, self.image_tag, self.logger)- self.image = self.client.get_image()-- def normalize_container_name(self, name: str) -> str:- return name.replace("/", "-") + "-" + str(uuid.uuid4())# Function to stream logsdef stream_logs(@@ -46,7 +42,7 @@ class UnstractWorker:) -> None:for line in container.logs(follow=True):log_message = line.decode().strip()- self.logger.debug(f"[{self.container_name}] - {log_message}")+ self.logger.debug(f"[{container.name}] - {log_message}")self.process_log_message(log_message=log_message,tool_instance_id=tool_instance_id,@@ -128,29 +124,26 @@ class UnstractWorker:Optional[Any]: Response from container or None if error occures."""command = command.upper()- self.container_name = self.normalize_container_name(self.image_name)- container_config = {- "name": self.container_name,- "image": self.image,- "command": ["--command", command],- "detach": True,- "stream": True,- "auto_remove": True,- "stderr": True,- "stdout": True,- }- self.logger.info(f"Docker config: {container_config}")+ container_config = self.client.get_container_run_config(+ command=["--command", command],+ organization_id="",+ workflow_id="",+ execution_id="",+ auto_remove=True,+ )+ container = None# Run the Docker containertry:container: ContainerInterface = self.client.run_container(container_config)- for line in container.logs():- text = line.decode().strip()- self.logger.info(text)- if f'"type": "${command}"' in text:+ for text in container.logs(follow=True):+ self.logger.info(f"[{container.name}] - {text}")+ if f'"type": "{command}"' in text:return json.loads(text)except Exception as e:self.logger.error(f"Failed to run docker container: {e}")+ if container:+ container.cleanup()return Nonedef run_container(@@ -174,41 +167,44 @@ class UnstractWorker:Returns:Optional[Any]: _description_"""- tool_data_dir = os.environ.get(Env.TOOL_DATA_DIR, "/data")+ tool_data_dir = os.getenv(Env.TOOL_DATA_DIR, "/data")envs[Env.TOOL_DATA_DIR] = tool_data_dir- self.organization_id = organization_id- self.workflow_id = workflow_id- self.execution_id = execution_id- self.params: dict[str, Any] = {}- self.settings = settings- self.envs = envs- self.messaging_channel = messaging_channel- self.container_name = (- self.normalize_container_name(self.image_name) + "-" + self.workflow_id+ container_config = self.client.get_container_run_config(+ command=[+ "--command",+ "RUN",+ "--settings",+ json.dumps(settings),+ "--log-level",+ "DEBUG",+ ],+ organization_id=organization_id,+ workflow_id=workflow_id,+ execution_id=execution_id,+ envs=envs,)- container_config = self.get_container_run_config()# Add labels to container for logging with Loki.# This only required for observability.try:- labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, "[]"))+ labels = ast.literal_eval(os.getenv(Env.TOOL_CONTAINER_LABELS, "[]"))container_config["labels"] = labelsexcept Exception as e:self.logger.info(f"Invalid labels for logging: {e}")- self.logger.info(f"Docker config: {container_config}")-# Run the Docker containercontainer = Noneresult = {"type": "RESULT", "result": None}try:container: ContainerInterface = self.client.run_container(container_config)- self.logger.info(f"Running Docker container: {self.container_name}")- tool_instance_id = str(self.settings.get(ToolKey.TOOL_INSTANCE_ID))+ self.logger.info(+ f"Running Docker container: {container_config.get('name')}"+ )+ tool_instance_id = str(settings.get(ToolKey.TOOL_INSTANCE_ID))# Stream logsself.stream_logs(container=container,tool_instance_id=tool_instance_id,- channel=self.messaging_channel,+ channel=messaging_channel,execution_id=execution_id,organization_id=organization_id,)@@ -218,36 +214,3 @@ class UnstractWorker:if container:container.cleanup()return result-- def get_container_run_config(self) -> dict[str, Any]:- return {- "name": self.container_name,- "image": self.image,- "command": [- "--command",- "RUN",- "--settings",- json.dumps(self.settings),- "--log-level",- "DEBUG",- ],- "detach": True,- "stream": True,- "auto_remove": False,- "environment": self.envs,- "stderr": True,- "stdout": True,- "network": os.environ.get(Env.TOOL_CONTAINER_NETWORK, ""),- "mounts": [- {- "type": "bind",- "source": os.path.join(- os.environ.get(Env.WORKFLOW_DATA_DIR, ""),- self.organization_id,- self.workflow_id,- self.execution_id,- ),- "target": os.environ.get(Env.TOOL_DATA_DIR, "/data"),- }- ],- }