Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Workflow file path for table extractor (#616)

Deepak Kbeeba6a08d51
diff --git a/worker/sample.env b/worker/sample.env
index e5edf7b17..f0ee9e337 100644
--- a/worker/sample.env
+++ b/worker/sample.env
@@ -19,3 +19,5 @@ REMOVE_CONTAINER_ON_EXIT=True
# Client module path of the container engine to be used.
CONTAINER_CLIENT_PATH=unstract.worker.clients.docker
+
+EXECUTION_RUN_DATA_FOLDER_PREFIX="/app/workflow_data"
diff --git a/worker/src/unstract/worker/clients/docker.py b/worker/src/unstract/worker/clients/docker.py
index 2391d61d7..8e8817bb7 100644
--- a/worker/src/unstract/worker/clients/docker.py
+++ b/worker/src/unstract/worker/clients/docker.py
@@ -181,7 +181,12 @@ class Client(ContainerClientInterface):
"target": os.getenv(Env.TOOL_DATA_DIR, "/data"),
}
)
- envs[Env.EXECUTION_RUN_DATA_FOLDER] = source_path
+ envs[Env.EXECUTION_RUN_DATA_FOLDER] = os.path.join(
+ os.getenv(Env.EXECUTION_RUN_DATA_FOLDER_PREFIX, ""),
+ organization_id,
+ workflow_id,
+ execution_id,
+ )
return {
"name": ContainerClientHelper.normalize_container_name(self.image_name),
"image": self.get_image(),
diff --git a/worker/src/unstract/worker/clients/test_docker.py b/worker/src/unstract/worker/clients/test_docker.py
index 4d2deead5..3f6fe2290 100644
--- a/worker/src/unstract/worker/clients/test_docker.py
+++ b/worker/src/unstract/worker/clients/test_docker.py
@@ -109,6 +109,7 @@ def test_get_image(docker_client, mocker):
def test_get_container_run_config(docker_client, mocker):
"""Test the get_container_run_config method."""
os.environ[Env.WORKFLOW_DATA_DIR] = "/source"
+ os.environ[Env.EXECUTION_RUN_DATA_FOLDER_PREFIX] = "/app/workflow_data"
command = ["echo", "hello"]
organization_id = "org123"
workflow_id = "wf123"
@@ -134,7 +135,7 @@ def test_get_container_run_config(docker_client, mocker):
assert config["command"] == ["echo", "hello"]
assert config["environment"] == {
"KEY": "VALUE",
- "EXECUTION_RUN_DATA_FOLDER": "/source/org123/wf123/ex123",
+ "EXECUTION_RUN_DATA_FOLDER": ("/app/workflow_data/org123/wf123/ex123"),
}
assert config["mounts"] == [
{
diff --git a/worker/src/unstract/worker/constants.py b/worker/src/unstract/worker/constants.py
index 4481d96a9..cfa8abcbf 100644
--- a/worker/src/unstract/worker/constants.py
+++ b/worker/src/unstract/worker/constants.py
@@ -22,6 +22,7 @@ class Env:
TOOL_CONTAINER_LABELS = "TOOL_CONTAINER_LABELS"
WORKFLOW_DATA_DIR = "WORKFLOW_DATA_DIR"
EXECUTION_RUN_DATA_FOLDER = "EXECUTION_RUN_DATA_FOLDER"
+ EXECUTION_RUN_DATA_FOLDER_PREFIX = "EXECUTION_RUN_DATA_FOLDER_PREFIX"
TOOL_DATA_DIR = "TOOL_DATA_DIR"
PRIVATE_REGISTRY_CREDENTIAL_PATH = "PRIVATE_REGISTRY_CREDENTIAL_PATH"
PRIVATE_REGISTRY_USERNAME = "PRIVATE_REGISTRY_USERNAME"