LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[FIX] Unblock OSS first-time setup — sample envs, host-gateway, ollama hint (#2107)
Chandrasekharan M550f9e5fbb55
diff --git a/backend/sample.env b/backend/sample.envindex 377016fde..743a31de8 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -64,7 +64,9 @@ SESSION_EXPIRATION_TIME_IN_SECOND=7200WEB_APP_ORIGIN_URL="http://frontend.unstract.localhost"# API keys for trusted services-INTERNAL_SERVICE_API_KEY=+# Workers send this as X-API-Key on internal calls to backend.+# Must match workers/sample.env INTERNAL_SERVICE_API_KEY; rotate in both for prod.+INTERNAL_SERVICE_API_KEY=dev-internal-key-123# Unstract Core envsBUILTIN_FUNCTIONS_API_KEY=@@ -199,7 +201,8 @@ API_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_ur#Remote storage related envsPERMANENT_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'-REMOTE_PROMPT_STUDIO_FILE_PATH="unstract/prompt-studio-data"+TEMPORARY_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'+REMOTE_PROMPT_STUDIO_FILE_PATH=unstract/prompt-studio-data# Storage Provider for Tool registryTOOL_REGISTRY_STORAGE_CREDENTIALS='{"provider":"local"}'diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yamlindex b7b49a761..5884b8899 100644--- a/docker/docker-compose.yaml+++ b/docker/docker-compose.yaml@@ -2,9 +2,21 @@ name: ${COMPOSE_PROJECT_NAME:-docker}include:- docker-compose-dev-essentials.yaml+# Reusable host-gateway mapping so containers can reach services on the host+# (e.g. host-installed Ollama at http://host.docker.internal:11434).+# NOTE: YAML merge (<<:) merges mappings shallowly and does NOT concatenate+# lists. If a service ever needs additional extra_hosts entries alongside+# this one, inline ALL entries under that service (including+# host.docker.internal:host-gateway) instead of using the anchor — a sibling+# extra_hosts key would silently shadow the anchor's list.+x-host-gateway: &host_gateway+ extra_hosts:+ - "host.docker.internal:host-gateway"+services:# Backend servicebackend:+ <<: *host_gatewayimage: unstract/backend:${VERSION}container_name: unstract-backendrestart: unless-stopped@@ -34,12 +46,10 @@ services:- traefik.enable=true- traefik.http.routers.backend.rule=Host(`frontend.unstract.localhost`) && (PathPrefix(`/api/v1`) || PathPrefix(`/deployment`) || PathPrefix(`/public`))- traefik.http.services.backend.loadbalancer.server.port=8000- extra_hosts:- # "host-gateway" is a special string that translates to host docker0 i/f IP.- - "host.docker.internal:host-gateway"# Celery worker for dashboard metrics processingworker-metrics:+ <<: *host_gatewayimage: unstract/backend:${VERSION}container_name: unstract-worker-metricsrestart: unless-stopped@@ -61,6 +71,7 @@ services:# Processes post-execution callbacks via InternalAPIClient (no Django).# Handles: ide_index_complete/error, ide_prompt_complete/error.worker-ide-callback:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-ide-callbackrestart: unless-stopped@@ -82,6 +93,7 @@ services:# Celery Flowercelery-flower:+ <<: *host_gatewayimage: unstract/backend:${VERSION}container_name: unstract-celery-flowerrestart: unless-stopped@@ -105,6 +117,7 @@ services:# Celery Beatcelery-beat:+ <<: *host_gatewayimage: unstract/backend:${VERSION}container_name: unstract-celery-beatrestart: unless-stopped@@ -152,6 +165,7 @@ services:- traefik.enable=falseprompt-service:+ <<: *host_gatewayimage: unstract/prompt-service:${VERSION}container_name: unstract-prompt-servicerestart: unless-stopped@@ -166,9 +180,6 @@ services:- ../prompt-service/.envlabels:- traefik.enable=false- extra_hosts:- # "host-gateway" is a special string that translates to host docker0 i/f IP.- - "host.docker.internal:host-gateway"x2text-service:image: unstract/x2text-service:${VERSION}@@ -184,6 +195,7 @@ services:- traefik.enable=falserunner:+ <<: *host_gatewayimage: unstract/runner:${VERSION}container_name: unstract-runnerrestart: unless-stopped@@ -206,6 +218,7 @@ services:# ====================================================================worker-api-deployment-v2:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-api-deployment-v2restart: unless-stopped@@ -238,6 +251,7 @@ services:- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_configworker-callback-v2:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-callback-v2restart: unless-stopped@@ -264,6 +278,7 @@ services:- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_configworker-file-processing-v2:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-file-processing-v2restart: unless-stopped@@ -316,6 +331,7 @@ services:- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_configworker-general-v2:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-general-v2restart: unless-stopped@@ -343,6 +359,7 @@ services:- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_configworker-notification-v2:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-notification-v2restart: unless-stopped@@ -391,6 +408,7 @@ services:- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_configworker-log-consumer-v2:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-log-consumer-v2restart: unless-stopped@@ -440,6 +458,7 @@ services:- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_configworker-log-history-scheduler-v2:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-log-history-scheduler-v2restart: unless-stopped@@ -463,6 +482,7 @@ services:- traefik.enable=falseworker-scheduler-v2:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-scheduler-v2restart: unless-stopped@@ -507,6 +527,7 @@ services:- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_configworker-executor-v2:+ <<: *host_gatewayimage: unstract/worker-unified:${VERSION}container_name: unstract-worker-executor-v2restart: unless-stoppeddiff --git a/run-platform.sh b/run-platform.shindex a2b793a13..40181806f 100755--- a/run-platform.sh+++ b/run-platform.sh@@ -33,6 +33,28 @@ check_dependencies() {echo "$red_text""docker not found. Exiting.""$default_text"exit 1fi+ if ! docker info >/dev/null 2>&1; then+ echo "$red_text""Cannot connect to the Docker daemon.""$default_text"+ case "$(uname -s)" in+ Linux*)+ echo " On Linux (daemon access via the 'docker' group):"+ echo " - Check group membership: getent group docker"+ echo " - Add your user to it: sudo usermod -aG docker \$USER"+ echo " - Activate in current shell: newgrp docker"+ echo " - For new shells, a full desktop logout (not just terminal close) is required."+ ;;+ Darwin*)+ echo " On macOS: ensure Docker Desktop is running (whale icon in the menu bar)."+ ;;+ MINGW*|MSYS*|CYGWIN*)+ echo " On Windows: ensure Docker Desktop is running and WSL integration is enabled if applicable."+ ;;+ *)+ echo " Ensure the Docker daemon is running and your user can reach its socket."+ ;;+ esac+ exit 1+ fi# For 'docker compose' vs 'docker-compose', see https://stackoverflow.com/a/66526176.docker compose >/dev/null 2>&1if [ $? -eq 0 ]; thendiff --git a/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/static/ollama.json b/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/static/ollama.jsonindex c241e5e71..e5434a095 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/static/ollama.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/static/ollama.json@@ -23,7 +23,7 @@"type": "string","title": "Base URL","default": "",- "description": "Provide the base URL where Ollama server is running. Example: `http://docker.host.internal:11434` or `http://localhost:11434`"+ "description": "Provide the base URL where Ollama server is running. Example: `http://host.docker.internal:11434` or `http://localhost:11434`"},"max_retries": {"type": "number",diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.jsonindex 3c8a4a5f1..8f2b9db7a 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/ollama.json@@ -23,7 +23,7 @@"type": "string","title": "Base URL","default": "",- "description": "Provide the base URL where Ollama server is running. Example: http://docker.host.internal:11434 or http://localhost:11434"+ "description": "Provide the base URL where Ollama server is running. Example: http://host.docker.internal:11434 or http://localhost:11434"},"max_tokens": {"type": "number",diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/env_helper.py b/unstract/sdk1/src/unstract/sdk1/file_storage/env_helper.pyindex 22b521094..830e7a1c3 100644--- a/unstract/sdk1/src/unstract/sdk1/file_storage/env_helper.py+++ b/unstract/sdk1/src/unstract/sdk1/file_storage/env_helper.py@@ -31,22 +31,42 @@ class EnvHelper:FileStorage: FIleStorage instance initialised using the providerand credentials configured in the env"""+ raw = os.environ.get(env_name)+ if not raw:+ raise FileStorageError(+ f"Required env var '{env_name}' is unset or empty. "+ f"Expected JSON config of the form: {EnvHelper.ENV_CONFIG_FORMAT}"+ )+ try:+ file_storage_creds = json.loads(raw)+ except json.JSONDecodeError as e:+ raise FileStorageError(+ f"Env var '{env_name}' is not valid JSON: {e}. "+ f"Expected: {EnvHelper.ENV_CONFIG_FORMAT}"+ ) from e+ if not isinstance(file_storage_creds, dict):+ raise FileStorageError(+ f"Env var '{env_name}' must be a JSON object. "+ f"Expected: {EnvHelper.ENV_CONFIG_FORMAT}"+ )try:- file_storage_creds = json.loads(os.environ.get(env_name, ""))provider = FileStorageProvider(file_storage_creds[CredentialKeyword.PROVIDER])- credentials = file_storage_creds.get(CredentialKeyword.CREDENTIALS, {})- if storage_type == StorageType.PERMANENT:- file_storage = PermanentFileStorage(provider=provider, **credentials)- elif storage_type == StorageType.SHARED_TEMPORARY:- file_storage = SharedTemporaryFileStorage(- provider=provider, **credentials- )- else:- raise NotImplementedError()- return file_storage- except KeyError as e:- logger.error(f"Required credentials are missing in the env: {str(e)}")+ except (KeyError, ValueError) as e:+ logger.error(f"Invalid storage configuration in env: {str(e)}")logger.error(f"The configuration format is {EnvHelper.ENV_CONFIG_FORMAT}")- raise e- except FileStorageError as e:- raise e+ raise FileStorageError(+ f"Invalid storage configuration in env var '{env_name}': {e}. "+ f"Expected: {EnvHelper.ENV_CONFIG_FORMAT}"+ ) from e+ credentials = file_storage_creds.get(CredentialKeyword.CREDENTIALS, {})+ if not isinstance(credentials, dict):+ raise FileStorageError(+ f"Env var '{env_name}' field '{CredentialKeyword.CREDENTIALS}' "+ f"must be a JSON object. Expected: {EnvHelper.ENV_CONFIG_FORMAT}"+ )+ if storage_type == StorageType.PERMANENT:+ return PermanentFileStorage(provider=provider, **credentials)+ elif storage_type == StorageType.SHARED_TEMPORARY:+ return SharedTemporaryFileStorage(provider=provider, **credentials)+ else:+ raise NotImplementedError()diff --git a/workers/sample.env b/workers/sample.envindex 4764fb0c5..7d9a70721 100644--- a/workers/sample.env+++ b/workers/sample.env@@ -316,6 +316,12 @@ UNSTRACT_RUNNER_API_BACKOFF_FACTOR=3WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'API_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'+# Remote storage for Prompt Studio / IDE flows. Must match backend/sample.env.+# Required by executor and ide-callback workers; missing/empty values raise FileStorageError in EnvHelper.get_storage().+PERMANENT_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'+TEMPORARY_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'+REMOTE_PROMPT_STUDIO_FILE_PATH=unstract/prompt-studio-data+# File Execution ConfigurationWORKFLOW_EXECUTION_DIR_PREFIX=unstract/executionAPI_EXECUTION_DIR_PREFIX=unstract/api