LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Run script tweaks (#120)
Hari John Kuriakosee8668a073bd5
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex f50036369..c67d62e4a 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -144,43 +144,6 @@ repos:entry: pyupgrade --py38-plus --keep-runtime-typingtypes:- python- - repo: https://github.com/pre-commit/mirrors-mypy- rev: v1.9.0- hooks:- - id: mypy- language: system- entry: mypy .- pass_filenames: false- # IMPORTANT!- # Keep args same as tool.mypy section in pyproject.toml- args:- [- --allow-subclassing-any,- --allow-untyped-decorators,- --check-untyped-defs,- # To use Unpack for typing kwargs- --enable-incomplete-feature=Unpack,- --exclude, ".*migrations/.*.py",- --exclude, "backend/prompt/.*",- --exclude, "document-service/.*",- --exclude, "unstract/connectors/tests/.*",- --exclude, "unstract/core/.*",- --exclude, "unstract/flags/src/unstract/flags/.*",- --exclude, "__pypackages__/.*",- --follow-imports, "silent",- --ignore-missing-imports,- --implicit-reexport,- --pretty,- --python-version=3.9,- --show-column-numbers,- --show-error-codes,- --strict,- --warn-redundant-casts,- --warn-return-any,- --warn-unreachable,- --warn-unused-configs,- --warn-unused-ignores,- ]- repo: https://github.com/igorshubovych/markdownlint-clirev: v0.39.0hooks:@@ -194,11 +157,14 @@ repos:- id: pdm-lock-check- repo: localhooks:- # ! IMPORTANT !- # Requires unstract-db to be running and accessible+ - id: run-mypy+ name: Run mypy+ entry: sh -c 'pdm run mypy .'+ language: system+ pass_filenames: false- id: check-django-migrationsname: Check django migrations- entry: sh -c 'pdm run `find . -name "manage.py" -not -path "*/.venv/*"` makemigrations --check --dry-run --no-input'+ entry: sh -c 'pdm run docker/scripts/check_django_migrations.sh'language: systemtypes: [python] # hook only runs if a python file is stagedpass_filenames: falsediff --git a/dev-env-cli.sh b/dev-env-cli.shnew file mode 100755index 000000000..47dc5340f--- /dev/null+++ b/dev-env-cli.sh@@ -0,0 +1,317 @@+#!/usr/bin/env bash++set -o nounset # exit if a variable is not set+set -o errexit # exit for any command failure"++# text color escape codes (\033 == \e but OSX doesn't respect the \e)+blue_text='\033[94m'+green_text='\033[32m'+red_text='\033[31m'+default_text='\033[39m'++# set -x/xtrace uses PS4 for more info+PS4="$blue_text""${0}:${LINENO}: ""$default_text"++debug() {+ if [ "$opt_verbose" = true ]; then+ echo $1+ fi+}++check_dependencies() {+ if ! command -v python3 &> /dev/null; then+ echo "$red_text""python3 not found. Exiting.""$default_text"+ exit 1+ fi+ if ! command -v docker compose &> /dev/null; then+ echo "$red_text""docker not found. Exiting.""$default_text"+ exit 1+ fi+ # For 'docker compose' vs 'docker-compose', see https://stackoverflow.com/a/66526176.+ if command -v docker compose &> /dev/null; then+ docker_compose_cmd="docker compose"+ elif command -v docker-compose &> /dev/null; then+ docker_compose_cmd="docker-compose"+ else+ echo "$red_text""Both 'docker compose' and 'docker-compose' not found. Exiting.""$default_text"+ exit 1+ fi+ if ! command -v pdm &> /dev/null; then+ echo "$red_text""pdm not found. Exiting.""$default_text"+ exit 1+ fi+}++display_banner() {+ # Make sure the console is huge+ if test $(tput cols) -ge 64; then+ echo " █████ █████"+ echo "░░███ ░░███ "+ echo " ░███ ░███ "+ echo " ░███ ░███ "+ echo " ░███ ░███ "+ echo " ░███ ░███ "+ echo " ░░█████████ >UNSTRACT COMMUNITY EDITION"+ echo " ░░░░░░░░░ "+ echo ""+ sleep 1+ fi+}++display_help() {+ printf "Dev environment CLI for Unstract platform services\n"+ echo+ echo -e "Syntax: $0 [options] -s service"+ echo -e "Options:"+ echo -e " -h, --help Display help information"+ echo -e " -e, --setup-venv Setup venv (requires service)"+ echo -e " -a, --activate-venv Activate venv (requires service)"+ echo -e " -i, --install-deps Install dependencies in venv (requires service)"+ echo -e " -d, --destroy-venv Destroy venv (requires service)"+ echo -e " -p, --install-pre-commit-hook Install Git pre-commit hook"+ echo -e " -r, --run-pre-commit-hook Run Git pre-commit hook"+ echo -e " -f, --force Force operation"+ echo -e " -s, --service Service name"+ echo -e " -x, --trace Enables trace mode"+ echo -e " -V, --verbose Print verbose logs"+ echo -e ""+}++parse_args() {+ while [[ $# -gt 0 ]]; do+ arg="$1"+ case $arg in+ -h | --help)+ display_help+ exit+ ;;+ -e | --setup-venv)+ opt_setup_venv=true+ ;;+ -a | --activate-venv)+ opt_activate_venv=true+ ;;+ -i | --install-deps)+ opt_install_deps=true+ ;;+ -d | --destroy-venv)+ opt_destroy_venv=true+ ;;+ -p | --install-pre-commit-hook)+ opt_install_pre_commit_hook=true+ ;;+ -r | --run-pre-commit-hook)+ opt_run_pre_commit_hook=true+ ;;+ -f | --force)+ opt_force="--force"+ ;;+ -s | --service)+ if [ -z "${2-}" ]; then+ echo "No service specified."+ echo+ display_help+ exit+ else+ opt_service="$2"+ fi+ shift+ ;;+ -x | --trace)+ set -o xtrace # display every line before execution; enables PS4+ ;;+ -V | --verbose)+ opt_verbose=true+ ;;+ *)+ echo "'$1' is not a known command."+ echo+ display_help+ exit+ ;;+ esac+ shift+ done++ if [ "$opt_install_pre_commit_hook" = false ] && [ "$opt_run_pre_commit_hook" = false ]; then+ if [ "$opt_service" = "" ]; then+ echo "No service specified."+ echo+ display_help+ exit+ fi+ ret=$(echo ${services[@]} | grep -ow "$opt_service" | wc -w)+ if [ $ret -eq 0 ]; then+ echo "Unknown service '$opt_service'."+ echo+ display_help+ exit+ fi+ fi+ if [ "$opt_setup_venv" = false ] && [ "$opt_activate_venv" = false ] &&+ [ "$opt_install_deps" = false ] && [ "$opt_destroy_venv" = false ] &&+ [ "$opt_install_pre_commit_hook" = false ] && [ "$opt_run_pre_commit_hook" = false ]; then+ echo "One of -e,-a,-i,-d,-p,-r options should be specified."+ echo+ display_help+ exit+ fi++ debug "OPTION setup venv: $opt_setup_venv"+ debug "OPTION activate venv: $opt_activate_venv"+ debug "OPTION install deps: $opt_install_deps"+ debug "OPTION destroy venv: $opt_destroy_venv"+ debug "OPTION install pre-commit hook: $opt_install_pre_commit_hook"+ debug "OPTION run pre-commit hook: $opt_run_pre_commit_hook"+ debug "OPTION service: $opt_service"+ debug "OPTION verbose: $opt_verbose"+}++setup_venv() {+ if [ "$opt_setup_venv" = false ]; then+ return+ fi++ pushd ${script_dir}/$opt_service 1>/dev/null++ if [ -e "package.json" ]; then+ echo -e "Nothing to do for ""$blue_text""$opt_service""$default_text"+ elif [ -e "pyproject.toml" ]; then+ echo -e "Setting up ""$blue_text""Python venv""$default_text"" for ""$blue_text""$opt_service""$default_text"+ pdm venv create -w virtualenv --with-pip $opt_force+ fi++ popd 1>/dev/null+}++activate_venv() {+ if [ "$opt_activate_venv" = false ]; then+ return+ fi++ pushd ${script_dir}/$opt_service 1>/dev/null++ if [ -e "package.json" ]; then+ echo -e "Nothing to do for ""$blue_text""$opt_service""$default_text"+ elif [ -e "pyproject.toml" ]; then+ if [ ! -e ".venv" ]; then+ echo -e "venv not found for ""$blue_text""$opt_service""$default_text"". Please run setup first."+ return+ fi+ echo -e "Run the following in a ""$blue_text""new terminal""$default_text"" to activate venv for ""$blue_text""$opt_service""$default_text"":"+ echo ""+ echo "cd ${script_dir}/$opt_service"+ echo "eval \"$(pdm venv activate in-project)\""+ fi++ popd 1>/dev/null+}++install_deps() {+ if [ "$opt_install_deps" = false ]; then+ return+ fi++ pushd ${script_dir}/$opt_service 1>/dev/null++ if [ -e "package.json" ]; then+ echo -e "Installing dependencies for ""$blue_text""$opt_service""$default_text"+ npm ci+ elif [ -e "pyproject.toml" ]; then+ if [ ! -e ".venv" ]; then+ echo -e "venv not found for ""$blue_text""$opt_service""$default_text"". Please run setup first."+ return+ fi+ echo -e "Installing dependencies in venv for ""$blue_text""$opt_service""$default_text"+ eval "$(pdm venv activate in-project)"+ pdm sync+ fi++ popd 1>/dev/null+}++destroy_venv() {+ if [ "$opt_destroy_venv" = false ]; then+ return+ fi++ pushd ${script_dir}/$opt_service 1>/dev/null++ if [ -e "package.json" ]; then+ echo -e "$blue_text""Nothing to do for $opt_service""$default_text"+ elif [ -e "pyproject.toml" ]; then+ if [ ! -e ".venv" ]; then+ echo -e "venv not found for ""$blue_text""$opt_service""$default_text"". Please run setup first."+ return+ fi+ echo -e "Destroying venv for ""$blue_text""$opt_service""$default_text"+ pdm venv remove in-project+ fi++ popd 1>/dev/null+}++install_pre_commit_hook() {+ if [ "$opt_install_pre_commit_hook" = false ]; then+ return+ fi++ pushd ${script_dir} 1>/dev/null++ echo -e "Installing ""$blue_text""Git pre-commit hook""$default_text"+ pdm venv create -w virtualenv --with-pip --force+ eval $(pdm venv activate in-project)+ pdm sync --dev -G lint -G hook-check-django-migrations+ pre-commit install++ popd 1>/dev/null+}++run_pre_commit_hook() {+ if [ "$opt_run_pre_commit_hook" = false ]; then+ return+ fi++ pushd ${script_dir} 1>/dev/null++ if [ ! -e ".venv" ]; then+ echo -e "$blue_text""Git pre-commit hook""$default_text"" not found. Please run install first."+ return+ fi+ eval "$(pdm venv activate in-project)"+ pre-commit run++ popd 1>/dev/null+}++#+# Run Unstract platform - BEGIN+#+check_dependencies++opt_setup_venv=false+opt_activate_venv=false+opt_install_deps=false+opt_destroy_venv=false+opt_install_pre_commit_hook=false+opt_run_pre_commit_hook=false+opt_force=""+opt_service=""+opt_verbose=false++script_dir=$(dirname "$(readlink -f "$BASH_SOURCE")")+# Extract service names from docker compose file.+services=($(VERSION="latest" $docker_compose_cmd -f $script_dir/docker/docker-compose.build.yaml config --services))++display_banner+parse_args $*++setup_venv+activate_venv+install_deps+destroy_venv+install_pre_commit_hook+run_pre_commit_hook+#+# Run Unstract platform - END+#diff --git a/docker/scripts/check_django_migrations.sh b/docker/scripts/check_django_migrations.shnew file mode 100755index 000000000..e2e004025--- /dev/null+++ b/docker/scripts/check_django_migrations.sh@@ -0,0 +1,49 @@+#! /bin/sh++script_dir=$(dirname "$(readlink -f "$BASH_SOURCE")")++cd $script_dir/../../backend++# Check Django default db connectivity.+# Use Python for cross-platform compatibility.+python3 << EOF+import os+import socket+import sys++import dotenv++dotenv.load_dotenv()++def _check_connectivity(host, port):+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:+ s.settimeout(2)+ errno = s.connect_ex((host, port))+ return errno++db_host = os.getenv("DB_HOST", "localhost")+db_port = int(os.getenv("DB_PORT", 5432))++errno = _check_connectivity(db_host, db_port)++if errno == 0:+ print("Django default db ok:", os.strerror(errno))+ sys.exit(0)+else:+ print("Django default db error:", os.strerror(errno))+ sys.exit(1)+EOF+if [ $? -ne 0 ]; then+ echo ""+ echo "Verify DB_HOST, DB_PORT settings for backend Django default db."+ echo "If db is running in a container, add DB_HOST value to /etc/hosts."+ echo ""+ exit 1+fi++# Check Django migrations.+python3 manage.py makemigrations --check --dry-run --no-input+# ! IMPORTANT !+# Above command does not return error exit code+# on network error when connecting to db.+exit 0diff --git a/docker/scripts/merge_env.py b/docker/scripts/merge_env.pynew file mode 100644index 000000000..5f72dc94e--- /dev/null+++ b/docker/scripts/merge_env.py@@ -0,0 +1,118 @@+#+# Usage:+# python merge_env.py [--dry-run] base_env_path target_env_path+#+# e.g.+# Dry run:+# python merge_env.py --dry-run sample.env .env+# Actual run:+# python merge_env.py sample.env .env+#+import sys++PREFERRED_BASE_ENV_KEYS = [+ "STRUCTURE_TOOL_IMAGE_URL",+ "STRUCTURE_TOOL_IMAGE_TAG",+]+++def _extract_kv_from_line(line: str) -> tuple[str, str]:+ parts = line.split("=", 1)+ if len(parts) != 2:+ raise ValueError(f"{line}")+ key, value = parts+ key = key.strip()+ value = value.strip()+ return key, value+++def _extract_from_env_file(file_path: str) -> dict[str, str]:+ env = {}++ with open(file_path) as file:+ for line in file:+ if not line.strip() or line.startswith("#"):+ continue++ key, value = _extract_kv_from_line(line)++ env[key.strip()] = value.strip()++ return env+++def _merge_to_env_file(+ base_env_file_path: str, target_env: dict[str, str] = {}+) -> str:+ """Generates file contents after merging input base env file path with+ target env.++ Args:+ base_env_path (string): Base env file path.+ target_env (dict, optional): Target env to use for merge.++ Returns:+ string: File contents after merge.+ """+ merged_contents = []++ with open(base_env_file_path) as file:+ for line in file:+ # Preserve location of empty lines and comments+ # for easy diff.+ if not line.strip() or line.startswith("#"):+ merged_contents.append(line)+ continue++ key, value = _extract_kv_from_line(line)++ # Preserve following keys at base env file path:+ # - preferred keys+ # - newly added keys+ # Everything else, take existing configured values+ # from target env.+ if key not in PREFERRED_BASE_ENV_KEYS and key in target_env:+ value = target_env.get(key, value)++ merged_contents.append(f"{key} = {value}\n")++ return "".join(merged_contents)+++def _save_merged_contents(+ file_path: str, file_contents: str, dry_run: bool = False+) -> None:+ if dry_run:+ print(f"===== merged:{file_path} =====")+ print(file_contents)+ print("-----------")+ else:+ with open(file_path, "w") as file:+ file.write(file_contents)+++def merge_env(+ base_env_file_path: str, target_env_file_path: str, dry_run: bool = False+) -> None:+ target_env = _extract_from_env_file(target_env_file_path)+ merged_contents = _merge_to_env_file(+ base_env_file_path, target_env=target_env+ )+ _save_merged_contents(+ target_env_file_path, merged_contents, dry_run=dry_run+ )+++if __name__ == "__main__":+ try:+ if len(sys.argv) == 4:+ dry_run = True if sys.argv[1] == "--dry-run" else False+ merge_env(sys.argv[2], sys.argv[3], dry_run=dry_run)+ else:+ merge_env(sys.argv[1], sys.argv[2])+ except IndexError:+ print(f"Invalid env paths for merge: {sys.argv}")+ sys.exit(1)+ except ValueError as e:+ print(f"Malformed env config: {str(e)}")+ sys.exit(1)diff --git a/pdm.lock b/pdm.lockindex 7e73037d7..f8e83626e 100644--- a/pdm.lock+++ b/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "hook-check-django-migrations", "lint"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:4856c6816ca68c8c298814a20efd155b658682d68431e88b2dfad61bdd45e658"+content_hash = "sha256:8f0ed82cbbbffa272795b941dfcec9ae77711e31c83c8b9047ccb1c15cd04186"[[package]]name = "absolufy-imports"@@ -321,7 +321,7 @@ files = [[[package]]name = "asgiref"-version = "3.8.0"+version = "3.8.1"requires_python = ">=3.8"summary = "ASGI specs, helper code, and adapters"groups = ["hook-check-django-migrations"]@@ -329,8 +329,8 @@ dependencies = ["typing-extensions>=4; python_version < \"3.11\"",]files = [- {file = "asgiref-3.8.0-py3-none-any.whl", hash = "sha256:30fc07797ad71a0abb8fe34aa03c8043308a8389abc7942d797ea9911540bc28"},- {file = "asgiref-3.8.0.tar.gz", hash = "sha256:ec75d9d0f04e2dbfedef1f20ee73a6594af80c333df47cdd31f37e6701f7c53a"},+ {file = "asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47"},+ {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"},][[package]]@@ -535,8 +535,8 @@ files = [[[package]]name = "black"-version = "23.3.0"-requires_python = ">=3.7"+version = "24.3.0"+requires_python = ">=3.8"summary = "The uncompromising code formatter."groups = ["lint"]dependencies = [@@ -546,26 +546,23 @@ dependencies = ["pathspec>=0.9.0","platformdirs>=2","tomli>=1.1.0; python_version < \"3.11\"",- "typing-extensions>=3.10.0.0; python_version < \"3.10\"",+ "typing-extensions>=4.0.1; python_version < \"3.11\"",]files = [- {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"},- {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"},- {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"},- {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"},- {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"},- {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"},- {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"},- {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"},- {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"},- {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"},- {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"},- {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"},- {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"},- {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"},- {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"},- {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"},- {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"},+ {file = "black-24.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7d5e026f8da0322b5662fa7a8e752b3fa2dac1c1cbc213c3d7ff9bdd0ab12395"},+ {file = "black-24.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9f50ea1132e2189d8dff0115ab75b65590a3e97de1e143795adb4ce317934995"},+ {file = "black-24.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7"},+ {file = "black-24.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:4be5bb28e090456adfc1255e03967fb67ca846a03be7aadf6249096100ee32d0"},+ {file = "black-24.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4f1373a7808a8f135b774039f61d59e4be7eb56b2513d3d2f02a8b9365b8a8a9"},+ {file = "black-24.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aadf7a02d947936ee418777e0247ea114f78aff0d0959461057cae8a04f20597"},+ {file = "black-24.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c02e4ea2ae09d16314d30912a58ada9a5c4fdfedf9512d23326128ac08ac3d"},+ {file = "black-24.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:bf21b7b230718a5f08bd32d5e4f1db7fc8788345c8aea1d155fc17852b3410f5"},+ {file = "black-24.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c45f8dff244b3c431b36e3224b6be4a127c6aca780853574c00faf99258041eb"},+ {file = "black-24.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6905238a754ceb7788a73f02b45637d820b2f5478b20fec82ea865e4f5d4d9f7"},+ {file = "black-24.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7de8d330763c66663661a1ffd432274a2f92f07feeddd89ffd085b5744f85e7"},+ {file = "black-24.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:7bb041dca0d784697af4646d3b62ba4a6b028276ae878e53f6b4f74ddd6db99f"},+ {file = "black-24.3.0-py3-none-any.whl", hash = "sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93"},+ {file = "black-24.3.0.tar.gz", hash = "sha256:a0c9c4a0771afc6919578cec71ce82a3e31e054904e7197deacbc9382671c41f"},][[package]]@@ -821,16 +818,16 @@ files = [[[package]]name = "click-didyoumean"-version = "0.3.0"-requires_python = ">=3.6.2,<4.0.0"+version = "0.3.1"+requires_python = ">=3.6.2"summary = "Enables git-like *did-you-mean* feature in click"groups = ["hook-check-django-migrations"]dependencies = ["click>=7",]files = [- {file = "click-didyoumean-0.3.0.tar.gz", hash = "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035"},- {file = "click_didyoumean-0.3.0-py3-none-any.whl", hash = "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667"},+ {file = "click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c"},+ {file = "click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463"},][[package]]@@ -1268,18 +1265,18 @@ files = [[[package]]name = "flake8"-version = "6.0.0"+version = "7.0.0"requires_python = ">=3.8.1"summary = "the modular source code checker: pep8 pyflakes and co"groups = ["lint"]dependencies = ["mccabe<0.8.0,>=0.7.0",- "pycodestyle<2.11.0,>=2.10.0",- "pyflakes<3.1.0,>=3.0.0",+ "pycodestyle<2.12.0,>=2.11.0",+ "pyflakes<3.3.0,>=3.2.0",]files = [- {file = "flake8-6.0.0-py2.py3-none-any.whl", hash = "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7"},- {file = "flake8-6.0.0.tar.gz", hash = "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181"},+ {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"},+ {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"},][[package]]@@ -1461,43 +1458,44 @@ files = [[[package]]name = "google-api-core"-version = "2.17.1"+version = "2.18.0"requires_python = ">=3.7"summary = "Google API client core library"groups = ["hook-check-django-migrations"]dependencies = ["google-auth<3.0.dev0,>=2.14.1","googleapis-common-protos<2.0.dev0,>=1.56.2",+ "proto-plus<2.0.0dev,>=1.22.3","protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5","requests<3.0.0.dev0,>=2.18.0",]files = [- {file = "google-api-core-2.17.1.tar.gz", hash = "sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95"},- {file = "google_api_core-2.17.1-py3-none-any.whl", hash = "sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e"},+ {file = "google-api-core-2.18.0.tar.gz", hash = "sha256:62d97417bfc674d6cef251e5c4d639a9655e00c45528c4364fbfebb478ce72a9"},+ {file = "google_api_core-2.18.0-py3-none-any.whl", hash = "sha256:5a63aa102e0049abe85b5b88cb9409234c1f70afcda21ce1e40b285b9629c1d6"},][[package]]name = "google-api-core"-version = "2.17.1"+version = "2.18.0"extras = ["grpc"]requires_python = ">=3.7"summary = "Google API client core library"groups = ["hook-check-django-migrations"]dependencies = [- "google-api-core==2.17.1",+ "google-api-core==2.18.0","grpcio-status<2.0.dev0,>=1.33.2","grpcio-status<2.0.dev0,>=1.49.1; python_version >= \"3.11\"","grpcio<2.0dev,>=1.33.2","grpcio<2.0dev,>=1.49.1; python_version >= \"3.11\"",]files = [- {file = "google-api-core-2.17.1.tar.gz", hash = "sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95"},- {file = "google_api_core-2.17.1-py3-none-any.whl", hash = "sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e"},+ {file = "google-api-core-2.18.0.tar.gz", hash = "sha256:62d97417bfc674d6cef251e5c4d639a9655e00c45528c4364fbfebb478ce72a9"},+ {file = "google_api_core-2.18.0-py3-none-any.whl", hash = "sha256:5a63aa102e0049abe85b5b88cb9409234c1f70afcda21ce1e40b285b9629c1d6"},][[package]]name = "google-api-python-client"-version = "2.122.0"+version = "2.123.0"requires_python = ">=3.7"summary = "Google API Client Library for Python"groups = ["hook-check-django-migrations"]@@ -1509,8 +1507,8 @@ dependencies = ["uritemplate<5,>=3.0.1",]files = [- {file = "google-api-python-client-2.122.0.tar.gz", hash = "sha256:77447bf2d6b6ea9e686fd66fc2f12ee7a63e3889b7427676429ebf09fcb5dcf9"},- {file = "google_api_python_client-2.122.0-py2.py3-none-any.whl", hash = "sha256:a5953e60394b77b98bcc7ff7c4971ed784b3b693e9a569c176eaccb1549330f2"},+ {file = "google-api-python-client-2.123.0.tar.gz", hash = "sha256:a17226b02f71de581afe045437b441844110a9cd91580b73549d41108cf1b9f0"},+ {file = "google_api_python_client-2.123.0-py2.py3-none-any.whl", hash = "sha256:1c2bcaa846acf5bac4d6f244d8373d4de9de73d64eb6e77b56767ab4cf681419"},][[package]]@@ -2186,13 +2184,13 @@ files = [[[package]]name = "isort"-version = "5.12.0"+version = "5.13.2"requires_python = ">=3.8.0"summary = "A Python utility / library to sort Python imports."groups = ["lint"]files = [- {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"},- {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"},+ {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"},+ {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"},][[package]]@@ -3170,8 +3168,8 @@ files = [[[package]]name = "pre-commit"-version = "3.3.3"-requires_python = ">=3.8"+version = "3.6.2"+requires_python = ">=3.9"summary = "A framework for managing and maintaining multi-language pre-commit hooks."groups = ["lint"]dependencies = [@@ -3182,8 +3180,8 @@ dependencies = ["virtualenv>=20.10.0",]files = [- {file = "pre_commit-3.3.3-py2.py3-none-any.whl", hash = "sha256:10badb65d6a38caff29703362271d7dca483d01da88f9d7e05d0b97171c136cb"},- {file = "pre_commit-3.3.3.tar.gz", hash = "sha256:a2256f489cd913d575c145132ae196fe335da32d91a8294b7afe6622335dd023"},+ {file = "pre_commit-3.6.2-py2.py3-none-any.whl", hash = "sha256:ba637c2d7a670c10daedc059f5c49b5bd0aadbccfcd7ec15592cf9665117532c"},+ {file = "pre_commit-3.6.2.tar.gz", hash = "sha256:c3ef34f463045c88658c5b99f38c1e297abdcc0ff13f98d3370055fbbfabc67e"},][[package]]@@ -3339,13 +3337,13 @@ files = [[[package]]name = "pycodestyle"-version = "2.10.0"-requires_python = ">=3.6"+version = "2.11.1"+requires_python = ">=3.8"summary = "Python style guide checker"groups = ["lint"]files = [- {file = "pycodestyle-2.10.0-py2.py3-none-any.whl", hash = "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610"},- {file = "pycodestyle-2.10.0.tar.gz", hash = "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053"},+ {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"},+ {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"},][[package]]@@ -3509,13 +3507,13 @@ files = [[[package]]name = "pyflakes"-version = "3.0.1"-requires_python = ">=3.6"+version = "3.2.0"+requires_python = ">=3.8"summary = "passive checker of Python programs"groups = ["lint"]files = [- {file = "pyflakes-3.0.1-py2.py3-none-any.whl", hash = "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf"},- {file = "pyflakes-3.0.1.tar.gz", hash = "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd"},+ {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"},+ {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"},][[package]]@@ -3929,8 +3927,8 @@ files = [[[package]]name = "requests-oauthlib"-version = "1.4.0"-requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"+version = "2.0.0"+requires_python = ">=3.4"summary = "OAuthlib authentication support for Requests."groups = ["hook-check-django-migrations"]dependencies = [@@ -3938,8 +3936,8 @@ dependencies = ["requests>=2.0.0",]files = [- {file = "requests-oauthlib-1.4.0.tar.gz", hash = "sha256:acee623221e4a39abcbb919312c8ff04bd44e7e417087fb4bd5e2a2f53d5e79a"},- {file = "requests_oauthlib-1.4.0-py2.py3-none-any.whl", hash = "sha256:7a3130d94a17520169e38db6c8d75f2c974643788465ecc2e4b36d288bf13033"},+ {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"},+ {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"},][[package]]@@ -4244,7 +4242,7 @@ files = [[[package]]name = "smart-open"-version = "7.0.2"+version = "7.0.3"requires_python = "<4.0,>=3.7"summary = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)"groups = ["hook-check-django-migrations"]@@ -4252,8 +4250,8 @@ dependencies = ["wrapt",]files = [- {file = "smart_open-7.0.2-py3-none-any.whl", hash = "sha256:361a8fadfc9c1cfb81e8d18a96b724e1d6216f8f3ad543c841e19fc0c56a003a"},- {file = "smart_open-7.0.2.tar.gz", hash = "sha256:f109517702453c3fa5ac60178698138b618d3670c2f1f0993ee8ce27193d3341"},+ {file = "smart_open-7.0.3-py3-none-any.whl", hash = "sha256:7969ac4f1a04fd829d50f086fd106c6b84c8f68786666a4fac7bdc208f24f3b6"},+ {file = "smart_open-7.0.3.tar.gz", hash = "sha256:dfea0d6815798f843fd2f4dbeb2ddf2d4a2304ece749c5251a97ac5f2dcb1a8e"},][[package]]@@ -4525,18 +4523,18 @@ files = [[[package]]name = "storage3"-version = "0.7.3"-requires_python = ">=3.8,<4.0"+version = "0.7.4"+requires_python = "<4.0,>=3.8"summary = "Supabase Storage client for Python."groups = ["hook-check-django-migrations"]dependencies = [- "httpx<0.26,>=0.24",+ "httpx<0.28,>=0.24","python-dateutil<3.0.0,>=2.8.2","typing-extensions<5.0.0,>=4.2.0",]files = [- {file = "storage3-0.7.3-py3-none-any.whl", hash = "sha256:dc6a59da801ee6fc00015da4967ac0b5c3e5508d31ffd796f0e4c83957e5c6a0"},- {file = "storage3-0.7.3.tar.gz", hash = "sha256:943c31de4a7c7490ad7960d963a6b410979ebd0e1b3d320d76cb61564ab0b528"},+ {file = "storage3-0.7.4-py3-none-any.whl", hash = "sha256:0b8e8839b10a64063796ce55a41462c7ffd6842e0ada74f25f5dcf37e1d1bade"},+ {file = "storage3-0.7.4.tar.gz", hash = "sha256:61fcbf836f566405981722abb7d56caa57025b261e7a316e73316701abf0c040"},][[package]]@@ -4882,15 +4880,16 @@ files = [[[package]]name = "types-pyopenssl"-version = "23.2.0.2"+version = "24.0.0.20240311"+requires_python = ">=3.8"summary = "Typing stubs for pyOpenSSL"groups = ["lint"]dependencies = ["cryptography>=35.0.0",]files = [- {file = "types-pyOpenSSL-23.2.0.2.tar.gz", hash = "sha256:6a010dac9ecd42b582d7dd2cc3e9e40486b79b3b64bb2fffba1474ff96af906d"},- {file = "types_pyOpenSSL-23.2.0.2-py3-none-any.whl", hash = "sha256:19536aa3debfbe25a918cf0d898e9f5fbbe6f3594a429da7914bf331deb1b342"},+ {file = "types-pyOpenSSL-24.0.0.20240311.tar.gz", hash = "sha256:7bca00cfc4e7ef9c5d2663c6a1c068c35798e59670595439f6296e7ba3d58083"},+ {file = "types_pyOpenSSL-24.0.0.20240311-py3-none-any.whl", hash = "sha256:6e8e8bfad34924067333232c93f7fc4b369856d8bea0d5c9d1808cb290ab1972"},][[package]]@@ -5246,13 +5245,13 @@ files = [[[package]]name = "validators"-version = "0.23.2"+version = "0.24.0"requires_python = ">=3.8"summary = "Python Data Validation for Humans™"groups = ["hook-check-django-migrations"]files = [- {file = "validators-0.23.2-py3-none-any.whl", hash = "sha256:cb68e489f946f4934dbd3c203f08486c53493ab1c204f39d13b13c209f60e2c1"},- {file = "validators-0.23.2.tar.gz", hash = "sha256:397ec4a1e86192e6e904b5afe8c12da0e6eaa3db088c0cc426e00cfee2ef07a8"},+ {file = "validators-0.24.0-py3-none-any.whl", hash = "sha256:4a99eb368747e60900bae947418eb21e230ff4ff5e7b7944b9308c456d86da32"},+ {file = "validators-0.24.0.tar.gz", hash = "sha256:cd23defb36de42d14e7559cf0757f761bb46b10d9de2998e6ef805f769d859e3"},][[package]]diff --git a/pyproject.toml b/pyproject.tomlindex eefc92a74..6b2df0787 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -12,15 +12,15 @@ build-backend = "pdm.backend"lint = ["absolufy-imports~=0.3.1","autopep8~=2.0.2",- "black~=23.3.0",+ "black~=24.3.0","docutils~=0.20.1",- "flake8~=6.0.0",+ "flake8~=7.0.0","flake8-pyproject~=1.2.2",- "isort~=5.12.0",+ "isort~=5.13.2","mypy~=1.9.0",- "pre-commit~=3.3.1",+ "pre-commit~=3.6.2","types-PyMySQL~=1.1.0.1",- "types-pyOpenSSL~=23.2.0.2",+ "types-pyOpenSSL~=24.0.0.20240311","types-PyYAML~=6.0.12.12","types-redis~=4.6.0.3","types-requests~=2.31.0.6",@@ -58,13 +58,6 @@ target-version = ['py39'][tool.pycln]all = true-[tool.pyupgrade]-keep_dict_typing = true--[tool.flake8]-max-line-length = 80-exclude = "pyproject.toml"-[tool.isort]line_length = 80multi_line_output = 3@@ -78,14 +71,10 @@ profile = "black"DJANGO_SETTINGS_MODULE = "backend.settings.test_cases"python_files = ["tests.py", "test_*.py", "*_tests.py"]-# IMPORTANT!-# Keep this same as mypy hook args in .pre-commit-config.yaml[tool.mypy]allow_subclassing_any = trueallow_untyped_decorators = truecheck_untyped_defs = true-# To use Unpack for typing kwargs-enable_incomplete_feature = "Unpack"exclude = '''(?x)^(.*migrations/.*\.py|backend/prompt/.*|diff --git a/run-platform.sh b/run-platform.shindex 2f21dddd5..1ec79c27a 100755--- a/run-platform.sh+++ b/run-platform.sh@@ -5,6 +5,7 @@ set -o errexit # exit for any command failure"# text color escape codes (\033 == \e but OSX doesn't respect the \e)blue_text='\033[94m'+green_text='\033[32m'red_text='\033[31m'default_text='\033[39m'@@ -17,20 +18,42 @@ debug() {fi}+check_dependencies() {+ if ! command -v git &> /dev/null; then+ echo "$red_text""git not found. Exiting.""$default_text"+ exit 1+ fi+ if ! command -v python3 &> /dev/null; then+ echo "$red_text""python3 not found. Exiting.""$default_text"+ exit 1+ fi+ if ! command -v docker &> /dev/null; then+ echo "$red_text""docker not found. Exiting.""$default_text"+ exit 1+ fi+ # For 'docker compose' vs 'docker-compose', see https://stackoverflow.com/a/66526176.+ if command -v docker compose &> /dev/null; then+ docker_compose_cmd="docker compose"+ elif command -v docker-compose &> /dev/null; then+ docker_compose_cmd="docker-compose"+ else+ echo "$red_text""Both 'docker compose' and 'docker-compose' not found. Exiting.""$default_text"+ exit 1+ fi+}+display_banner() {# Make sure the console is hugeif test $(tput cols) -ge 64; then- # Make it green!- echo -e "\033[32m"- echo -e " _ _ _ _ _____ _______ _____ _____ _______"- echo -e "| | | | \ | |/ ____|__ __| __ \ /\ / ____|__ __|"- echo -e "| | | | \| | (___ | | | |__) | / \ | | | |"- echo -e "| | | | . |\___ \ | | | _ / / /\ \| | | |"- echo -e "| |__| | |\ |____) | | | | | \ \ / ____ \ |____ | |"- echo -e " \____/|_| \_|_____/ |_| |_| \_\/_/ \_\_____| |_|"- echo -e " "- # Make it less green- echo -e "\033[0m"+ echo " █████ █████"+ echo "░░███ ░░███ "+ echo " ░███ ░███ "+ echo " ░███ ░███ "+ echo " ░███ ░███ "+ echo " ░███ ░███ "+ echo " ░░█████████ >UNSTRACT COMMUNITY EDITION"+ echo " ░░░░░░░░░ "+ echo ""sleep 1fi}@@ -40,11 +63,11 @@ display_help() {echoecho -e "Syntax: $0 [options]"echo -e "Options:"- echo -e " -h, --help Displays the help information"+ echo -e " -h, --help Display help information"echo -e " -e, --only-env Only do env files setup"echo -e " -p, --only-pull Only do docker images pull"echo -e " -b, --build-local Build docker images locally"- echo -e " -d, --detach Run docker containers in detached mode"+ echo -e " -u, --upgrade Upgrade services"echo -e " -x, --trace Enables trace mode"echo -e " -V, --verbose Print verbose logs"echo -e " -v, --version Docker images version tag (default \"latest\")"@@ -68,8 +91,8 @@ parse_args() {-b | --build-local)opt_build_local=true;;- -d | --detach)- opt_detach="-d"+ -u | --upgrade)+ opt_upgrade=true;;-x | --trace)set -o xtrace # display every line before execution; enables PS4@@ -101,26 +124,21 @@ parse_args() {debug "OPTION only_env: $opt_only_env"debug "OPTION only_pull: $opt_only_pull"debug "OPTION build_local: $opt_build_local"- debug "OPTION detach: $opt_detach"+ debug "OPTION upgrade: $opt_upgrade"debug "OPTION verbose: $opt_verbose"debug "OPTION version: $opt_version"}-check_required_packages() {- echo "Checking if the required packages are installed..."- if ! command -v docker &> /dev/null; then- echo "$red_text""docker not found. Please install it and try again.""$default_text"- exit 1- fi- if ! (command -v docker compose &> /dev/null && command -v docker-compose &> /dev/null); then- echo "$red_text""docker compose not found. Please install it and try again.""$default_text"- exit 1- fi- if ! command -v python3 &> /dev/null; then- echo "$red_text""python3 not found. Please install it and try again.""$default_text"- exit 1+do_git_pull() {+ if [ "$opt_upgrade" = false ]; then+ returnfi+ echo -e "Performing git switch to ""$blue_text""main branch""$default_text".+ # git switch main++ echo -e "Performing ""$blue_text""git pull""$default_text"" on main branch."+ git pull}setup_env() {@@ -137,17 +155,30 @@ setup_env() {# Add encryption secret for backend and platform-service.if [[ "$service" == "backend" || "$service" == "platform-service" ]]; thenecho -e "$blue_text""Adding encryption secret to $service""$default_text"- echo "ENCRYPTION_KEY=\"$ENCRYPTION_KEY\"" >> $env_path+ if [[ "$OSTYPE" == "darwin"* ]]; then+ sed -i '' "s/ENCRYPTION_KEY.*/ENCRYPTION_KEY=\"$ENCRYPTION_KEY\"/" $env_path+ else+ sed -i "s/ENCRYPTION_KEY.*/ENCRYPTION_KEY=\"$ENCRYPTION_KEY\"/" $env_path+ fifi# Add default auth credentials for backend.if [ "$service" == "backend" ]; thenecho -e "$blue_text""Adding default auth credentials to $service""$default_text"- echo "DEFAULT_AUTH_USERNAME=\"$DEFAULT_AUTH_KEY\"" >> $env_path- echo "DEFAULT_AUTH_PASSWORD=\"$DEFAULT_AUTH_KEY\"" >> $env_path+ if [[ "$OSTYPE" == "darwin"* ]]; then+ sed -i '' "s/DEFAULT_AUTH_USERNAME.*/DEFAULT_AUTH_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path+ sed -i '' "s/DEFAULT_AUTH_PASSWORD.*/DEFAULT_AUTH_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path+ else+ sed -i "s/DEFAULT_AUTH_USERNAME.*/DEFAULT_AUTH_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path+ sed -i "s/DEFAULT_AUTH_PASSWORD.*/DEFAULT_AUTH_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path+ fifiecho -e "Created env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"."else- echo -e "Found existing env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"."+ python3 $script_dir/docker/scripts/merge_env.py $sample_env_path $env_path+ if [ $? -ne 0 ]; then+ exit 1+ fi+ echo -e "Merged env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"."fidone@@ -155,7 +186,11 @@ setup_env() {cp "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env"echo -e "Created env for ""$blue_text""essential services""$default_text"" at ""$blue_text""$script_dir/docker/essentials.env""$default_text""."else- echo -e "Found existing env for ""$blue_text""essential services""$default_text"" at ""$blue_text""$script_dir/docker/essentials.env""$default_text""."+ python3 $script_dir/docker/scripts/merge_env.py "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env"+ if [ $? -ne 0 ]; then+ exit 1+ fi+ echo -e "Merged env for ""$blue_text""essential services""$default_text"" at ""$blue_text""$script_dir/docker/essentials.env""$default_text""."fiif [ ! -e "$script_dir/docker/proxy_overrides.yaml" ]; then@@ -165,7 +200,7 @@ setup_env() {fiif [ "$opt_only_env" = true ]; then- echo "Done." && exit 0+ echo -e "$green_text""Done.""$default_text" && exit 0fi}@@ -173,43 +208,46 @@ build_services() {pushd ${script_dir}/docker 1>/dev/nullif [ "$opt_build_local" = true ]; then- echo "Building docker images locally."- VERSION=$opt_version docker-compose -f $script_dir/docker/docker-compose.build.yaml build || {+ echo -e "$blue_text""Building""$default_text"" docker images ""$blue_text""$opt_version""$default_text"" locally."+ VERSION=$opt_version $docker_compose_cmd -f $script_dir/docker/docker-compose.build.yaml build || {echo -e "$red_text""Failed to build docker images.""$default_text"exit 1}else- for service in "${services[@]}"; do- if ! docker image inspect "unstract/${service}:$opt_version" &> /dev/null; then- echo -e "$red_text""Docker image 'unstract/${service}:$opt_version' not found. Pulling...""$default_text"- docker pull unstract/${service}:$opt_version || {- echo -e "$red_text""Failed to pull 'unstract/${service}:$opt_version'. Check if version exists.""$default_text"- exit 1- }- else- echo -e "Found docker image ""$blue_text""'unstract/${service}:$opt_version'""$default_text""."- fi- done+ echo -e "$blue_text""Pulling""$default_text"" docker images tag ""$blue_text""$opt_version""$default_text""."++ pull_policy="missing"+ if [ "$opt_upgrade" = true ] && [ "$opt_version" = "latest" ]; then+ pull_policy="always"+ fi++ VERSION=$opt_version $docker_compose_cmd -f $script_dir/docker/docker-compose.yaml pull --policy $pull_policy || {+ echo -e "$red_text""Failed to pull docker images. Check the version.""$default_text"+ echo -e "$red_text""Also make sure docker is running and try again.""$default_text"+ exit 1+ }fipopd 1>/dev/nullif [ "$opt_only_pull" = true ]; then- echo "Done." && exit 0+ echo -e "$green_text""Done.""$default_text" && exit 0fi}run_services() {pushd ${script_dir}/docker 1>/dev/null- if [ -z "$opt_detach" ]; then- echo -e "$blue_text""Starting docker containers""$default_text"- else- echo -e "$blue_text""Starting docker containers in detach mode""$default_text"- fi- echo -e "Once the services are up, visit ""$blue_text""http://frontend.unstract.localhost""$default_text"" in your browser."+ echo -e "$blue_text""Starting docker containers in detached mode""$default_text"+ VERSION=$opt_version $docker_compose_cmd up -d- VERSION=$opt_version docker compose up $opt_detach+ if [ "$opt_upgrade" = true ]; then+ echo ""+ echo -e "$green_text""Upgraded platform to $opt_version version.""$default_text"+ fi+ echo -e "\nOnce the services are up, visit ""$blue_text""http://frontend.unstract.localhost""$default_text"" in your browser."+ echo "See logs with:"+ echo -e " ""$blue_text""$docker_compose_cmd -f docker/docker-compose.yaml logs -f""$default_text"popd 1>/dev/null}@@ -217,22 +255,23 @@ run_services() {## Run Unstract platform - BEGIN#-check_required_packages+check_dependenciesopt_only_env=falseopt_only_pull=falseopt_build_local=false-opt_detach=""+opt_upgrade=falseopt_verbose=falseopt_version="latest"script_dir=$(dirname "$(readlink -f "$BASH_SOURCE")")# Extract service names from docker compose file.-services=($(VERSION=$opt_version docker compose -f $script_dir/docker/docker-compose.build.yaml config --services))+services=($(VERSION=$opt_version $docker_compose_cmd -f $script_dir/docker/docker-compose.build.yaml config --services))display_bannerparse_args $*+do_git_pullsetup_envbuild_servicesrun_services