Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Run script tweaks (#120)

Hari John Kuriakosee8668a073bd5
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f50036369..c67d62e4a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -144,43 +144,6 @@ repos:
entry: pyupgrade --py38-plus --keep-runtime-typing
types:
- python
- - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.9.0
- hooks:
- - id: mypy
- language: system
- entry: mypy .
- pass_filenames: false
- # IMPORTANT!
- # Keep args same as tool.mypy section in pyproject.toml
- args:
- [
- --allow-subclassing-any,
- --allow-untyped-decorators,
- --check-untyped-defs,
- # To use Unpack for typing kwargs
- --enable-incomplete-feature=Unpack,
- --exclude, ".*migrations/.*.py",
- --exclude, "backend/prompt/.*",
- --exclude, "document-service/.*",
- --exclude, "unstract/connectors/tests/.*",
- --exclude, "unstract/core/.*",
- --exclude, "unstract/flags/src/unstract/flags/.*",
- --exclude, "__pypackages__/.*",
- --follow-imports, "silent",
- --ignore-missing-imports,
- --implicit-reexport,
- --pretty,
- --python-version=3.9,
- --show-column-numbers,
- --show-error-codes,
- --strict,
- --warn-redundant-casts,
- --warn-return-any,
- --warn-unreachable,
- --warn-unused-configs,
- --warn-unused-ignores,
- ]
- repo: https://github.com/igorshubovych/markdownlint-cli
rev: v0.39.0
hooks:
@@ -194,11 +157,14 @@ repos:
- id: pdm-lock-check
- repo: local
hooks:
- # ! IMPORTANT !
- # Requires unstract-db to be running and accessible
+ - id: run-mypy
+ name: Run mypy
+ entry: sh -c 'pdm run mypy .'
+ language: system
+ pass_filenames: false
- id: check-django-migrations
name: Check django migrations
- entry: sh -c 'pdm run `find . -name "manage.py" -not -path "*/.venv/*"` makemigrations --check --dry-run --no-input'
+ entry: sh -c 'pdm run docker/scripts/check_django_migrations.sh'
language: system
types: [python] # hook only runs if a python file is staged
pass_filenames: false
diff --git a/dev-env-cli.sh b/dev-env-cli.sh
new file mode 100755
index 000000000..47dc5340f
--- /dev/null
+++ b/dev-env-cli.sh
@@ -0,0 +1,317 @@
+#!/usr/bin/env bash
+
+set -o nounset # exit if a variable is not set
+set -o errexit # exit for any command failure"
+
+# text color escape codes (\033 == \e but OSX doesn't respect the \e)
+blue_text='\033[94m'
+green_text='\033[32m'
+red_text='\033[31m'
+default_text='\033[39m'
+
+# set -x/xtrace uses PS4 for more info
+PS4="$blue_text""${0}:${LINENO}: ""$default_text"
+
+debug() {
+ if [ "$opt_verbose" = true ]; then
+ echo $1
+ fi
+}
+
+check_dependencies() {
+ if ! command -v python3 &> /dev/null; then
+ echo "$red_text""python3 not found. Exiting.""$default_text"
+ exit 1
+ fi
+ if ! command -v docker compose &> /dev/null; then
+ echo "$red_text""docker not found. Exiting.""$default_text"
+ exit 1
+ fi
+ # For 'docker compose' vs 'docker-compose', see https://stackoverflow.com/a/66526176.
+ if command -v docker compose &> /dev/null; then
+ docker_compose_cmd="docker compose"
+ elif command -v docker-compose &> /dev/null; then
+ docker_compose_cmd="docker-compose"
+ else
+ echo "$red_text""Both 'docker compose' and 'docker-compose' not found. Exiting.""$default_text"
+ exit 1
+ fi
+ if ! command -v pdm &> /dev/null; then
+ echo "$red_text""pdm not found. Exiting.""$default_text"
+ exit 1
+ fi
+}
+
+display_banner() {
+ # Make sure the console is huge
+ if test $(tput cols) -ge 64; then
+ echo " █████ █████"
+ echo "░░███ ░░███ "
+ echo " ░███ ░███ "
+ echo " ░███ ░███ "
+ echo " ░███ ░███ "
+ echo " ░███ ░███ "
+ echo " ░░█████████ >UNSTRACT COMMUNITY EDITION"
+ echo " ░░░░░░░░░ "
+ echo ""
+ sleep 1
+ fi
+}
+
+display_help() {
+ printf "Dev environment CLI for Unstract platform services\n"
+ echo
+ echo -e "Syntax: $0 [options] -s service"
+ echo -e "Options:"
+ echo -e " -h, --help Display help information"
+ echo -e " -e, --setup-venv Setup venv (requires service)"
+ echo -e " -a, --activate-venv Activate venv (requires service)"
+ echo -e " -i, --install-deps Install dependencies in venv (requires service)"
+ echo -e " -d, --destroy-venv Destroy venv (requires service)"
+ echo -e " -p, --install-pre-commit-hook Install Git pre-commit hook"
+ echo -e " -r, --run-pre-commit-hook Run Git pre-commit hook"
+ echo -e " -f, --force Force operation"
+ echo -e " -s, --service Service name"
+ echo -e " -x, --trace Enables trace mode"
+ echo -e " -V, --verbose Print verbose logs"
+ echo -e ""
+}
+
+parse_args() {
+ while [[ $# -gt 0 ]]; do
+ arg="$1"
+ case $arg in
+ -h | --help)
+ display_help
+ exit
+ ;;
+ -e | --setup-venv)
+ opt_setup_venv=true
+ ;;
+ -a | --activate-venv)
+ opt_activate_venv=true
+ ;;
+ -i | --install-deps)
+ opt_install_deps=true
+ ;;
+ -d | --destroy-venv)
+ opt_destroy_venv=true
+ ;;
+ -p | --install-pre-commit-hook)
+ opt_install_pre_commit_hook=true
+ ;;
+ -r | --run-pre-commit-hook)
+ opt_run_pre_commit_hook=true
+ ;;
+ -f | --force)
+ opt_force="--force"
+ ;;
+ -s | --service)
+ if [ -z "${2-}" ]; then
+ echo "No service specified."
+ echo
+ display_help
+ exit
+ else
+ opt_service="$2"
+ fi
+ shift
+ ;;
+ -x | --trace)
+ set -o xtrace # display every line before execution; enables PS4
+ ;;
+ -V | --verbose)
+ opt_verbose=true
+ ;;
+ *)
+ echo "'$1' is not a known command."
+ echo
+ display_help
+ exit
+ ;;
+ esac
+ shift
+ done
+
+ if [ "$opt_install_pre_commit_hook" = false ] && [ "$opt_run_pre_commit_hook" = false ]; then
+ if [ "$opt_service" = "" ]; then
+ echo "No service specified."
+ echo
+ display_help
+ exit
+ fi
+ ret=$(echo ${services[@]} | grep -ow "$opt_service" | wc -w)
+ if [ $ret -eq 0 ]; then
+ echo "Unknown service '$opt_service'."
+ echo
+ display_help
+ exit
+ fi
+ fi
+ if [ "$opt_setup_venv" = false ] && [ "$opt_activate_venv" = false ] &&
+ [ "$opt_install_deps" = false ] && [ "$opt_destroy_venv" = false ] &&
+ [ "$opt_install_pre_commit_hook" = false ] && [ "$opt_run_pre_commit_hook" = false ]; then
+ echo "One of -e,-a,-i,-d,-p,-r options should be specified."
+ echo
+ display_help
+ exit
+ fi
+
+ debug "OPTION setup venv: $opt_setup_venv"
+ debug "OPTION activate venv: $opt_activate_venv"
+ debug "OPTION install deps: $opt_install_deps"
+ debug "OPTION destroy venv: $opt_destroy_venv"
+ debug "OPTION install pre-commit hook: $opt_install_pre_commit_hook"
+ debug "OPTION run pre-commit hook: $opt_run_pre_commit_hook"
+ debug "OPTION service: $opt_service"
+ debug "OPTION verbose: $opt_verbose"
+}
+
+setup_venv() {
+ if [ "$opt_setup_venv" = false ]; then
+ return
+ fi
+
+ pushd ${script_dir}/$opt_service 1>/dev/null
+
+ if [ -e "package.json" ]; then
+ echo -e "Nothing to do for ""$blue_text""$opt_service""$default_text"
+ elif [ -e "pyproject.toml" ]; then
+ echo -e "Setting up ""$blue_text""Python venv""$default_text"" for ""$blue_text""$opt_service""$default_text"
+ pdm venv create -w virtualenv --with-pip $opt_force
+ fi
+
+ popd 1>/dev/null
+}
+
+activate_venv() {
+ if [ "$opt_activate_venv" = false ]; then
+ return
+ fi
+
+ pushd ${script_dir}/$opt_service 1>/dev/null
+
+ if [ -e "package.json" ]; then
+ echo -e "Nothing to do for ""$blue_text""$opt_service""$default_text"
+ elif [ -e "pyproject.toml" ]; then
+ if [ ! -e ".venv" ]; then
+ echo -e "venv not found for ""$blue_text""$opt_service""$default_text"". Please run setup first."
+ return
+ fi
+ echo -e "Run the following in a ""$blue_text""new terminal""$default_text"" to activate venv for ""$blue_text""$opt_service""$default_text"":"
+ echo ""
+ echo "cd ${script_dir}/$opt_service"
+ echo "eval \"$(pdm venv activate in-project)\""
+ fi
+
+ popd 1>/dev/null
+}
+
+install_deps() {
+ if [ "$opt_install_deps" = false ]; then
+ return
+ fi
+
+ pushd ${script_dir}/$opt_service 1>/dev/null
+
+ if [ -e "package.json" ]; then
+ echo -e "Installing dependencies for ""$blue_text""$opt_service""$default_text"
+ npm ci
+ elif [ -e "pyproject.toml" ]; then
+ if [ ! -e ".venv" ]; then
+ echo -e "venv not found for ""$blue_text""$opt_service""$default_text"". Please run setup first."
+ return
+ fi
+ echo -e "Installing dependencies in venv for ""$blue_text""$opt_service""$default_text"
+ eval "$(pdm venv activate in-project)"
+ pdm sync
+ fi
+
+ popd 1>/dev/null
+}
+
+destroy_venv() {
+ if [ "$opt_destroy_venv" = false ]; then
+ return
+ fi
+
+ pushd ${script_dir}/$opt_service 1>/dev/null
+
+ if [ -e "package.json" ]; then
+ echo -e "$blue_text""Nothing to do for $opt_service""$default_text"
+ elif [ -e "pyproject.toml" ]; then
+ if [ ! -e ".venv" ]; then
+ echo -e "venv not found for ""$blue_text""$opt_service""$default_text"". Please run setup first."
+ return
+ fi
+ echo -e "Destroying venv for ""$blue_text""$opt_service""$default_text"
+ pdm venv remove in-project
+ fi
+
+ popd 1>/dev/null
+}
+
+install_pre_commit_hook() {
+ if [ "$opt_install_pre_commit_hook" = false ]; then
+ return
+ fi
+
+ pushd ${script_dir} 1>/dev/null
+
+ echo -e "Installing ""$blue_text""Git pre-commit hook""$default_text"
+ pdm venv create -w virtualenv --with-pip --force
+ eval $(pdm venv activate in-project)
+ pdm sync --dev -G lint -G hook-check-django-migrations
+ pre-commit install
+
+ popd 1>/dev/null
+}
+
+run_pre_commit_hook() {
+ if [ "$opt_run_pre_commit_hook" = false ]; then
+ return
+ fi
+
+ pushd ${script_dir} 1>/dev/null
+
+ if [ ! -e ".venv" ]; then
+ echo -e "$blue_text""Git pre-commit hook""$default_text"" not found. Please run install first."
+ return
+ fi
+ eval "$(pdm venv activate in-project)"
+ pre-commit run
+
+ popd 1>/dev/null
+}
+
+#
+# Run Unstract platform - BEGIN
+#
+check_dependencies
+
+opt_setup_venv=false
+opt_activate_venv=false
+opt_install_deps=false
+opt_destroy_venv=false
+opt_install_pre_commit_hook=false
+opt_run_pre_commit_hook=false
+opt_force=""
+opt_service=""
+opt_verbose=false
+
+script_dir=$(dirname "$(readlink -f "$BASH_SOURCE")")
+# Extract service names from docker compose file.
+services=($(VERSION="latest" $docker_compose_cmd -f $script_dir/docker/docker-compose.build.yaml config --services))
+
+display_banner
+parse_args $*
+
+setup_venv
+activate_venv
+install_deps
+destroy_venv
+install_pre_commit_hook
+run_pre_commit_hook
+#
+# Run Unstract platform - END
+#
diff --git a/docker/scripts/check_django_migrations.sh b/docker/scripts/check_django_migrations.sh
new file mode 100755
index 000000000..e2e004025
--- /dev/null
+++ b/docker/scripts/check_django_migrations.sh
@@ -0,0 +1,49 @@
+#! /bin/sh
+
+script_dir=$(dirname "$(readlink -f "$BASH_SOURCE")")
+
+cd $script_dir/../../backend
+
+# Check Django default db connectivity.
+# Use Python for cross-platform compatibility.
+python3 << EOF
+import os
+import socket
+import sys
+
+import dotenv
+
+dotenv.load_dotenv()
+
+def _check_connectivity(host, port):
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ s.settimeout(2)
+ errno = s.connect_ex((host, port))
+ return errno
+
+db_host = os.getenv("DB_HOST", "localhost")
+db_port = int(os.getenv("DB_PORT", 5432))
+
+errno = _check_connectivity(db_host, db_port)
+
+if errno == 0:
+ print("Django default db ok:", os.strerror(errno))
+ sys.exit(0)
+else:
+ print("Django default db error:", os.strerror(errno))
+ sys.exit(1)
+EOF
+if [ $? -ne 0 ]; then
+ echo ""
+ echo "Verify DB_HOST, DB_PORT settings for backend Django default db."
+ echo "If db is running in a container, add DB_HOST value to /etc/hosts."
+ echo ""
+ exit 1
+fi
+
+# Check Django migrations.
+python3 manage.py makemigrations --check --dry-run --no-input
+# ! IMPORTANT !
+# Above command does not return error exit code
+# on network error when connecting to db.
+exit 0
diff --git a/docker/scripts/merge_env.py b/docker/scripts/merge_env.py
new file mode 100644
index 000000000..5f72dc94e
--- /dev/null
+++ b/docker/scripts/merge_env.py
@@ -0,0 +1,118 @@
+#
+# Usage:
+# python merge_env.py [--dry-run] base_env_path target_env_path
+#
+# e.g.
+# Dry run:
+# python merge_env.py --dry-run sample.env .env
+# Actual run:
+# python merge_env.py sample.env .env
+#
+import sys
+
+PREFERRED_BASE_ENV_KEYS = [
+ "STRUCTURE_TOOL_IMAGE_URL",
+ "STRUCTURE_TOOL_IMAGE_TAG",
+]
+
+
+def _extract_kv_from_line(line: str) -> tuple[str, str]:
+ parts = line.split("=", 1)
+ if len(parts) != 2:
+ raise ValueError(f"{line}")
+ key, value = parts
+ key = key.strip()
+ value = value.strip()
+ return key, value
+
+
+def _extract_from_env_file(file_path: str) -> dict[str, str]:
+ env = {}
+
+ with open(file_path) as file:
+ for line in file:
+ if not line.strip() or line.startswith("#"):
+ continue
+
+ key, value = _extract_kv_from_line(line)
+
+ env[key.strip()] = value.strip()
+
+ return env
+
+
+def _merge_to_env_file(
+ base_env_file_path: str, target_env: dict[str, str] = {}
+) -> str:
+ """Generates file contents after merging input base env file path with
+ target env.
+
+ Args:
+ base_env_path (string): Base env file path.
+ target_env (dict, optional): Target env to use for merge.
+
+ Returns:
+ string: File contents after merge.
+ """
+ merged_contents = []
+
+ with open(base_env_file_path) as file:
+ for line in file:
+ # Preserve location of empty lines and comments
+ # for easy diff.
+ if not line.strip() or line.startswith("#"):
+ merged_contents.append(line)
+ continue
+
+ key, value = _extract_kv_from_line(line)
+
+ # Preserve following keys at base env file path:
+ # - preferred keys
+ # - newly added keys
+ # Everything else, take existing configured values
+ # from target env.
+ if key not in PREFERRED_BASE_ENV_KEYS and key in target_env:
+ value = target_env.get(key, value)
+
+ merged_contents.append(f"{key} = {value}\n")
+
+ return "".join(merged_contents)
+
+
+def _save_merged_contents(
+ file_path: str, file_contents: str, dry_run: bool = False
+) -> None:
+ if dry_run:
+ print(f"===== merged:{file_path} =====")
+ print(file_contents)
+ print("-----------")
+ else:
+ with open(file_path, "w") as file:
+ file.write(file_contents)
+
+
+def merge_env(
+ base_env_file_path: str, target_env_file_path: str, dry_run: bool = False
+) -> None:
+ target_env = _extract_from_env_file(target_env_file_path)
+ merged_contents = _merge_to_env_file(
+ base_env_file_path, target_env=target_env
+ )
+ _save_merged_contents(
+ target_env_file_path, merged_contents, dry_run=dry_run
+ )
+
+
+if __name__ == "__main__":
+ try:
+ if len(sys.argv) == 4:
+ dry_run = True if sys.argv[1] == "--dry-run" else False
+ merge_env(sys.argv[2], sys.argv[3], dry_run=dry_run)
+ else:
+ merge_env(sys.argv[1], sys.argv[2])
+ except IndexError:
+ print(f"Invalid env paths for merge: {sys.argv}")
+ sys.exit(1)
+ except ValueError as e:
+ print(f"Malformed env config: {str(e)}")
+ sys.exit(1)
diff --git a/pdm.lock b/pdm.lock
index 7e73037d7..f8e83626e 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "hook-check-django-migrations", "lint"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:4856c6816ca68c8c298814a20efd155b658682d68431e88b2dfad61bdd45e658"
+content_hash = "sha256:8f0ed82cbbbffa272795b941dfcec9ae77711e31c83c8b9047ccb1c15cd04186"
[[package]]
name = "absolufy-imports"
@@ -321,7 +321,7 @@ files = [
[[package]]
name = "asgiref"
-version = "3.8.0"
+version = "3.8.1"
requires_python = ">=3.8"
summary = "ASGI specs, helper code, and adapters"
groups = ["hook-check-django-migrations"]
@@ -329,8 +329,8 @@ dependencies = [
"typing-extensions>=4; python_version < \"3.11\"",
]
files = [
- {file = "asgiref-3.8.0-py3-none-any.whl", hash = "sha256:30fc07797ad71a0abb8fe34aa03c8043308a8389abc7942d797ea9911540bc28"},
- {file = "asgiref-3.8.0.tar.gz", hash = "sha256:ec75d9d0f04e2dbfedef1f20ee73a6594af80c333df47cdd31f37e6701f7c53a"},
+ {file = "asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47"},
+ {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"},
]
[[package]]
@@ -535,8 +535,8 @@ files = [
[[package]]
name = "black"
-version = "23.3.0"
-requires_python = ">=3.7"
+version = "24.3.0"
+requires_python = ">=3.8"
summary = "The uncompromising code formatter."
groups = ["lint"]
dependencies = [
@@ -546,26 +546,23 @@ dependencies = [
"pathspec>=0.9.0",
"platformdirs>=2",
"tomli>=1.1.0; python_version < \"3.11\"",
- "typing-extensions>=3.10.0.0; python_version < \"3.10\"",
+ "typing-extensions>=4.0.1; python_version < \"3.11\"",
]
files = [
- {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"},
- {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"},
- {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"},
- {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"},
- {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"},
- {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"},
- {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"},
- {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"},
- {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"},
- {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"},
- {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"},
- {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"},
- {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"},
- {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"},
- {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"},
- {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"},
- {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"},
+ {file = "black-24.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7d5e026f8da0322b5662fa7a8e752b3fa2dac1c1cbc213c3d7ff9bdd0ab12395"},
+ {file = "black-24.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9f50ea1132e2189d8dff0115ab75b65590a3e97de1e143795adb4ce317934995"},
+ {file = "black-24.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2af80566f43c85f5797365077fb64a393861a3730bd110971ab7a0c94e873e7"},
+ {file = "black-24.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:4be5bb28e090456adfc1255e03967fb67ca846a03be7aadf6249096100ee32d0"},
+ {file = "black-24.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4f1373a7808a8f135b774039f61d59e4be7eb56b2513d3d2f02a8b9365b8a8a9"},
+ {file = "black-24.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aadf7a02d947936ee418777e0247ea114f78aff0d0959461057cae8a04f20597"},
+ {file = "black-24.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c02e4ea2ae09d16314d30912a58ada9a5c4fdfedf9512d23326128ac08ac3d"},
+ {file = "black-24.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:bf21b7b230718a5f08bd32d5e4f1db7fc8788345c8aea1d155fc17852b3410f5"},
+ {file = "black-24.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c45f8dff244b3c431b36e3224b6be4a127c6aca780853574c00faf99258041eb"},
+ {file = "black-24.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6905238a754ceb7788a73f02b45637d820b2f5478b20fec82ea865e4f5d4d9f7"},
+ {file = "black-24.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7de8d330763c66663661a1ffd432274a2f92f07feeddd89ffd085b5744f85e7"},
+ {file = "black-24.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:7bb041dca0d784697af4646d3b62ba4a6b028276ae878e53f6b4f74ddd6db99f"},
+ {file = "black-24.3.0-py3-none-any.whl", hash = "sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93"},
+ {file = "black-24.3.0.tar.gz", hash = "sha256:a0c9c4a0771afc6919578cec71ce82a3e31e054904e7197deacbc9382671c41f"},
]
[[package]]
@@ -821,16 +818,16 @@ files = [
[[package]]
name = "click-didyoumean"
-version = "0.3.0"
-requires_python = ">=3.6.2,<4.0.0"
+version = "0.3.1"
+requires_python = ">=3.6.2"
summary = "Enables git-like *did-you-mean* feature in click"
groups = ["hook-check-django-migrations"]
dependencies = [
"click>=7",
]
files = [
- {file = "click-didyoumean-0.3.0.tar.gz", hash = "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035"},
- {file = "click_didyoumean-0.3.0-py3-none-any.whl", hash = "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667"},
+ {file = "click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c"},
+ {file = "click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463"},
]
[[package]]
@@ -1268,18 +1265,18 @@ files = [
[[package]]
name = "flake8"
-version = "6.0.0"
+version = "7.0.0"
requires_python = ">=3.8.1"
summary = "the modular source code checker: pep8 pyflakes and co"
groups = ["lint"]
dependencies = [
"mccabe<0.8.0,>=0.7.0",
- "pycodestyle<2.11.0,>=2.10.0",
- "pyflakes<3.1.0,>=3.0.0",
+ "pycodestyle<2.12.0,>=2.11.0",
+ "pyflakes<3.3.0,>=3.2.0",
]
files = [
- {file = "flake8-6.0.0-py2.py3-none-any.whl", hash = "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7"},
- {file = "flake8-6.0.0.tar.gz", hash = "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181"},
+ {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"},
+ {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"},
]
[[package]]
@@ -1461,43 +1458,44 @@ files = [
[[package]]
name = "google-api-core"
-version = "2.17.1"
+version = "2.18.0"
requires_python = ">=3.7"
summary = "Google API client core library"
groups = ["hook-check-django-migrations"]
dependencies = [
"google-auth<3.0.dev0,>=2.14.1",
"googleapis-common-protos<2.0.dev0,>=1.56.2",
+ "proto-plus<2.0.0dev,>=1.22.3",
"protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5",
"requests<3.0.0.dev0,>=2.18.0",
]
files = [
- {file = "google-api-core-2.17.1.tar.gz", hash = "sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95"},
- {file = "google_api_core-2.17.1-py3-none-any.whl", hash = "sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e"},
+ {file = "google-api-core-2.18.0.tar.gz", hash = "sha256:62d97417bfc674d6cef251e5c4d639a9655e00c45528c4364fbfebb478ce72a9"},
+ {file = "google_api_core-2.18.0-py3-none-any.whl", hash = "sha256:5a63aa102e0049abe85b5b88cb9409234c1f70afcda21ce1e40b285b9629c1d6"},
]
[[package]]
name = "google-api-core"
-version = "2.17.1"
+version = "2.18.0"
extras = ["grpc"]
requires_python = ">=3.7"
summary = "Google API client core library"
groups = ["hook-check-django-migrations"]
dependencies = [
- "google-api-core==2.17.1",
+ "google-api-core==2.18.0",
"grpcio-status<2.0.dev0,>=1.33.2",
"grpcio-status<2.0.dev0,>=1.49.1; python_version >= \"3.11\"",
"grpcio<2.0dev,>=1.33.2",
"grpcio<2.0dev,>=1.49.1; python_version >= \"3.11\"",
]
files = [
- {file = "google-api-core-2.17.1.tar.gz", hash = "sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95"},
- {file = "google_api_core-2.17.1-py3-none-any.whl", hash = "sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e"},
+ {file = "google-api-core-2.18.0.tar.gz", hash = "sha256:62d97417bfc674d6cef251e5c4d639a9655e00c45528c4364fbfebb478ce72a9"},
+ {file = "google_api_core-2.18.0-py3-none-any.whl", hash = "sha256:5a63aa102e0049abe85b5b88cb9409234c1f70afcda21ce1e40b285b9629c1d6"},
]
[[package]]
name = "google-api-python-client"
-version = "2.122.0"
+version = "2.123.0"
requires_python = ">=3.7"
summary = "Google API Client Library for Python"
groups = ["hook-check-django-migrations"]
@@ -1509,8 +1507,8 @@ dependencies = [
"uritemplate<5,>=3.0.1",
]
files = [
- {file = "google-api-python-client-2.122.0.tar.gz", hash = "sha256:77447bf2d6b6ea9e686fd66fc2f12ee7a63e3889b7427676429ebf09fcb5dcf9"},
- {file = "google_api_python_client-2.122.0-py2.py3-none-any.whl", hash = "sha256:a5953e60394b77b98bcc7ff7c4971ed784b3b693e9a569c176eaccb1549330f2"},
+ {file = "google-api-python-client-2.123.0.tar.gz", hash = "sha256:a17226b02f71de581afe045437b441844110a9cd91580b73549d41108cf1b9f0"},
+ {file = "google_api_python_client-2.123.0-py2.py3-none-any.whl", hash = "sha256:1c2bcaa846acf5bac4d6f244d8373d4de9de73d64eb6e77b56767ab4cf681419"},
]
[[package]]
@@ -2186,13 +2184,13 @@ files = [
[[package]]
name = "isort"
-version = "5.12.0"
+version = "5.13.2"
requires_python = ">=3.8.0"
summary = "A Python utility / library to sort Python imports."
groups = ["lint"]
files = [
- {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"},
- {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"},
+ {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"},
+ {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"},
]
[[package]]
@@ -3170,8 +3168,8 @@ files = [
[[package]]
name = "pre-commit"
-version = "3.3.3"
-requires_python = ">=3.8"
+version = "3.6.2"
+requires_python = ">=3.9"
summary = "A framework for managing and maintaining multi-language pre-commit hooks."
groups = ["lint"]
dependencies = [
@@ -3182,8 +3180,8 @@ dependencies = [
"virtualenv>=20.10.0",
]
files = [
- {file = "pre_commit-3.3.3-py2.py3-none-any.whl", hash = "sha256:10badb65d6a38caff29703362271d7dca483d01da88f9d7e05d0b97171c136cb"},
- {file = "pre_commit-3.3.3.tar.gz", hash = "sha256:a2256f489cd913d575c145132ae196fe335da32d91a8294b7afe6622335dd023"},
+ {file = "pre_commit-3.6.2-py2.py3-none-any.whl", hash = "sha256:ba637c2d7a670c10daedc059f5c49b5bd0aadbccfcd7ec15592cf9665117532c"},
+ {file = "pre_commit-3.6.2.tar.gz", hash = "sha256:c3ef34f463045c88658c5b99f38c1e297abdcc0ff13f98d3370055fbbfabc67e"},
]
[[package]]
@@ -3339,13 +3337,13 @@ files = [
[[package]]
name = "pycodestyle"
-version = "2.10.0"
-requires_python = ">=3.6"
+version = "2.11.1"
+requires_python = ">=3.8"
summary = "Python style guide checker"
groups = ["lint"]
files = [
- {file = "pycodestyle-2.10.0-py2.py3-none-any.whl", hash = "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610"},
- {file = "pycodestyle-2.10.0.tar.gz", hash = "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053"},
+ {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"},
+ {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"},
]
[[package]]
@@ -3509,13 +3507,13 @@ files = [
[[package]]
name = "pyflakes"
-version = "3.0.1"
-requires_python = ">=3.6"
+version = "3.2.0"
+requires_python = ">=3.8"
summary = "passive checker of Python programs"
groups = ["lint"]
files = [
- {file = "pyflakes-3.0.1-py2.py3-none-any.whl", hash = "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf"},
- {file = "pyflakes-3.0.1.tar.gz", hash = "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd"},
+ {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"},
+ {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"},
]
[[package]]
@@ -3929,8 +3927,8 @@ files = [
[[package]]
name = "requests-oauthlib"
-version = "1.4.0"
-requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "2.0.0"
+requires_python = ">=3.4"
summary = "OAuthlib authentication support for Requests."
groups = ["hook-check-django-migrations"]
dependencies = [
@@ -3938,8 +3936,8 @@ dependencies = [
"requests>=2.0.0",
]
files = [
- {file = "requests-oauthlib-1.4.0.tar.gz", hash = "sha256:acee623221e4a39abcbb919312c8ff04bd44e7e417087fb4bd5e2a2f53d5e79a"},
- {file = "requests_oauthlib-1.4.0-py2.py3-none-any.whl", hash = "sha256:7a3130d94a17520169e38db6c8d75f2c974643788465ecc2e4b36d288bf13033"},
+ {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"},
+ {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"},
]
[[package]]
@@ -4244,7 +4242,7 @@ files = [
[[package]]
name = "smart-open"
-version = "7.0.2"
+version = "7.0.3"
requires_python = "<4.0,>=3.7"
summary = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)"
groups = ["hook-check-django-migrations"]
@@ -4252,8 +4250,8 @@ dependencies = [
"wrapt",
]
files = [
- {file = "smart_open-7.0.2-py3-none-any.whl", hash = "sha256:361a8fadfc9c1cfb81e8d18a96b724e1d6216f8f3ad543c841e19fc0c56a003a"},
- {file = "smart_open-7.0.2.tar.gz", hash = "sha256:f109517702453c3fa5ac60178698138b618d3670c2f1f0993ee8ce27193d3341"},
+ {file = "smart_open-7.0.3-py3-none-any.whl", hash = "sha256:7969ac4f1a04fd829d50f086fd106c6b84c8f68786666a4fac7bdc208f24f3b6"},
+ {file = "smart_open-7.0.3.tar.gz", hash = "sha256:dfea0d6815798f843fd2f4dbeb2ddf2d4a2304ece749c5251a97ac5f2dcb1a8e"},
]
[[package]]
@@ -4525,18 +4523,18 @@ files = [
[[package]]
name = "storage3"
-version = "0.7.3"
-requires_python = ">=3.8,<4.0"
+version = "0.7.4"
+requires_python = "<4.0,>=3.8"
summary = "Supabase Storage client for Python."
groups = ["hook-check-django-migrations"]
dependencies = [
- "httpx<0.26,>=0.24",
+ "httpx<0.28,>=0.24",
"python-dateutil<3.0.0,>=2.8.2",
"typing-extensions<5.0.0,>=4.2.0",
]
files = [
- {file = "storage3-0.7.3-py3-none-any.whl", hash = "sha256:dc6a59da801ee6fc00015da4967ac0b5c3e5508d31ffd796f0e4c83957e5c6a0"},
- {file = "storage3-0.7.3.tar.gz", hash = "sha256:943c31de4a7c7490ad7960d963a6b410979ebd0e1b3d320d76cb61564ab0b528"},
+ {file = "storage3-0.7.4-py3-none-any.whl", hash = "sha256:0b8e8839b10a64063796ce55a41462c7ffd6842e0ada74f25f5dcf37e1d1bade"},
+ {file = "storage3-0.7.4.tar.gz", hash = "sha256:61fcbf836f566405981722abb7d56caa57025b261e7a316e73316701abf0c040"},
]
[[package]]
@@ -4882,15 +4880,16 @@ files = [
[[package]]
name = "types-pyopenssl"
-version = "23.2.0.2"
+version = "24.0.0.20240311"
+requires_python = ">=3.8"
summary = "Typing stubs for pyOpenSSL"
groups = ["lint"]
dependencies = [
"cryptography>=35.0.0",
]
files = [
- {file = "types-pyOpenSSL-23.2.0.2.tar.gz", hash = "sha256:6a010dac9ecd42b582d7dd2cc3e9e40486b79b3b64bb2fffba1474ff96af906d"},
- {file = "types_pyOpenSSL-23.2.0.2-py3-none-any.whl", hash = "sha256:19536aa3debfbe25a918cf0d898e9f5fbbe6f3594a429da7914bf331deb1b342"},
+ {file = "types-pyOpenSSL-24.0.0.20240311.tar.gz", hash = "sha256:7bca00cfc4e7ef9c5d2663c6a1c068c35798e59670595439f6296e7ba3d58083"},
+ {file = "types_pyOpenSSL-24.0.0.20240311-py3-none-any.whl", hash = "sha256:6e8e8bfad34924067333232c93f7fc4b369856d8bea0d5c9d1808cb290ab1972"},
]
[[package]]
@@ -5246,13 +5245,13 @@ files = [
[[package]]
name = "validators"
-version = "0.23.2"
+version = "0.24.0"
requires_python = ">=3.8"
summary = "Python Data Validation for Humans™"
groups = ["hook-check-django-migrations"]
files = [
- {file = "validators-0.23.2-py3-none-any.whl", hash = "sha256:cb68e489f946f4934dbd3c203f08486c53493ab1c204f39d13b13c209f60e2c1"},
- {file = "validators-0.23.2.tar.gz", hash = "sha256:397ec4a1e86192e6e904b5afe8c12da0e6eaa3db088c0cc426e00cfee2ef07a8"},
+ {file = "validators-0.24.0-py3-none-any.whl", hash = "sha256:4a99eb368747e60900bae947418eb21e230ff4ff5e7b7944b9308c456d86da32"},
+ {file = "validators-0.24.0.tar.gz", hash = "sha256:cd23defb36de42d14e7559cf0757f761bb46b10d9de2998e6ef805f769d859e3"},
]
[[package]]
diff --git a/pyproject.toml b/pyproject.toml
index eefc92a74..6b2df0787 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,15 +12,15 @@ build-backend = "pdm.backend"
lint = [
"absolufy-imports~=0.3.1",
"autopep8~=2.0.2",
- "black~=23.3.0",
+ "black~=24.3.0",
"docutils~=0.20.1",
- "flake8~=6.0.0",
+ "flake8~=7.0.0",
"flake8-pyproject~=1.2.2",
- "isort~=5.12.0",
+ "isort~=5.13.2",
"mypy~=1.9.0",
- "pre-commit~=3.3.1",
+ "pre-commit~=3.6.2",
"types-PyMySQL~=1.1.0.1",
- "types-pyOpenSSL~=23.2.0.2",
+ "types-pyOpenSSL~=24.0.0.20240311",
"types-PyYAML~=6.0.12.12",
"types-redis~=4.6.0.3",
"types-requests~=2.31.0.6",
@@ -58,13 +58,6 @@ target-version = ['py39']
[tool.pycln]
all = true
-[tool.pyupgrade]
-keep_dict_typing = true
-
-[tool.flake8]
-max-line-length = 80
-exclude = "pyproject.toml"
-
[tool.isort]
line_length = 80
multi_line_output = 3
@@ -78,14 +71,10 @@ profile = "black"
DJANGO_SETTINGS_MODULE = "backend.settings.test_cases"
python_files = ["tests.py", "test_*.py", "*_tests.py"]
-# IMPORTANT!
-# Keep this same as mypy hook args in .pre-commit-config.yaml
[tool.mypy]
allow_subclassing_any = true
allow_untyped_decorators = true
check_untyped_defs = true
-# To use Unpack for typing kwargs
-enable_incomplete_feature = "Unpack"
exclude = '''(?x)^(
.*migrations/.*\.py|
backend/prompt/.*|
diff --git a/run-platform.sh b/run-platform.sh
index 2f21dddd5..1ec79c27a 100755
--- a/run-platform.sh
+++ b/run-platform.sh
@@ -5,6 +5,7 @@ set -o errexit # exit for any command failure"
# text color escape codes (\033 == \e but OSX doesn't respect the \e)
blue_text='\033[94m'
+green_text='\033[32m'
red_text='\033[31m'
default_text='\033[39m'
@@ -17,20 +18,42 @@ debug() {
fi
}
+check_dependencies() {
+ if ! command -v git &> /dev/null; then
+ echo "$red_text""git not found. Exiting.""$default_text"
+ exit 1
+ fi
+ if ! command -v python3 &> /dev/null; then
+ echo "$red_text""python3 not found. Exiting.""$default_text"
+ exit 1
+ fi
+ if ! command -v docker &> /dev/null; then
+ echo "$red_text""docker not found. Exiting.""$default_text"
+ exit 1
+ fi
+ # For 'docker compose' vs 'docker-compose', see https://stackoverflow.com/a/66526176.
+ if command -v docker compose &> /dev/null; then
+ docker_compose_cmd="docker compose"
+ elif command -v docker-compose &> /dev/null; then
+ docker_compose_cmd="docker-compose"
+ else
+ echo "$red_text""Both 'docker compose' and 'docker-compose' not found. Exiting.""$default_text"
+ exit 1
+ fi
+}
+
display_banner() {
# Make sure the console is huge
if test $(tput cols) -ge 64; then
- # Make it green!
- echo -e "\033[32m"
- echo -e " _ _ _ _ _____ _______ _____ _____ _______"
- echo -e "| | | | \ | |/ ____|__ __| __ \ /\ / ____|__ __|"
- echo -e "| | | | \| | (___ | | | |__) | / \ | | | |"
- echo -e "| | | | . |\___ \ | | | _ / / /\ \| | | |"
- echo -e "| |__| | |\ |____) | | | | | \ \ / ____ \ |____ | |"
- echo -e " \____/|_| \_|_____/ |_| |_| \_\/_/ \_\_____| |_|"
- echo -e " "
- # Make it less green
- echo -e "\033[0m"
+ echo " █████ █████"
+ echo "░░███ ░░███ "
+ echo " ░███ ░███ "
+ echo " ░███ ░███ "
+ echo " ░███ ░███ "
+ echo " ░███ ░███ "
+ echo " ░░█████████ >UNSTRACT COMMUNITY EDITION"
+ echo " ░░░░░░░░░ "
+ echo ""
sleep 1
fi
}
@@ -40,11 +63,11 @@ display_help() {
echo
echo -e "Syntax: $0 [options]"
echo -e "Options:"
- echo -e " -h, --help Displays the help information"
+ echo -e " -h, --help Display help information"
echo -e " -e, --only-env Only do env files setup"
echo -e " -p, --only-pull Only do docker images pull"
echo -e " -b, --build-local Build docker images locally"
- echo -e " -d, --detach Run docker containers in detached mode"
+ echo -e " -u, --upgrade Upgrade services"
echo -e " -x, --trace Enables trace mode"
echo -e " -V, --verbose Print verbose logs"
echo -e " -v, --version Docker images version tag (default \"latest\")"
@@ -68,8 +91,8 @@ parse_args() {
-b | --build-local)
opt_build_local=true
;;
- -d | --detach)
- opt_detach="-d"
+ -u | --upgrade)
+ opt_upgrade=true
;;
-x | --trace)
set -o xtrace # display every line before execution; enables PS4
@@ -101,26 +124,21 @@ parse_args() {
debug "OPTION only_env: $opt_only_env"
debug "OPTION only_pull: $opt_only_pull"
debug "OPTION build_local: $opt_build_local"
- debug "OPTION detach: $opt_detach"
+ debug "OPTION upgrade: $opt_upgrade"
debug "OPTION verbose: $opt_verbose"
debug "OPTION version: $opt_version"
}
-check_required_packages() {
- echo "Checking if the required packages are installed..."
- if ! command -v docker &> /dev/null; then
- echo "$red_text""docker not found. Please install it and try again.""$default_text"
- exit 1
- fi
- if ! (command -v docker compose &> /dev/null && command -v docker-compose &> /dev/null); then
- echo "$red_text""docker compose not found. Please install it and try again.""$default_text"
- exit 1
- fi
- if ! command -v python3 &> /dev/null; then
- echo "$red_text""python3 not found. Please install it and try again.""$default_text"
- exit 1
+do_git_pull() {
+ if [ "$opt_upgrade" = false ]; then
+ return
fi
+ echo -e "Performing git switch to ""$blue_text""main branch""$default_text".
+ # git switch main
+
+ echo -e "Performing ""$blue_text""git pull""$default_text"" on main branch."
+ git pull
}
setup_env() {
@@ -137,17 +155,30 @@ setup_env() {
# Add encryption secret for backend and platform-service.
if [[ "$service" == "backend" || "$service" == "platform-service" ]]; then
echo -e "$blue_text""Adding encryption secret to $service""$default_text"
- echo "ENCRYPTION_KEY=\"$ENCRYPTION_KEY\"" >> $env_path
+ if [[ "$OSTYPE" == "darwin"* ]]; then
+ sed -i '' "s/ENCRYPTION_KEY.*/ENCRYPTION_KEY=\"$ENCRYPTION_KEY\"/" $env_path
+ else
+ sed -i "s/ENCRYPTION_KEY.*/ENCRYPTION_KEY=\"$ENCRYPTION_KEY\"/" $env_path
+ fi
fi
# Add default auth credentials for backend.
if [ "$service" == "backend" ]; then
echo -e "$blue_text""Adding default auth credentials to $service""$default_text"
- echo "DEFAULT_AUTH_USERNAME=\"$DEFAULT_AUTH_KEY\"" >> $env_path
- echo "DEFAULT_AUTH_PASSWORD=\"$DEFAULT_AUTH_KEY\"" >> $env_path
+ if [[ "$OSTYPE" == "darwin"* ]]; then
+ sed -i '' "s/DEFAULT_AUTH_USERNAME.*/DEFAULT_AUTH_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path
+ sed -i '' "s/DEFAULT_AUTH_PASSWORD.*/DEFAULT_AUTH_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path
+ else
+ sed -i "s/DEFAULT_AUTH_USERNAME.*/DEFAULT_AUTH_USERNAME=\"$DEFAULT_AUTH_KEY\"/" $env_path
+ sed -i "s/DEFAULT_AUTH_PASSWORD.*/DEFAULT_AUTH_PASSWORD=\"$DEFAULT_AUTH_KEY\"/" $env_path
+ fi
fi
echo -e "Created env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"."
else
- echo -e "Found existing env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"."
+ python3 $script_dir/docker/scripts/merge_env.py $sample_env_path $env_path
+ if [ $? -ne 0 ]; then
+ exit 1
+ fi
+ echo -e "Merged env for ""$blue_text""$service""$default_text" at ""$blue_text""$env_path""$default_text"."
fi
done
@@ -155,7 +186,11 @@ setup_env() {
cp "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env"
echo -e "Created env for ""$blue_text""essential services""$default_text"" at ""$blue_text""$script_dir/docker/essentials.env""$default_text""."
else
- echo -e "Found existing env for ""$blue_text""essential services""$default_text"" at ""$blue_text""$script_dir/docker/essentials.env""$default_text""."
+ python3 $script_dir/docker/scripts/merge_env.py "$script_dir/docker/sample.essentials.env" "$script_dir/docker/essentials.env"
+ if [ $? -ne 0 ]; then
+ exit 1
+ fi
+ echo -e "Merged env for ""$blue_text""essential services""$default_text"" at ""$blue_text""$script_dir/docker/essentials.env""$default_text""."
fi
if [ ! -e "$script_dir/docker/proxy_overrides.yaml" ]; then
@@ -165,7 +200,7 @@ setup_env() {
fi
if [ "$opt_only_env" = true ]; then
- echo "Done." && exit 0
+ echo -e "$green_text""Done.""$default_text" && exit 0
fi
}
@@ -173,43 +208,46 @@ build_services() {
pushd ${script_dir}/docker 1>/dev/null
if [ "$opt_build_local" = true ]; then
- echo "Building docker images locally."
- VERSION=$opt_version docker-compose -f $script_dir/docker/docker-compose.build.yaml build || {
+ echo -e "$blue_text""Building""$default_text"" docker images ""$blue_text""$opt_version""$default_text"" locally."
+ VERSION=$opt_version $docker_compose_cmd -f $script_dir/docker/docker-compose.build.yaml build || {
echo -e "$red_text""Failed to build docker images.""$default_text"
exit 1
}
else
- for service in "${services[@]}"; do
- if ! docker image inspect "unstract/${service}:$opt_version" &> /dev/null; then
- echo -e "$red_text""Docker image 'unstract/${service}:$opt_version' not found. Pulling...""$default_text"
- docker pull unstract/${service}:$opt_version || {
- echo -e "$red_text""Failed to pull 'unstract/${service}:$opt_version'. Check if version exists.""$default_text"
- exit 1
- }
- else
- echo -e "Found docker image ""$blue_text""'unstract/${service}:$opt_version'""$default_text""."
- fi
- done
+ echo -e "$blue_text""Pulling""$default_text"" docker images tag ""$blue_text""$opt_version""$default_text""."
+
+ pull_policy="missing"
+ if [ "$opt_upgrade" = true ] && [ "$opt_version" = "latest" ]; then
+ pull_policy="always"
+ fi
+
+ VERSION=$opt_version $docker_compose_cmd -f $script_dir/docker/docker-compose.yaml pull --policy $pull_policy || {
+ echo -e "$red_text""Failed to pull docker images. Check the version.""$default_text"
+ echo -e "$red_text""Also make sure docker is running and try again.""$default_text"
+ exit 1
+ }
fi
popd 1>/dev/null
if [ "$opt_only_pull" = true ]; then
- echo "Done." && exit 0
+ echo -e "$green_text""Done.""$default_text" && exit 0
fi
}
run_services() {
pushd ${script_dir}/docker 1>/dev/null
- if [ -z "$opt_detach" ]; then
- echo -e "$blue_text""Starting docker containers""$default_text"
- else
- echo -e "$blue_text""Starting docker containers in detach mode""$default_text"
- fi
- echo -e "Once the services are up, visit ""$blue_text""http://frontend.unstract.localhost""$default_text"" in your browser."
+ echo -e "$blue_text""Starting docker containers in detached mode""$default_text"
+ VERSION=$opt_version $docker_compose_cmd up -d
- VERSION=$opt_version docker compose up $opt_detach
+ if [ "$opt_upgrade" = true ]; then
+ echo ""
+ echo -e "$green_text""Upgraded platform to $opt_version version.""$default_text"
+ fi
+ echo -e "\nOnce the services are up, visit ""$blue_text""http://frontend.unstract.localhost""$default_text"" in your browser."
+ echo "See logs with:"
+ echo -e " ""$blue_text""$docker_compose_cmd -f docker/docker-compose.yaml logs -f""$default_text"
popd 1>/dev/null
}
@@ -217,22 +255,23 @@ run_services() {
#
# Run Unstract platform - BEGIN
#
-check_required_packages
+check_dependencies
opt_only_env=false
opt_only_pull=false
opt_build_local=false
-opt_detach=""
+opt_upgrade=false
opt_verbose=false
opt_version="latest"
script_dir=$(dirname "$(readlink -f "$BASH_SOURCE")")
# Extract service names from docker compose file.
-services=($(VERSION=$opt_version docker compose -f $script_dir/docker/docker-compose.build.yaml config --services))
+services=($(VERSION=$opt_version $docker_compose_cmd -f $script_dir/docker/docker-compose.build.yaml config --services))
display_banner
parse_args $*
+do_git_pull
setup_env
build_services
run_services