LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: [OSS] Schema creation moved to DB init (#864)
Chandrasekharan M2d2915419468
diff --git a/.gitignore b/.gitignoreindex 9128508d7..0f029490b 100644--- a/.gitignore+++ b/.gitignore@@ -632,7 +632,6 @@ unstract/tool-registry/tool_registry_config/registry.yaml# Docker related# End of https://www.toptal.com/developers/gitignore/api/windows,macos,linux,pycharm,pycharm+all,pycharm+iml,python,visualstudiocode,react,djangodocker/temp/*-docker/init.sql/*docker/*.env!docker/sample*.envdocker/public_tools.jsondiff --git a/backend/entrypoint.sh b/backend/entrypoint.shindex 1d58e8e27..686755124 100755--- a/backend/entrypoint.sh+++ b/backend/entrypoint.sh@@ -4,11 +4,6 @@ cmd=$1if [ "$cmd" = "migrate" ]; thenecho "Migration initiated".venv/bin/python manage.py migrate-elif [ "$cmd" = "prepare_and_migrate" ]; then- echo "Creating schema in database"- .venv/bin/python manage.py create_schema- echo "Migration initiated"- .venv/bin/python manage.py migratefi# NOTE: Leaving below for reference incase required in the futurediff --git a/backend/init.sql b/backend/init.sqldeleted file mode 100644index 6ae1249d8..000000000--- a/backend/init.sql+++ /dev/null@@ -1,6 +0,0 @@-ALTER ROLE unstract_dev SET client_encoding TO 'utf8';-ALTER ROLE unstract_dev SET default_transaction_isolation TO 'read committed';-ALTER ROLE unstract_dev SET timezone TO 'UTC';-ALTER USER unstract_dev CREATEDB;-GRANT ALL PRIVILEGES ON DATABASE unstract_db TO unstract_dev;-CREATE DATABASE unstract;diff --git a/docker/docker-compose-dev-essentials.yaml b/docker/docker-compose-dev-essentials.yamlindex a0c231f66..7ad99fdba 100644--- a/docker/docker-compose-dev-essentials.yaml+++ b/docker/docker-compose-dev-essentials.yaml@@ -9,7 +9,7 @@ services:- "5432:5432"volumes:- postgres_data:/var/lib/postgresql/data/- - ../backend/init.sql:/docker-entrypoint-initdb.d/init.sql+ - ./scripts/db-setup/db_setup.sh:/docker-entrypoint-initdb.d/db_setup.shenv_file:- ./essentials.envlabels:diff --git a/docker/sample.essentials.env b/docker/sample.essentials.envindex b5c6c9ff0..d8bc2ab04 100644--- a/docker/sample.essentials.env+++ b/docker/sample.essentials.env@@ -1,7 +1,9 @@-+# Refer https://hub.docker.com/_/postgres#:~:text=How%20to%20extend%20this%20imagePOSTGRES_USER=unstract_devPOSTGRES_PASSWORD=unstract_passPOSTGRES_DB=unstract_db+# Used by db setup script+POSTGRES_SCHEMA=unstractMINIO_ROOT_USER=minioMINIO_ROOT_PASSWORD=minio123diff --git a/docker/scripts/db-setup/README.md b/docker/scripts/db-setup/README.mdnew file mode 100644index 000000000..e24b9c875--- /dev/null+++ b/docker/scripts/db-setup/README.md@@ -0,0 +1,9 @@+# Unstract DB Setup Script++[The db_setup.sh](/docker/scripts/db-setup/db_setup.sh) script helps setup the postgres database by making use of environment variables defined in the `.essentials.env` (user copy of the [sample.essentials.env](/docker/sample.essentials.env))++- POSTGRES_USER+- POSTGRES_DB+- POSTGRES_SCHEMA++This script helps setup the DB user and creates a new schema as well.diff --git a/docker/scripts/db-setup/db_setup.sh b/docker/scripts/db-setup/db_setup.shnew file mode 100755index 000000000..60c7b1cde--- /dev/null+++ b/docker/scripts/db-setup/db_setup.sh@@ -0,0 +1,12 @@+#!/bin/bash+echo "Creating DB '$POSTGRES_DB' and schema '$POSTGRES_SCHEMA' with user '$POSTGRES_USER'"++psql -U ${POSTGRES_USER} -d ${POSTGRES_DB}<<-END+ ALTER ROLE ${POSTGRES_USER} SET client_encoding TO 'utf8';+ ALTER ROLE ${POSTGRES_USER} SET default_transaction_isolation TO 'read committed';+ ALTER ROLE ${POSTGRES_USER} SET timezone TO 'UTC';+ ALTER USER ${POSTGRES_USER} CREATEDB;+ CREATE DATABASE ${POSTGRES_DB};+ GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} TO ${POSTGRES_USER};+ CREATE SCHEMA IF NOT EXISTS ${POSTGRES_SCHEMA}+ENDdiff --git a/run-platform.sh b/run-platform.shindex c906bf048..87e9ff6f2 100755--- a/run-platform.sh+++ b/run-platform.sh@@ -268,20 +268,6 @@ build_services() {fi}-create_backend_schema() {-- if [ "$first_setup" = false ]; then- return- fi-- pushd ${script_dir}/docker 1>/dev/null-- echo -e "$blue_text""Creating a schema for Unstract in the database""$default_text"- VERSION=$opt_version $docker_compose_cmd run backend prepare_and_migrate- # TODO: Run migrations here once its removed from backend's entrypoint-- popd 1>/dev/null-}run_services() {pushd ${script_dir}/docker 1>/dev/null@@ -345,7 +331,6 @@ parse_args $*do_git_pullsetup_envbuild_services-create_backend_schemarun_services## Run Unstract platform - END