LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Added script to update lock files (#764)
Chandrasekharan Ma0d798ae80de
diff --git a/.github/workflows/pdm-lock-automation.yaml b/.github/workflows/pdm-lock-automation.yamlindex db276f913..bd65e5546 100644--- a/.github/workflows/pdm-lock-automation.yaml+++ b/.github/workflows/pdm-lock-automation.yaml@@ -5,39 +5,23 @@ on:types: [opened, synchronize, reopened, ready_for_review, review_requested]branches: [main]paths:- - 'pyproject.toml'- - 'backend/pyproject.toml'- - 'prompt-service/pyproject.toml'- - 'worker/pyproject.toml'- - 'unstract/core/pyproject.toml'- - 'unstract/flags/pyproject.toml'- - 'platform-service/pyproject.toml'- - 'x2text-service/pyproject.toml'- - 'unstract/connectors/pyproject.toml'- # - 'unstract/tool-registry' # commenting due to transitive dependency on local libraries/pyproject.toml- - 'unstract/tool-sandbox/pyproject.toml'- # - 'unstract/workflow-execution/pyproject.toml' # commenting due to transitive dependency on local libraries+ - '**/pyproject.toml'workflow_dispatch:+ inputs:+ directories:+ description: 'Comma-separated list of directories to update'+ required: false+ default: '' # Run for all dirs specified in scripts/pdm-lock.shjobs:- check_pyproject_changes:- name: ${{ matrix.directory }} PDM lock update+ update_pdm_lock:+ name: Update PDM lock in all directoriesruns-on: ubuntu-latest- strategy:- matrix:- directory:- - root- - backend- - prompt-service- - worker- - unstract/core- - unstract/flags- - platform-service- - x2text-service- - unstract/connectors- # - unstract/tool-registry # commenting due to transitive dependency on local libraries- - unstract/tool-sandbox- # - unstract/workflow-execution # commenting due to transitive dependency on local libraries++ permissions:+ # Give the default GITHUB_TOKEN write permission to commit and push the+ # added or changed files to the repository.+ contents: writesteps:- name: Checkout repository@@ -45,87 +29,44 @@ jobs:with:ref: ${{ github.head_ref }}-- - name: Check file changes- id: check- env:- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}- run: |- if [ "${{ matrix.directory }}" = "root" ]; then- file_path="pyproject.toml"- else- file_path="${{ matrix.directory }}/pyproject.toml"- fi- echo "checking changes for $file_path for origin/main branch"- echo "changed=false" >> $GITHUB_OUTPUT- git config --global user.name 'github-actions[bot]'- git config --global user.email 'github-actions[bot]@users.noreply.github.com'- git fetch origin- if git diff --quiet origin/main -- "$file_path"; then- echo "No changes detected in $file_path against origin/main branch."- else- echo "Changes detected in $file_path against origin/main branch."- echo "changed=true" >> $GITHUB_OUTPUT- fi-- name: Set up Python- if: steps.check.outputs.changed == 'true'uses: actions/setup-python@v4with:python-version: '3.9'- name: Install PDM- if: steps.check.outputs.changed == 'true'run: python -m pip install pdm==2.16.1- - name: Update and Push pdm.lock- if: steps.check.outputs.changed == 'true'+ - name: Generate PDM lockfilesenv:- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}run: |- if [ "${{ matrix.directory }}" != "root" ]; then- cd ${{ matrix.directory }}- fi+ chmod +x ./scripts/pdm-lock.sh- if [ ! -d ".venv" ]; then- echo 'Creating virtual environment inside "${{ matrix.directory }}".'- pdm venv create -w virtualenv --with-pip- else- echo "Virtual environment already exists."- fi-- source .venv/bin/activate-- echo "Generating/Updating pdm.lock..."- pdm lock -G :all -v-- echo "Checking pdm lock..."- pdm lock --check-- echo "Stashing changes if there are any..."- git stash || echo "No changes to stash, continuing..."+ # Get the input from the workflow or use the default value+ dirs="${{ github.event.inputs.directories }}"- echo "Pulling remote ${{ github.head_ref }} branch..."- git pull origin ${{ github.head_ref }} --allow-unrelated-histories --no-rebase+ # Check if directories input is empty+ if [[ -z "$dirs" ]]; then+ # No directories input given, run the script without arguments (process all directories)+ echo "No directories specified, running on all dirs listed in scripts/pdm-lock.sh"- echo "Trying to pop stash if any..."- if git stash list | grep -q 'stash@{0}'; then- echo "Popping stash..."- git stash pop || echo "Failed to pop stash. Continue with other changes."+ ./scripts/pdm-lock.shelse- echo "No stash found, skipping pop."- fi-- if git diff --quiet; then- echo "No changes between the current branch and the ${{ github.head_ref }} branch."- echo "Nothing to commit"- else- echo "Changes detected between the current branch and the ${{ github.head_ref }} branch."+ # Convert comma-separated list into an array of directories+ IFS=',' read -r -a dir_array <<< "$dirs"- echo "Adding pdm lock changes..."- git add pdm.lock+ # Print directories being processed+ echo "Processing specified directories: ${dir_array[@]}"- echo "Committing and pushing pdm lock ..."- git commit -m "Update pdm.lock for ${{ matrix.directory }}"- git push origin HEAD:${{ github.head_ref }}+ # Pass directories as command-line arguments to the script+ ./scripts/pdm-lock.sh "${dir_array[@]}"fi+ shell: bash++ - name: Commit pdm.lock changes+ uses: stefanzweifel/git-auto-commit-action@v5+ with:+ commit_message: Commit pdm.lock changes+ commit_user_name: pdm-lock-automation[bot]+ commit_user_email: [email protected]diff --git a/scripts/README.md b/scripts/README.mdnew file mode 100644index 000000000..6ddf702e7--- /dev/null+++ b/scripts/README.md@@ -0,0 +1,26 @@+# Developer Scripts++## pdm-lock.sh++Helps generate pdm's lockfiles by running the command `pdm lock -G :all -v` on all necessary packages and services.++- project root+- `backend`+- `prompt-service`+- `worker`+- `unstract/core`+- `unstract/flags`+- `platform-service`+- `x2text-service"`+- `unstract/connectors`+- `unstract/tool-sandbox`++Can be run without any arguments to check for lockfile generation on all necessary packages and services+```shell+./pdm-lock.sh+```++Accepts a list of directories to generate for as command line arguments+```shell+./pdm-lock.sh backend prompt-service+```diff --git a/scripts/pdm-lock.sh b/scripts/pdm-lock.shnew file mode 100755index 000000000..df050beb5--- /dev/null+++ b/scripts/pdm-lock.sh@@ -0,0 +1,80 @@+#!/bin/bash++# Function to update the lockfile in a directory+update_lockfile() {+ dir="$1"+ file_path="$dir/pyproject.toml"++ if [[ ! -f "$file_path" ]]; then+ echo "[$dir] No pyproject.toml found in $dir"+ return+ fi++ echo "[$dir] Checking $file_path for changes against origin/main..."+ if ! git diff --quiet origin/main -- "$file_path"; then+ echo "[$dir] Changes detected in $file_path, updating pdm.lock for $dir..."++ # Move to the directory if it's not root+ if [[ "$dir" != "." ]]; then+ cd "$dir"+ fi++ # Set up virtual environment if not exists+ if [[ ! -d ".venv" ]]; then+ echo '[$dir] Creating virtual environment in directory: '"$dir"+ pdm venv create -w virtualenv --with-pip+ else+ echo "[$dir] Virtual environment already exists in $dir"+ fi++ # Activate virtual environment+ source .venv/bin/activate++ # Perform lockfile update if required+ if pdm lock --check; then+ echo "[$dir] No changes required for pdm.lock in $dir"+ else+ echo "[$dir] Updating pdm.lock in $dir..."+ pdm lock -G :all -v+ fi++ # Go back to root if moved to a subdirectory+ if [[ "$dir" != "." ]]; then+ cd -+ fi+ else+ echo "[$dir] No changes detected in $file_path"+ fi+}+++# Default directories list+directories=(+ "."+ "backend"+ "prompt-service"+ "worker"+ "unstract/core"+ "unstract/flags"+ "platform-service"+ "x2text-service"+ "unstract/connectors"+ "unstract/tool-sandbox"+)++# If directories are passed as arguments, override the default+if [ "$#" -gt 0 ]; then+ directories=("$@")+fi++# To compare against main branch+git fetch origin main+++# Run lockfile updates in parallel+for dir in "${directories[@]}"; do+ update_lockfile "$dir" &+done++# Wait for all background processes to complete+wait