Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

feat: Added script to update lock files (#764)

Chandrasekharan Ma0d798ae80de
diff --git a/.github/workflows/pdm-lock-automation.yaml b/.github/workflows/pdm-lock-automation.yaml
index db276f913..bd65e5546 100644
--- a/.github/workflows/pdm-lock-automation.yaml
+++ b/.github/workflows/pdm-lock-automation.yaml
@@ -5,39 +5,23 @@ on:
types: [opened, synchronize, reopened, ready_for_review, review_requested]
branches: [main]
paths:
- - 'pyproject.toml'
- - 'backend/pyproject.toml'
- - 'prompt-service/pyproject.toml'
- - 'worker/pyproject.toml'
- - 'unstract/core/pyproject.toml'
- - 'unstract/flags/pyproject.toml'
- - 'platform-service/pyproject.toml'
- - 'x2text-service/pyproject.toml'
- - 'unstract/connectors/pyproject.toml'
- # - 'unstract/tool-registry' # commenting due to transitive dependency on local libraries/pyproject.toml
- - 'unstract/tool-sandbox/pyproject.toml'
- # - 'unstract/workflow-execution/pyproject.toml' # commenting due to transitive dependency on local libraries
+ - '**/pyproject.toml'
workflow_dispatch:
+ inputs:
+ directories:
+ description: 'Comma-separated list of directories to update'
+ required: false
+ default: '' # Run for all dirs specified in scripts/pdm-lock.sh
jobs:
- check_pyproject_changes:
- name: ${{ matrix.directory }} PDM lock update
+ update_pdm_lock:
+ name: Update PDM lock in all directories
runs-on: ubuntu-latest
- strategy:
- matrix:
- directory:
- - root
- - backend
- - prompt-service
- - worker
- - unstract/core
- - unstract/flags
- - platform-service
- - x2text-service
- - unstract/connectors
- # - unstract/tool-registry # commenting due to transitive dependency on local libraries
- - unstract/tool-sandbox
- # - unstract/workflow-execution # commenting due to transitive dependency on local libraries
+
+ permissions:
+ # Give the default GITHUB_TOKEN write permission to commit and push the
+ # added or changed files to the repository.
+ contents: write
steps:
- name: Checkout repository
@@ -45,87 +29,44 @@ jobs:
with:
ref: ${{ github.head_ref }}
-
- - name: Check file changes
- id: check
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- if [ "${{ matrix.directory }}" = "root" ]; then
- file_path="pyproject.toml"
- else
- file_path="${{ matrix.directory }}/pyproject.toml"
- fi
- echo "checking changes for $file_path for origin/main branch"
- echo "changed=false" >> $GITHUB_OUTPUT
- git config --global user.name 'github-actions[bot]'
- git config --global user.email 'github-actions[bot]@users.noreply.github.com'
- git fetch origin
- if git diff --quiet origin/main -- "$file_path"; then
- echo "No changes detected in $file_path against origin/main branch."
- else
- echo "Changes detected in $file_path against origin/main branch."
- echo "changed=true" >> $GITHUB_OUTPUT
- fi
-
- name: Set up Python
- if: steps.check.outputs.changed == 'true'
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Install PDM
- if: steps.check.outputs.changed == 'true'
run: python -m pip install pdm==2.16.1
- - name: Update and Push pdm.lock
- if: steps.check.outputs.changed == 'true'
+ - name: Generate PDM lockfiles
env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
- if [ "${{ matrix.directory }}" != "root" ]; then
- cd ${{ matrix.directory }}
- fi
+ chmod +x ./scripts/pdm-lock.sh
- if [ ! -d ".venv" ]; then
- echo 'Creating virtual environment inside "${{ matrix.directory }}".'
- pdm venv create -w virtualenv --with-pip
- else
- echo "Virtual environment already exists."
- fi
-
- source .venv/bin/activate
-
- echo "Generating/Updating pdm.lock..."
- pdm lock -G :all -v
-
- echo "Checking pdm lock..."
- pdm lock --check
-
- echo "Stashing changes if there are any..."
- git stash || echo "No changes to stash, continuing..."
+ # Get the input from the workflow or use the default value
+ dirs="${{ github.event.inputs.directories }}"
- echo "Pulling remote ${{ github.head_ref }} branch..."
- git pull origin ${{ github.head_ref }} --allow-unrelated-histories --no-rebase
+ # Check if directories input is empty
+ if [[ -z "$dirs" ]]; then
+ # No directories input given, run the script without arguments (process all directories)
+ echo "No directories specified, running on all dirs listed in scripts/pdm-lock.sh"
- echo "Trying to pop stash if any..."
- if git stash list | grep -q 'stash@{0}'; then
- echo "Popping stash..."
- git stash pop || echo "Failed to pop stash. Continue with other changes."
+ ./scripts/pdm-lock.sh
else
- echo "No stash found, skipping pop."
- fi
-
- if git diff --quiet; then
- echo "No changes between the current branch and the ${{ github.head_ref }} branch."
- echo "Nothing to commit"
- else
- echo "Changes detected between the current branch and the ${{ github.head_ref }} branch."
+ # Convert comma-separated list into an array of directories
+ IFS=',' read -r -a dir_array <<< "$dirs"
- echo "Adding pdm lock changes..."
- git add pdm.lock
+ # Print directories being processed
+ echo "Processing specified directories: ${dir_array[@]}"
- echo "Committing and pushing pdm lock ..."
- git commit -m "Update pdm.lock for ${{ matrix.directory }}"
- git push origin HEAD:${{ github.head_ref }}
+ # Pass directories as command-line arguments to the script
+ ./scripts/pdm-lock.sh "${dir_array[@]}"
fi
+ shell: bash
+
+ - name: Commit pdm.lock changes
+ uses: stefanzweifel/git-auto-commit-action@v5
+ with:
+ commit_message: Commit pdm.lock changes
+ commit_user_name: pdm-lock-automation[bot]
+ commit_user_email: [email protected]
diff --git a/scripts/README.md b/scripts/README.md
new file mode 100644
index 000000000..6ddf702e7
--- /dev/null
+++ b/scripts/README.md
@@ -0,0 +1,26 @@
+# Developer Scripts
+
+## pdm-lock.sh
+
+Helps generate pdm's lockfiles by running the command `pdm lock -G :all -v` on all necessary packages and services.
+
+- project root
+- `backend`
+- `prompt-service`
+- `worker`
+- `unstract/core`
+- `unstract/flags`
+- `platform-service`
+- `x2text-service"`
+- `unstract/connectors`
+- `unstract/tool-sandbox`
+
+Can be run without any arguments to check for lockfile generation on all necessary packages and services
+```shell
+./pdm-lock.sh
+```
+
+Accepts a list of directories to generate for as command line arguments
+```shell
+./pdm-lock.sh backend prompt-service
+```
diff --git a/scripts/pdm-lock.sh b/scripts/pdm-lock.sh
new file mode 100755
index 000000000..df050beb5
--- /dev/null
+++ b/scripts/pdm-lock.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Function to update the lockfile in a directory
+update_lockfile() {
+ dir="$1"
+ file_path="$dir/pyproject.toml"
+
+ if [[ ! -f "$file_path" ]]; then
+ echo "[$dir] No pyproject.toml found in $dir"
+ return
+ fi
+
+ echo "[$dir] Checking $file_path for changes against origin/main..."
+ if ! git diff --quiet origin/main -- "$file_path"; then
+ echo "[$dir] Changes detected in $file_path, updating pdm.lock for $dir..."
+
+ # Move to the directory if it's not root
+ if [[ "$dir" != "." ]]; then
+ cd "$dir"
+ fi
+
+ # Set up virtual environment if not exists
+ if [[ ! -d ".venv" ]]; then
+ echo '[$dir] Creating virtual environment in directory: '"$dir"
+ pdm venv create -w virtualenv --with-pip
+ else
+ echo "[$dir] Virtual environment already exists in $dir"
+ fi
+
+ # Activate virtual environment
+ source .venv/bin/activate
+
+ # Perform lockfile update if required
+ if pdm lock --check; then
+ echo "[$dir] No changes required for pdm.lock in $dir"
+ else
+ echo "[$dir] Updating pdm.lock in $dir..."
+ pdm lock -G :all -v
+ fi
+
+ # Go back to root if moved to a subdirectory
+ if [[ "$dir" != "." ]]; then
+ cd -
+ fi
+ else
+ echo "[$dir] No changes detected in $file_path"
+ fi
+}
+
+
+# Default directories list
+directories=(
+ "."
+ "backend"
+ "prompt-service"
+ "worker"
+ "unstract/core"
+ "unstract/flags"
+ "platform-service"
+ "x2text-service"
+ "unstract/connectors"
+ "unstract/tool-sandbox"
+)
+
+# If directories are passed as arguments, override the default
+if [ "$#" -gt 0 ]; then
+ directories=("$@")
+fi
+
+# To compare against main branch
+git fetch origin main
+
+
+# Run lockfile updates in parallel
+for dir in "${directories[@]}"; do
+ update_lockfile "$dir" &
+done
+
+# Wait for all background processes to complete
+wait