LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
#!/bin/bash
# Script to automate dependency version bumps in the Unstract project
# Author: Cascade
# Date: 2025-07-09
set -e
# Default paths
UNSTRACT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
BACKEND_DIR="${UNSTRACT_ROOT}/backend"
PLATFORM_SERVICE_DIR="${UNSTRACT_ROOT}/platform-service"
FILESYSTEM_DIR="${UNSTRACT_ROOT}/unstract/filesystem"
TOOL_REGISTRY_DIR="${UNSTRACT_ROOT}/unstract/tool-registry"
CLASSIFIER_DIR="${UNSTRACT_ROOT}/tools/classifier"
TEXT_EXTRACTOR_DIR="${UNSTRACT_ROOT}/tools/text_extractor"
# Organized directory arrays
CUSTOM_TOOL_DIRS=("$CLASSIFIER_DIR" "$TEXT_EXTRACTOR_DIR")
PACKAGE_DIRS=("$FILESYSTEM_DIR" "$TOOL_REGISTRY_DIR")
SERVICE_DIRS=("$BACKEND_DIR" "$PLATFORM_SERVICE_DIR")
ROOT_DIR="$UNSTRACT_ROOT"
# Additional files having version
SAMPLE_ENV_FILE="${BACKEND_DIR}/sample.env"
TOOL_REGISTRY_JSON_FILE="${TOOL_REGISTRY_DIR}/tool_registry_config/public_tools.json"
# Global variables
VERBOSE=false
DRY_RUN=false
RESET_MODE=false
BUMP_MODE=false
TARGET_VERSION=""
# Version bumping functions
bump_version() {
local version="$1"
local bump_type="$2"
# If the bump_type is not a special keyword, return it as is
if [[ "$bump_type" != "patch" && "$bump_type" != "minor" && "$bump_type" != "major" ]]; then
echo "$bump_type"
return
fi
# Split version into major, minor, patch
IFS='.' read -r major minor patch <<< "$version"
# Handle special bump types
case "$bump_type" in
patch)
patch=$((patch + 1))
;;
minor)
minor=$((minor + 1))
patch=0
;;
major)
major=$((major + 1))
minor=0
patch=0
;;
esac
echo "$major.$minor.$patch"
}
# Display usage information
usage() {
echo "Usage: $0 --bump [options] | --reset [options]"
echo ""
echo "This script automates semantic version bumps for Unstract SDK and related tools."
echo ""
echo "Core Commands:"
echo " --bump Perform the version bump."
echo " --target-version <version> Required with --bump. Specifies the new version."
echo " Accepts 'major', 'minor', 'patch', or a specific version (e.g., 1.2.3)."
echo " --reset Reset all changes made by this script."
echo ""
echo "Options:"
echo " -v, --verbose Enable verbose logging."
echo " --dry-run Run the script without making any changes."
echo " -h, --help Display this help message."
echo ""
echo "Examples:"
echo " $0 --bump --verbose # Update with verbose output"
echo " $0 --bump --dry-run # Preview changes without applying them"
echo " $0 --reset --dry-run # Preview what files would be reset"
echo " $0 --dry-run # Preview changes without applying them"
echo " $0 --reset # Reset all files modified by this script"
echo " $0 --reset --dry-run # Preview what files would be reset"
exit 1
}
# Parse command-line arguments
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--bump)
BUMP_MODE=true
shift
;;
--target-version)
if [[ -z "$2" || "$2" == -* ]]; then
echo "Error: --target-version requires an argument."
usage
exit 1
fi
TARGET_VERSION="$2"
shift 2
;;
-v|--verbose)
VERBOSE=true
shift
;;
-d|--dry-run)
DRY_RUN=true
shift
;;
--reset)
RESET_MODE=true
shift
;;
-h|--help)
usage
;;
*)
echo "Unknown option: $1"
usage
;;
esac
done
}
# Log messages when verbose mode is enabled
log() {
if [[ "$VERBOSE" == true ]]; then
echo " DEBUG: $1"
fi
}
# Check if a file exists
check_file() {
if [[ ! -f "$1" ]]; then
echo "Error: File not found: $1"
exit 1
fi
}
# Update custom tool version and its SDK dependency
update_custom_tool_version() {
local dir="$1"
local sdk_version_arg="$2"
local registry_json_file="$3"
local tool_name=$(basename "$dir")
local properties_json_file="${dir}/src/config/properties.json"
local requirements_file="${dir}/requirements.txt"
# NOTE: Currently we bump only patch version for tools
local tool_version_arg="patch"
local tool_old_version
local tool_new_version
echo "Processing custom tool: $tool_name"
log "Checking $tool_name version in $properties_json_file"
check_file "$properties_json_file"
# tool_version_arg="$version_arg"
# if [[ "$version_arg" != *"."* ]]; then
# tool_version_arg="patch"
# echo "Set target version for custom tool $tool_name as $version_arg"
# fi
# Fetch tool's version from properties.json
tool_old_version=$(grep -o '"toolVersion": "[0-9.]\+"' "$properties_json_file" | grep -o '[0-9.]\+')
if [[ -z "$tool_old_version" ]]; then
echo "Error: Could not find $tool_name version in $properties_json_file"
exit 1
fi
log "Found $tool_name version: $tool_old_version"
tool_new_version=$(bump_version "$tool_old_version" "$tool_version_arg")
if [[ "$tool_new_version" == "$tool_old_version" ]]; then
echo "$tool_name version already at $tool_new_version, skipping update."
return
fi
if [[ "$DRY_RUN" == true ]]; then
echo "Would update $tool_name version from $tool_old_version to $tool_new_version in $properties_json_file"
return
fi
# 1. Update the tool's own version in properties json
echo "Updating $tool_name version from $tool_old_version to $tool_new_version in $properties_json_file"
sed -i "s/\"toolVersion\": \"$tool_old_version\"/\"toolVersion\": \"$tool_new_version\"/g" "$properties_json_file"
# 2. Update the tool's own version in registry json
echo "Updating $tool_name version from $tool_old_version to $tool_new_version in $registry_json_file"
adjusted_tool_name=$tool_name
if [[ "$tool_name" == "classifier" ]]; then
adjusted_tool_name="classify"
fi
# Modify the line after `"functionName": "$adjusted_tool_name"`
sed -i "/\"functionName\": \"$adjusted_tool_name\"/{ n; s/\"toolVersion\": \"$tool_old_version\"/\"toolVersion\": \"$tool_new_version\"/; }" "$registry_json_file"
adjusted_tool_name=${tool_name//_/-} # Replace "_" with "-"
# Modify the line `"image_url": "docker:unstract/tool-$tool_name:$old_version"`
sed -i "s|\"image_url\": \"docker:unstract/tool-$adjusted_tool_name:$tool_old_version\"|\"image_url\": \"docker:unstract/tool-$adjusted_tool_name:$tool_new_version\"|" "$registry_json_file"
# Modify the line after `"image_name": "unstract/tool-$tool_name"`
sed -i "/\"image_name\": \"unstract\\/tool-$adjusted_tool_name\"/{ n; s/\"image_tag\": \"$tool_old_version\"/\"image_tag\": \"$tool_new_version\"/; }" "$registry_json_file"
# 3. Update the unstract-sdk version in the tool's requirements.txt
if [[ -f "$requirements_file" ]]; then
echo "Updating SDK version in $tool_name requirements.txt"
update_sdk_version "$requirements_file" "$sdk_version_arg"
fi
}
# Update SDK version in pyproject.toml or requirements.txt
update_sdk_version() {
local file="$1"
local version_arg="$2"
local dir_name=$(dirname "$file")
local old_version
local new_version
local pattern
local replacement
echo "Processing SDK version for $dir_name..."
# Check if file exists
if [[ ! -f "$file" ]]; then
echo "Warning: $file not found, skipping SDK update"
return
fi
# Check if file has unstract-sdk dependency
if ! grep -q "unstract-sdk" "$file"; then
log "unstract-sdk dependency not found in $file, skipping"
return
fi
log "Checking SDK version in $file"
# Check for different SDK formats based on file type
if grep -q 'unstract-sdk\[gcs, azure, aws\]' "$file"; then
pattern='unstract-sdk\[gcs, azure, aws\]~='
log "Found unstract-sdk[gcs, azure, aws] in $file"
elif grep -q 'unstract-sdk\[azure\]' "$file"; then
pattern='unstract-sdk\[azure\]~='
log "Found unstract-sdk[azure] in $file"
elif grep -q 'unstract-sdk\[aws\]' "$file"; then
pattern='unstract-sdk\[aws\]~='
log "Found unstract-sdk[aws] in $file"
elif grep -q 'unstract-sdk~=' "$file"; then
pattern='unstract-sdk~='
log "Found unstract-sdk in $file"
else
echo "Warning: Could not identify unstract-sdk pattern in $file"
return 1
fi
# Extract current SDK version using the identified pattern
old_version=$(grep -o "${pattern}[0-9.]\+" "$file" | grep -o '[0-9.]\+')
if [[ -z "$old_version" ]]; then
echo "Error: Could not find unstract-sdk version in $file"
return 1
fi
log "Found SDK version: $old_version"
# Handle special version keywords
if [[ "$version_arg" == "patch" || "$version_arg" == "minor" || "$version_arg" == "major" ]]; then
new_version=$(bump_version "$old_version" "$version_arg")
log "Auto-bumped SDK version: $old_version -> $new_version ($version_arg)"
else
new_version="$version_arg"
fi
if [[ "$old_version" == "$new_version" ]]; then
echo "SDK version already at $new_version, skipping update in $file"
return
fi
if [[ "$DRY_RUN" == true ]]; then
echo "Would update SDK version from $old_version to $new_version in $file"
else
echo "Updating SDK version from $old_version to $new_version in $file"
# Use the detected pattern to update the version
sed -i "s/${pattern}${old_version}/${pattern}${new_version}/g" "$file"
# Also update any other references to the SDK version in the file
sed -i "s/unstract-sdk\", specifier = \"~=$old_version/unstract-sdk\", specifier = \"~=$new_version/g" "$file"
fi
}
# Generate or update uv.lock file in a directory
generate_uvlock() {
local dir="$1"
local uvlock_file="$dir/uv.lock"
log "Processing uv lock for $dir..."
# Check if pyproject.toml exists
if [[ ! -f "$dir/pyproject.toml" ]]; then
echo "Warning: pyproject.toml not found in $dir, skipping uv.lock generation"
return 1
fi
if [[ "$DRY_RUN" == true ]]; then
echo "Would generate uv.lock file in $dir"
else
echo "Generating uv.lock file in $dir"
# Use uv pip compile to generate the lock file
(cd "$dir" && uv lock) || {
echo "Error: Failed to generate uv.lock in $dir"
return 1
}
fi
return 0
}
# Reset a single file to its original state
reset_file() {
local file="$1"
local rel_path="${file#$UNSTRACT_ROOT/}"
if [[ "$DRY_RUN" == true ]]; then
echo "[DRY RUN] Would reset $file to its original state"
elif [[ -f "$file" ]]; then
echo "Resetting $file to its original state"
(cd "$UNSTRACT_ROOT" && git checkout -- "$rel_path") || {
echo "Warning: Failed to reset $file"
}
fi
}
# Reset changes in a specific directory
reset_directory_changes() {
# Reset root directory
reset_file "$UNSTRACT_ROOT/pyproject.toml"
reset_file "$UNSTRACT_ROOT/uv.lock"
# Reset service directories
reset_file "$BACKEND_DIR/pyproject.toml"
reset_file "$BACKEND_DIR/uv.lock"
reset_file "$PLATFORM_SERVICE_DIR/pyproject.toml"
reset_file "$PLATFORM_SERVICE_DIR/uv.lock"
# Reset package directories
reset_file "$FILESYSTEM_DIR/pyproject.toml"
reset_file "$FILESYSTEM_DIR/uv.lock"
reset_file "$TOOL_REGISTRY_DIR/pyproject.toml"
reset_file "$TOOL_REGISTRY_DIR/uv.lock"
# Reset custom tool directories
reset_file "$CLASSIFIER_DIR/requirements.txt"
reset_file "$CLASSIFIER_DIR/src/config/properties.json"
reset_file "$TEXT_EXTRACTOR_DIR/requirements.txt"
reset_file "$TEXT_EXTRACTOR_DIR/src/config/properties.json"
# Reset additional files
reset_file "$SAMPLE_ENV_FILE"
reset_file "$TOOL_REGISTRY_JSON_FILE"
}
# Process each directory based on flags
process_directories() {
# If in reset mode, handle resets and exit
if [[ "$RESET_MODE" == true ]]; then
echo "[-] Reset mode: Restoring files to their original state..."
# Prompt user for confirmation
echo "This will reset ALL UNRELATED changes too in each modified file. Do you want to continue?"
read -p "Enter 'yes' to proceed: " confirm
if [[ "$confirm" != "yes" ]]; then
echo "[!] Operation cancelled."
else
reset_directory_changes
echo "[*] All changes have been reset"
fi
return
fi
# If not in bump mode, do nothing and exit
if [[ "$BUMP_MODE" == false ]]; then
return
fi
echo "[-] Processing all directories and files for version bump..."
# 1. Process custom tools
echo "[-] Processing custom tools..."
for dir in "${CUSTOM_TOOL_DIRS[@]}"; do
update_custom_tool_version "$dir" "$TARGET_VERSION" "$TOOL_REGISTRY_JSON_FILE"
done
# 2. Process packages (similar to services)
echo "[-] Processing packages..."
for dir in "${PACKAGE_DIRS[@]}"; do
update_sdk_version "$dir/pyproject.toml" "$TARGET_VERSION"
generate_uvlock "$dir"
done
# 3. Process services
echo "[-] Processing services..."
for dir in "${SERVICE_DIRS[@]}"; do
update_sdk_version "$dir/pyproject.toml" "$TARGET_VERSION"
generate_uvlock "$dir"
done
# 4. Process root directory
echo "[-] Processing root directory..."
update_sdk_version "$ROOT_DIR/pyproject.toml" "$TARGET_VERSION"
generate_uvlock "$ROOT_DIR"
echo "[*] Done."
}
# Main function
main() {
# Parse command-line arguments
parse_args "$@"
# Check if a core command is provided
if [[ "$BUMP_MODE" == false && "$RESET_MODE" == false ]]; then
usage
exit 1
fi
# Check if --target-version is provided with --bump
if [[ "$BUMP_MODE" == true && -z "$TARGET_VERSION" ]]; then
echo "[!] Error: --target-version is required when using --bump."
usage
exit 1
fi
if [[ "$VERBOSE" == true ]]; then
echo "[*] Running with options:"
echo " Reset Mode: $RESET_MODE"
echo " Dry Run: $DRY_RUN"
echo " Verbose: $VERBOSE"
echo ""
fi
# Process directories based on specified options
process_directories
}
# Execute the main function with all arguments
main "$@"