100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Fix manager docker version issue with CI/CD deployment in custom projects (#1667)
Rich Turner1a2e3f035c68
diff --git a/.ci_cd/README.md b/.ci_cd/README.mdindex 049874fc8..f1b18b488 100644--- a/.ci_cd/README.md+++ b/.ci_cd/README.md@@ -20,7 +20,7 @@ docker image tag updates. The file layout is:},// Singleton or array of deployments to execute, a deployment consists of (environment and/or managerTag see variables for explanation)"deploy": {- "managerTag": "develop", // Can use #ref or exclude this to build COMMIT specific manager docker image (i.e. don't use an image from docker hub)+ "managerTag": "develop", // Can use #ref or exclude this to build COMMIT specific manager docker image (i.e. don't use an image from docker hub) for custom projects will always use openremoteVersion"environment": "staging"}},@@ -40,19 +40,9 @@ docker image tag updates. The file layout is:},// Singleton or array of deployments to execute, a deployment consists of (environment and/or managerTag see variables for explanation)"deploy": {- "managerTag": "latest", // Can use #ref or exclude this to build COMMIT specific manager docker image+ "managerTag": "latest", // Can use #ref or exclude this to build COMMIT specific manager docker image (i.e. don't use an image from docker hub) for custom projects will always use openremoteVersion"environment": "production"}- },- // Configure behaviour on manager docker image update (must be triggered by schedule event from a custom project or fork)- "managerDockerPush": {- // Docker tags to monitor can only be used if not the main openremote repo (openremote/openremote)- "develop": {- // Singleton or array of deployments to execute, a deployment consists of (environment and/or managerTag see variables for explanation)- "deploy": {- "environment": "staging"- }- }}}```diff --git a/.ci_cd/env/.env b/.ci_cd/env/.envindex b04ebf9a6..c22e96b28 100644--- a/.ci_cd/env/.env+++ b/.ci_cd/env/.env@@ -5,3 +5,4 @@ OR_SETUP_TYPE=demoOR_EMAIL_TO="[email protected]"OR_SETUP_RUN_ON_RESTART=trueDAILY_RESTART=true+CLEAN_INSTALL=truediff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.ymlindex d69602acf..73db9fa73 100644--- a/.github/workflows/ci_cd.yml+++ b/.github/workflows/ci_cd.yml@@ -1,6 +1,6 @@# ---------------------------------------------------------------------------------------------------------------------# This workflow handles all CI/CD related tasks and can be re-used in custom projects-# (https://github.com/openremote/custom-project i.e. has openremote submodule) or forks of this repo, it handles:+# (https://github.com/openremote/custom-project) or forks of this repo, it handles:## - Running tests on push/release# - Distributing openremote/manager images to docker hub (DOCKERHUB_USER and DOCKERHUB_PASSWORD must be set) on push/release@@ -182,13 +182,11 @@ jobs:with:files_ignore: |.ci_cd/**- console/**deployment/**- setup/**ui/****/*.md**/*.yml- manager/src/web/**+ yarn.lock# Check which files have changed to only run appropriate tests and checks- name: UI files changed@@ -197,8 +195,7 @@ jobs:uses: tj-actions/changed-files@v45with:files: |- ui/app/manager/**- ui/component/**+ ui/**files_ignore: |**/*.md**/*.yml@@ -213,20 +210,14 @@ jobs:if: steps.ui-files-changed.outputs.any_modified != 'true'run: echo "value=true" >> $GITHUB_OUTPUT- - name: Make cache dirs and set file permissions- run: |- chmod +x gradlew- mkdir -p ~/manager-tags-old- mkdir -p ~/manager-tags-new-- name: Check if main repoid: is_main_reporun: |- if [ $(grep -E '.+/openremote$' <<< "$GITHUB_REPOSITORY") ]; then+ if [ -f manager/src/main/java/org/openremote/manager/Main.java ]; thenecho "value=true" >> $GITHUB_OUTPUTelseif [ -e 'openremote' ]; then- echo "::error::Custom project should not have an 'openremote' file or directory!"+ echo "::error::openremote submodule no longer supported, see custom project template!"exit 1fi@@ -234,11 +225,13 @@ jobs:echo "repository=$(sed -E 's#(.+)/.+#\1/openremote#' <<< $GITHUB_REPOSITORY)" >> $GITHUB_OUTPUTversion=$(cat gradle.properties | grep openremoteVersion | sed -E 's#.+=(.+)#\1#' | xargs)- if [ $(grep -E '.+SNAPSHOT$' <<< "$version") ]; then- echo "ref=master" >> $GITHUB_OUTPUT- else- echo "ref=$version" >> $GITHUB_OUTPUT++ if [ -z "$version" ]; then+ echo "openremoteVersion must be set in gradle.properties"+ exit 1fi++ echo "ref=$version" >> $GITHUB_OUTPUTfi- name: Checkout main repo@@ -271,16 +264,6 @@ jobs:with:files: ".ci_cd/ci_cd.json"- - name: Check is custom project when scheduled trigger- if: ${{ github.event_name == 'schedule' }}- run: |- if [ "$IS_MAIN_REPO" == 'true' ]; then- echo "Schedule can only be used on a custom project repo"- exit 1- fi- env:- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}-- name: Get version detailsid: get-version-detailsrun: |@@ -291,54 +274,10 @@ jobs:if [ "$version" == "$versionWithQualifier" ]; thenisRelease=truefi-echo "version=$version" >> $GITHUB_OUTPUTecho "version_with_qualifier=$versionWithQualifier" >> $GITHUB_OUTPUTecho "is_release=$isRelease" >> $GITHUB_OUTPUT- # For scheduled execution from custom projects we need to be able to determine if the manager docker image has- # changed since the last execution, we do this using a cache- - name: Get docker tags to monitor- if: ${{ steps.check_cicd_json.outputs.files_exists == 'true' && github.event_name == 'schedule' }}- id: monitor-tags- shell: python- run: |- import json- import os-- eventName = os.getenv('GITHUB_EVENT_NAME')- refName = os.getenv('GITHUB_REF_NAME')- pushTag = None- deployTag = None- deployEnvironment = None-- f = open(".ci_cd/ci_cd.json")- data = json.load(f)- f.close()-- if "managerDockerPush" in data:- managerPushConfig = data["managerDockerPush"]-- if managerPushConfig is not None:- if len(managerPushConfig.keys()) > 0:- for key in managerPushConfig.keys():- # Output new tag cache files (for generating hash)- os.system(f"echo 'Outputting manifest info for tag: {key}'")- os.system(f"docker buildx imagetools inspect --raw openremote/manager:{key} > ~/manager-tags-new/{key}")- os.system("echo 'Tag manifests generated:'")- os.system("find ~/manager-tags-new")-- - name: Load manager tag cache- if: ${{ steps.check_cicd_json.outputs.files_exists == 'true' && github.event_name == 'schedule' }}- uses: actions/cache@v4- id: manager-tag-cache- with:- path: |- ~/manager-tags-old- key: ${{ runner.os }}-manager-tags---${{ hashFiles('~/manager-tags-new/**') }}- restore-keys: |- ${{ runner.os }}-manager-tags----- name: Process ci_cd.json fileif: ${{ steps.check_cicd_json.outputs.files_exists == 'true' && (github.ref_type == 'tag' || (github.event_name != 'workflow_dispatch' && github.event_name != 'pull_request')) }}id: ci-cd-output@@ -363,47 +302,14 @@ jobs:data = json.load(f)f.close()- if eventName == "schedule":- eventName = "managerDockerPush"- elif refType == "tag":+ if isRelease:eventName = "release"if data is not None and eventName in data:eventConfig = data[eventName]- if eventName == "managerDockerPush":-- # Iterate through keys and check if that manager docker image has changed since last execution- deploys=[]- for key in eventConfig.keys():- os.system(f"echo 'Checking manager docker tag: {key}'")- newManifestStr = None- oldManifestStr = None-- if os.path.exists(os.path.expanduser(f"~/manager-tags-new/{key}")):- f = open(os.path.expanduser(f"~/manager-tags-new/{key}"))- newManifestStr = f.read()- f.close()- if os.path.exists(os.path.expanduser(f"~/manager-tags-old/{key}")):- f = open(os.path.expanduser(f"~/manager-tags-old/{key}"))- oldManifestStr = f.read()- f.close()-- if newManifestStr != oldManifestStr: # Not a great way to compare but we'll assume manifest output is consistent- os.system(f"echo 'Manager docker tag has been updated: {key}'")- if 'deploy' in eventConfig[key]:- deployConfig = eventConfig[key]['deploy']- if deployConfig is not None:- # Inject manager tag into deploy config (no point reacting to the manager image updating otherwise)- deployConfig['managerTag'] = key- deploys.append(deployConfig)- else:- os.system(f"echo 'No deploy config in ci_cd.json for manager tag {key}'")- else:- os.system(f"echo 'Manager docker tag has not changed: {key}'")-- elif eventName == "push" and refType == "branch" and refName in eventConfig:- eventConfig = eventConfig[refName]+ if eventName == "push" and refType == "branch" and refName in eventConfig:+ eventConfig = eventConfig[refName]if eventConfig is not None:deploys = eventConfig['deploy'] if 'deploy' in eventConfig else None@@ -418,10 +324,15 @@ jobs:if isMainRepo and ((eventName == "release" and isRelease) or (eventName != "release" and not isRelease)):if dockerPublishTags is not None:- dockerPublishTags = dockerPublishTags.replace("$version", version)- firstDockerTag = dockerPublishTags.split(",")[0]+ if "$version" in dockerPublishTags:+ firstDockerTag = version+ dockerPublishTags = dockerPublishTags.replace("$version", version)+ else:+ dockerPublishTags = dockerPublishTags.replace("$version", version)+ firstDockerTag = dockerPublishTags.split(",")[0]+os.system(f"echo 'firstDockerTag={firstDockerTag}' >> $GITHUB_OUTPUT")- os.system(f" echo 'Manager tags to push to docker: {dockerPublishTags}'")+ os.system(f" echo 'Manager tags to push to docker hub: {dockerPublishTags}'")dockerPublishTags = " ".join(map(lambda t: f"-t openremote/manager:{t.strip()}", dockerPublishTags.split(",")))os.system(f"echo 'dockerTags={dockerPublishTags}' >> $GITHUB_OUTPUT")@@ -441,16 +352,25 @@ jobs:deploys = [deploys]deployStr = ""+ managerTagDefault = '#ref'for deploy in deploys:if 'environment' in deploy:deployStr += deploy['environment']deployStr += ":"if 'managerTag' in deploy:- deployStr += deploy['managerTag']+ if isMainRepo:+ deployStr += deploy['managerTag']+ else:+ # Custom projects manager docker image must match the openremoteVersion from gradle.properties+ deployStr += "$MANAGER_VERSION"else:- os.system("echo 'Manager tag not specified so using commit SHA'")- deployStr += '#ref'+ if isMainRepo:+ os.system("echo 'Manager tag not specified so using commit SHA'")+ deployStr += managerTagDefault+ else:+ # Custom projects manager docker image must match the openremoteVersion from gradle.properties+ deployStr += "$MANAGER_VERSION"deployStr += ";"@@ -461,29 +381,42 @@ jobs:os.system(f"echo 'deploys={deployStr}' >> $GITHUB_OUTPUT")env:IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}+ MANAGER_VERSION: ${{ steps.is_main_repo.outputs.ref }}IS_RELEASE: ${{ steps.get-version-details.outputs.is_release }}VERSION: ${{ steps.get-version-details.outputs.version }}- - name: Copy new tag cache to old- if: ${{ steps.check_cicd_json.outputs.files_exists == 'true' }}- run: |- cd ~/manager-tags-new- cp -r . ~/manager-tags-old/-- name: Sanitize deployments valueid: deployments+ shell: pythonrun: |- deployments=$DEPLOYMENTS+ import os- if [ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ] && [ "$GITHUB_REF_TYPE" != 'tag' ]; then- tag="$INPUT_MANAGER_TAG"- if [ -z "$INPUT_MANAGER_TAG" ]; then- tag='#ref'- fi- deployments="$INPUT_ENVIRONMENT:$tag"- fi- echo "value=$deployments" >> $GITHUB_OUTPUT+ isMainRepo = os.getenv('IS_MAIN_REPO') == 'true'+ deployments = os.getenv('DEPLOYMENTS')+ eventName = os.getenv('GITHUB_EVENT_NAME')+ refType = os.getenv('GITHUB_REF_TYPE')+ inputTag = os.getenv('INPUT_MANAGER_TAG')+ inputEnv = os.getenv('INPUT_ENVIRONMENT')+ managerVersion = os.getenv('MANAGER_VERSION')++ if eventName == 'workflow_dispatch' and refType != 'tag':+ if isMainRepo:+ tag=inputTag+ if inputTag is None:+ tag='#ref'+ else:+ tag=managerVersion+ deployments= f'{inputEnv}:{tag}'++ if isMainRepo:+ if deployments and deployments.contains('-SNAPSHOT'):+ os.system("SNAPSHOT manager docker images not currently supported for custom project deployment")+ sys.exit(1)++ os.system(f"echo 'value={deployments}' >> $GITHUB_OUTPUT")env:+ IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}+ MANAGER_VERSION: ${{ steps.is_main_repo.outputs.ref }}DEPLOYMENTS: ${{ steps.ci-cd-output.outputs.deploys }}INPUT_ENVIRONMENT: ${{ steps.inputs-and-secrets.outputs.ENVIRONMENT }}INPUT_MANAGER_TAG: ${{steps.inputs-and-secrets.outputs.MANAGER_TAG }}@@ -500,32 +433,30 @@ jobs:- name: Define manager docker build commandid: manager-docker-command+ if: ${{ steps.is_main_repo.outputs.value == 'true' }}shell: bashrun: |- if [ $IS_MAIN_REPO == 'true' ]; then- buildPath="manager/build/install/manager"- commitSha=$(git rev-parse HEAD)- commitShaShort=$(git rev-parse --short HEAD)-- if [ -n "$MANAGER_TAGS" ] || [[ "$DEPLOYMENTS" == *"#ref"* ]] || [ -n "$TEST_UI_CMD" ]; then- if [ -n "$MANAGER_TAGS" ]; then- command="docker build --push --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 $MANAGER_TAGS $buildPath"- echo "pushRequired=true" >> $GITHUB_OUTPUT- echo "managerDockerImage=openremote/manager:$FIRST_MANAGER_TAG" >> $GITHUB_OUTPUT- else- command="docker build --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 -t openremote/manager:$commitShaShort $buildPath"- echo "managerDockerImage=openremote/manager:$commitShaShort" >> $GITHUB_OUTPUT- fi- echo "value=$command" >> $GITHUB_OUTPUT+ buildPath="manager/build/install/manager"+ commitSha=$(git rev-parse HEAD)+ commitShaShort=$(git rev-parse --short HEAD)++ if [ -n "$MANAGER_TAGS" ] || [[ "$DEPLOYMENTS" == *"#ref"* ]] || [ -n "$TEST_UI_CMD" ]; then+ if [ -n "$MANAGER_TAGS" ]; then+ command="docker build --push --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 $MANAGER_TAGS $buildPath"+ echo "pushRequired=true" >> $GITHUB_OUTPUT+ echo "managerDockerImage=openremote/manager:$FIRST_MANAGER_TAG" >> $GITHUB_OUTPUT+ else+ command="docker build --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 -t openremote/manager:$commitShaShort $buildPath"+ echo "managerDockerImage=openremote/manager:$commitShaShort" >> $GITHUB_OUTPUTfi- echo "buildPath=$buildPath" >> $GITHUB_OUTPUT- echo "refTag=$commitShaShort" >> $GITHUB_OUTPUT+ echo "value=$command" >> $GITHUB_OUTPUTfi+ echo "buildPath=$buildPath" >> $GITHUB_OUTPUT+ echo "refTag=$commitShaShort" >> $GITHUB_OUTPUTenv:FIRST_MANAGER_TAG: ${{ steps.ci-cd-output.outputs.firstDockerTag }}MANAGER_TAGS: ${{ steps.ci-cd-output.outputs.dockerTags }}DEPLOYMENTS: ${{ steps.deployments.outputs.value }}- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}TEST_UI_CMD: ${{ steps.test-ui-command.outputs.value }}- name: Define maven publish command@@ -634,6 +565,7 @@ jobs:echo 'Trigger event: ${{ github.event_name }}'echo 'Is main repo: ${{ steps.is_main_repo.outputs.value == 'true' }}'echo 'Has deployment dockerfile: ${{ steps.check_deployment_dockerfile.outputs.files_exists == 'true' }}'+ echo 'OpenRemote version: ${{ steps.is_main_repo.outputs.ref }}'echo 'Manager commit SHA: ${{ steps.manager-docker-command.outputs.refTag }}'echo 'Deployment commit SHA: ${{ steps.deployment-docker-command.outputs.refTag }}'echo 'Deployments: ${{ steps.deployments.outputs.value }}'@@ -647,9 +579,6 @@ jobs:echo "Java version: $(java --version)"echo "Yarn version: $(yarn -v)"echo "Node version: $(node -v)"- echo 'Gradle cache: ${{ steps.java.outputs.cache-hit == 'true' }}'- echo 'Yarn cache: ${{ steps.yarn-cache.outputs.cache-hit == 'true' }}'- echo 'Manager tag cache: ${{ steps.manager-tag-cache.outputs.cache-hit == 'true' }}'echo "************************************************************"echo "************************************************************"@@ -663,14 +592,9 @@ jobs:composeProfile='profile/dev-ui.yml'fi- if [ $IS_MAIN_REPO == 'false' ]; then- composeProfile="openremote/$composeProfile"- fi-# Pull the imagesdocker compose -f $composeProfile pullenv:- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}TEST_UI_COMMAND: ${{ steps.test-ui-command.outputs.value }}# Use docker layer caching to speed up image building@@ -685,10 +609,6 @@ jobs:run: |composeProfile='profile/dev-testing.yml'- if [ $IS_MAIN_REPO == 'false' ]; then- composeProfile="openremote/$composeProfile"- fi-# Make temp dir and set mask to 777 as docker seems to run as rootmkdir -p tmpchmod 777 tmp@@ -702,8 +622,6 @@ jobs:# Run the tests${{ steps.test-backend-command.outputs.value }}- env:- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}timeout-minutes: 20#continue-on-error: true@@ -718,6 +636,26 @@ jobs:if: ${{ steps.test-backend-command.outputs.value != '' && steps.test-ui-command.outputs.value != '' }}run: ${{ steps.run-backend-tests.outputs.cleanup }}+ - name: Run frontend tests+ if: steps.test-ui-command.outputs.value != ''+ run: |+ composeProfile='profile/dev-ui.yml'++ if [ $IS_MAIN_REPO == 'false' ]; then+ composeProfile="openremote/$composeProfile"+ fi++ # Start the stack+ MANAGER_VERSION=$MANAGER_TAG docker compose -f $composeProfile up -d --no-build++ # Run the tests+ ${{ steps.test-ui-command.outputs.value }}+ env:+ IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}+ MANAGER_TAG: ${{ steps.manager-docker-command.outputs.refTag }}+ timeout-minutes: 20+ continue-on-error: true+- name: Run install distif: steps.install-command.outputs.value != ''shell: python@@ -789,26 +727,6 @@ jobs:run: |${{ steps.maven-publish-command.outputs.value }}- - name: Run frontend tests- if: steps.test-ui-command.outputs.value != ''- run: |- composeProfile='profile/dev-ui.yml'-- if [ $IS_MAIN_REPO == 'false' ]; then- composeProfile="openremote/$composeProfile"- fi-- # Start the stack- MANAGER_VERSION=$MANAGER_TAG docker compose -f $composeProfile up -d --no-build-- # Run the tests- ${{ steps.test-ui-command.outputs.value }}- env:- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}- MANAGER_TAG: ${{ steps.manager-docker-command.outputs.refTag }}- timeout-minutes: 20- continue-on-error: true-- name: Run npm publish commandif: ${{ steps.ci-cd-output.outputs.npmTag != '' }}run: |diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex fc4ba75c6..a6c9289fd 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -34,11 +34,11 @@ jobs:- name: Checkoutuses: actions/checkout@v4- - name: Set up JDK 17 and gradle cache+ - name: Set up JDK 21 and gradle cacheuses: actions/setup-java@v4with:distribution: 'temurin'- java-version: '17'+ java-version: '21'cache: 'gradle'- name: Run release