Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Fix manager docker version issue with CI/CD deployment in custom projects (#1667)

Rich Turner1a2e3f035c68
diff --git a/.ci_cd/README.md b/.ci_cd/README.md
index 049874fc8..f1b18b488 100644
--- a/.ci_cd/README.md
+++ b/.ci_cd/README.md
@@ -20,7 +20,7 @@ docker image tag updates. The file layout is:
},
// Singleton or array of deployments to execute, a deployment consists of (environment and/or managerTag see variables for explanation)
"deploy": {
- "managerTag": "develop", // Can use #ref or exclude this to build COMMIT specific manager docker image (i.e. don't use an image from docker hub)
+ "managerTag": "develop", // Can use #ref or exclude this to build COMMIT specific manager docker image (i.e. don't use an image from docker hub) for custom projects will always use openremoteVersion
"environment": "staging"
}
},
@@ -40,19 +40,9 @@ docker image tag updates. The file layout is:
},
// Singleton or array of deployments to execute, a deployment consists of (environment and/or managerTag see variables for explanation)
"deploy": {
- "managerTag": "latest", // Can use #ref or exclude this to build COMMIT specific manager docker image
+ "managerTag": "latest", // Can use #ref or exclude this to build COMMIT specific manager docker image (i.e. don't use an image from docker hub) for custom projects will always use openremoteVersion
"environment": "production"
}
- },
- // Configure behaviour on manager docker image update (must be triggered by schedule event from a custom project or fork)
- "managerDockerPush": {
- // Docker tags to monitor can only be used if not the main openremote repo (openremote/openremote)
- "develop": {
- // Singleton or array of deployments to execute, a deployment consists of (environment and/or managerTag see variables for explanation)
- "deploy": {
- "environment": "staging"
- }
- }
}
}
```
diff --git a/.ci_cd/env/.env b/.ci_cd/env/.env
index b04ebf9a6..c22e96b28 100644
--- a/.ci_cd/env/.env
+++ b/.ci_cd/env/.env
@@ -5,3 +5,4 @@ OR_SETUP_TYPE=demo
OR_EMAIL_TO="[email protected]"
OR_SETUP_RUN_ON_RESTART=true
DAILY_RESTART=true
+CLEAN_INSTALL=true
diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml
index d69602acf..73db9fa73 100644
--- a/.github/workflows/ci_cd.yml
+++ b/.github/workflows/ci_cd.yml
@@ -1,6 +1,6 @@
# ---------------------------------------------------------------------------------------------------------------------
# This workflow handles all CI/CD related tasks and can be re-used in custom projects
-# (https://github.com/openremote/custom-project i.e. has openremote submodule) or forks of this repo, it handles:
+# (https://github.com/openremote/custom-project) or forks of this repo, it handles:
#
# - Running tests on push/release
# - Distributing openremote/manager images to docker hub (DOCKERHUB_USER and DOCKERHUB_PASSWORD must be set) on push/release
@@ -182,13 +182,11 @@ jobs:
with:
files_ignore: |
.ci_cd/**
- console/**
deployment/**
- setup/**
ui/**
**/*.md
**/*.yml
- manager/src/web/**
+ yarn.lock
# Check which files have changed to only run appropriate tests and checks
- name: UI files changed
@@ -197,8 +195,7 @@ jobs:
uses: tj-actions/changed-files@v45
with:
files: |
- ui/app/manager/**
- ui/component/**
+ ui/**
files_ignore: |
**/*.md
**/*.yml
@@ -213,20 +210,14 @@ jobs:
if: steps.ui-files-changed.outputs.any_modified != 'true'
run: echo "value=true" >> $GITHUB_OUTPUT
- - name: Make cache dirs and set file permissions
- run: |
- chmod +x gradlew
- mkdir -p ~/manager-tags-old
- mkdir -p ~/manager-tags-new
-
- name: Check if main repo
id: is_main_repo
run: |
- if [ $(grep -E '.+/openremote$' <<< "$GITHUB_REPOSITORY") ]; then
+ if [ -f manager/src/main/java/org/openremote/manager/Main.java ]; then
echo "value=true" >> $GITHUB_OUTPUT
else
if [ -e 'openremote' ]; then
- echo "::error::Custom project should not have an 'openremote' file or directory!"
+ echo "::error::openremote submodule no longer supported, see custom project template!"
exit 1
fi
@@ -234,11 +225,13 @@ jobs:
echo "repository=$(sed -E 's#(.+)/.+#\1/openremote#' <<< $GITHUB_REPOSITORY)" >> $GITHUB_OUTPUT
version=$(cat gradle.properties | grep openremoteVersion | sed -E 's#.+=(.+)#\1#' | xargs)
- if [ $(grep -E '.+SNAPSHOT$' <<< "$version") ]; then
- echo "ref=master" >> $GITHUB_OUTPUT
- else
- echo "ref=$version" >> $GITHUB_OUTPUT
+
+ if [ -z "$version" ]; then
+ echo "openremoteVersion must be set in gradle.properties"
+ exit 1
fi
+
+ echo "ref=$version" >> $GITHUB_OUTPUT
fi
- name: Checkout main repo
@@ -271,16 +264,6 @@ jobs:
with:
files: ".ci_cd/ci_cd.json"
- - name: Check is custom project when scheduled trigger
- if: ${{ github.event_name == 'schedule' }}
- run: |
- if [ "$IS_MAIN_REPO" == 'true' ]; then
- echo "Schedule can only be used on a custom project repo"
- exit 1
- fi
- env:
- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}
-
- name: Get version details
id: get-version-details
run: |
@@ -291,54 +274,10 @@ jobs:
if [ "$version" == "$versionWithQualifier" ]; then
isRelease=true
fi
-
echo "version=$version" >> $GITHUB_OUTPUT
echo "version_with_qualifier=$versionWithQualifier" >> $GITHUB_OUTPUT
echo "is_release=$isRelease" >> $GITHUB_OUTPUT
- # For scheduled execution from custom projects we need to be able to determine if the manager docker image has
- # changed since the last execution, we do this using a cache
- - name: Get docker tags to monitor
- if: ${{ steps.check_cicd_json.outputs.files_exists == 'true' && github.event_name == 'schedule' }}
- id: monitor-tags
- shell: python
- run: |
- import json
- import os
-
- eventName = os.getenv('GITHUB_EVENT_NAME')
- refName = os.getenv('GITHUB_REF_NAME')
- pushTag = None
- deployTag = None
- deployEnvironment = None
-
- f = open(".ci_cd/ci_cd.json")
- data = json.load(f)
- f.close()
-
- if "managerDockerPush" in data:
- managerPushConfig = data["managerDockerPush"]
-
- if managerPushConfig is not None:
- if len(managerPushConfig.keys()) > 0:
- for key in managerPushConfig.keys():
- # Output new tag cache files (for generating hash)
- os.system(f"echo 'Outputting manifest info for tag: {key}'")
- os.system(f"docker buildx imagetools inspect --raw openremote/manager:{key} > ~/manager-tags-new/{key}")
- os.system("echo 'Tag manifests generated:'")
- os.system("find ~/manager-tags-new")
-
- - name: Load manager tag cache
- if: ${{ steps.check_cicd_json.outputs.files_exists == 'true' && github.event_name == 'schedule' }}
- uses: actions/cache@v4
- id: manager-tag-cache
- with:
- path: |
- ~/manager-tags-old
- key: ${{ runner.os }}-manager-tags---${{ hashFiles('~/manager-tags-new/**') }}
- restore-keys: |
- ${{ runner.os }}-manager-tags---
-
- name: Process ci_cd.json file
if: ${{ steps.check_cicd_json.outputs.files_exists == 'true' && (github.ref_type == 'tag' || (github.event_name != 'workflow_dispatch' && github.event_name != 'pull_request')) }}
id: ci-cd-output
@@ -363,47 +302,14 @@ jobs:
data = json.load(f)
f.close()
- if eventName == "schedule":
- eventName = "managerDockerPush"
- elif refType == "tag":
+ if isRelease:
eventName = "release"
if data is not None and eventName in data:
eventConfig = data[eventName]
- if eventName == "managerDockerPush":
-
- # Iterate through keys and check if that manager docker image has changed since last execution
- deploys=[]
- for key in eventConfig.keys():
- os.system(f"echo 'Checking manager docker tag: {key}'")
- newManifestStr = None
- oldManifestStr = None
-
- if os.path.exists(os.path.expanduser(f"~/manager-tags-new/{key}")):
- f = open(os.path.expanduser(f"~/manager-tags-new/{key}"))
- newManifestStr = f.read()
- f.close()
- if os.path.exists(os.path.expanduser(f"~/manager-tags-old/{key}")):
- f = open(os.path.expanduser(f"~/manager-tags-old/{key}"))
- oldManifestStr = f.read()
- f.close()
-
- if newManifestStr != oldManifestStr: # Not a great way to compare but we'll assume manifest output is consistent
- os.system(f"echo 'Manager docker tag has been updated: {key}'")
- if 'deploy' in eventConfig[key]:
- deployConfig = eventConfig[key]['deploy']
- if deployConfig is not None:
- # Inject manager tag into deploy config (no point reacting to the manager image updating otherwise)
- deployConfig['managerTag'] = key
- deploys.append(deployConfig)
- else:
- os.system(f"echo 'No deploy config in ci_cd.json for manager tag {key}'")
- else:
- os.system(f"echo 'Manager docker tag has not changed: {key}'")
-
- elif eventName == "push" and refType == "branch" and refName in eventConfig:
- eventConfig = eventConfig[refName]
+ if eventName == "push" and refType == "branch" and refName in eventConfig:
+ eventConfig = eventConfig[refName]
if eventConfig is not None:
deploys = eventConfig['deploy'] if 'deploy' in eventConfig else None
@@ -418,10 +324,15 @@ jobs:
if isMainRepo and ((eventName == "release" and isRelease) or (eventName != "release" and not isRelease)):
if dockerPublishTags is not None:
- dockerPublishTags = dockerPublishTags.replace("$version", version)
- firstDockerTag = dockerPublishTags.split(",")[0]
+ if "$version" in dockerPublishTags:
+ firstDockerTag = version
+ dockerPublishTags = dockerPublishTags.replace("$version", version)
+ else:
+ dockerPublishTags = dockerPublishTags.replace("$version", version)
+ firstDockerTag = dockerPublishTags.split(",")[0]
+
os.system(f"echo 'firstDockerTag={firstDockerTag}' >> $GITHUB_OUTPUT")
- os.system(f" echo 'Manager tags to push to docker: {dockerPublishTags}'")
+ os.system(f" echo 'Manager tags to push to docker hub: {dockerPublishTags}'")
dockerPublishTags = " ".join(map(lambda t: f"-t openremote/manager:{t.strip()}", dockerPublishTags.split(",")))
os.system(f"echo 'dockerTags={dockerPublishTags}' >> $GITHUB_OUTPUT")
@@ -441,16 +352,25 @@ jobs:
deploys = [deploys]
deployStr = ""
+ managerTagDefault = '#ref'
for deploy in deploys:
if 'environment' in deploy:
deployStr += deploy['environment']
deployStr += ":"
if 'managerTag' in deploy:
- deployStr += deploy['managerTag']
+ if isMainRepo:
+ deployStr += deploy['managerTag']
+ else:
+ # Custom projects manager docker image must match the openremoteVersion from gradle.properties
+ deployStr += "$MANAGER_VERSION"
else:
- os.system("echo 'Manager tag not specified so using commit SHA'")
- deployStr += '#ref'
+ if isMainRepo:
+ os.system("echo 'Manager tag not specified so using commit SHA'")
+ deployStr += managerTagDefault
+ else:
+ # Custom projects manager docker image must match the openremoteVersion from gradle.properties
+ deployStr += "$MANAGER_VERSION"
deployStr += ";"
@@ -461,29 +381,42 @@ jobs:
os.system(f"echo 'deploys={deployStr}' >> $GITHUB_OUTPUT")
env:
IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}
+ MANAGER_VERSION: ${{ steps.is_main_repo.outputs.ref }}
IS_RELEASE: ${{ steps.get-version-details.outputs.is_release }}
VERSION: ${{ steps.get-version-details.outputs.version }}
- - name: Copy new tag cache to old
- if: ${{ steps.check_cicd_json.outputs.files_exists == 'true' }}
- run: |
- cd ~/manager-tags-new
- cp -r . ~/manager-tags-old/
-
- name: Sanitize deployments value
id: deployments
+ shell: python
run: |
- deployments=$DEPLOYMENTS
+ import os
- if [ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ] && [ "$GITHUB_REF_TYPE" != 'tag' ]; then
- tag="$INPUT_MANAGER_TAG"
- if [ -z "$INPUT_MANAGER_TAG" ]; then
- tag='#ref'
- fi
- deployments="$INPUT_ENVIRONMENT:$tag"
- fi
- echo "value=$deployments" >> $GITHUB_OUTPUT
+ isMainRepo = os.getenv('IS_MAIN_REPO') == 'true'
+ deployments = os.getenv('DEPLOYMENTS')
+ eventName = os.getenv('GITHUB_EVENT_NAME')
+ refType = os.getenv('GITHUB_REF_TYPE')
+ inputTag = os.getenv('INPUT_MANAGER_TAG')
+ inputEnv = os.getenv('INPUT_ENVIRONMENT')
+ managerVersion = os.getenv('MANAGER_VERSION')
+
+ if eventName == 'workflow_dispatch' and refType != 'tag':
+ if isMainRepo:
+ tag=inputTag
+ if inputTag is None:
+ tag='#ref'
+ else:
+ tag=managerVersion
+ deployments= f'{inputEnv}:{tag}'
+
+ if isMainRepo:
+ if deployments and deployments.contains('-SNAPSHOT'):
+ os.system("SNAPSHOT manager docker images not currently supported for custom project deployment")
+ sys.exit(1)
+
+ os.system(f"echo 'value={deployments}' >> $GITHUB_OUTPUT")
env:
+ IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}
+ MANAGER_VERSION: ${{ steps.is_main_repo.outputs.ref }}
DEPLOYMENTS: ${{ steps.ci-cd-output.outputs.deploys }}
INPUT_ENVIRONMENT: ${{ steps.inputs-and-secrets.outputs.ENVIRONMENT }}
INPUT_MANAGER_TAG: ${{steps.inputs-and-secrets.outputs.MANAGER_TAG }}
@@ -500,32 +433,30 @@ jobs:
- name: Define manager docker build command
id: manager-docker-command
+ if: ${{ steps.is_main_repo.outputs.value == 'true' }}
shell: bash
run: |
- if [ $IS_MAIN_REPO == 'true' ]; then
- buildPath="manager/build/install/manager"
- commitSha=$(git rev-parse HEAD)
- commitShaShort=$(git rev-parse --short HEAD)
-
- if [ -n "$MANAGER_TAGS" ] || [[ "$DEPLOYMENTS" == *"#ref"* ]] || [ -n "$TEST_UI_CMD" ]; then
- if [ -n "$MANAGER_TAGS" ]; then
- command="docker build --push --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 $MANAGER_TAGS $buildPath"
- echo "pushRequired=true" >> $GITHUB_OUTPUT
- echo "managerDockerImage=openremote/manager:$FIRST_MANAGER_TAG" >> $GITHUB_OUTPUT
- else
- command="docker build --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 -t openremote/manager:$commitShaShort $buildPath"
- echo "managerDockerImage=openremote/manager:$commitShaShort" >> $GITHUB_OUTPUT
- fi
- echo "value=$command" >> $GITHUB_OUTPUT
+ buildPath="manager/build/install/manager"
+ commitSha=$(git rev-parse HEAD)
+ commitShaShort=$(git rev-parse --short HEAD)
+
+ if [ -n "$MANAGER_TAGS" ] || [[ "$DEPLOYMENTS" == *"#ref"* ]] || [ -n "$TEST_UI_CMD" ]; then
+ if [ -n "$MANAGER_TAGS" ]; then
+ command="docker build --push --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 $MANAGER_TAGS $buildPath"
+ echo "pushRequired=true" >> $GITHUB_OUTPUT
+ echo "managerDockerImage=openremote/manager:$FIRST_MANAGER_TAG" >> $GITHUB_OUTPUT
+ else
+ command="docker build --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 -t openremote/manager:$commitShaShort $buildPath"
+ echo "managerDockerImage=openremote/manager:$commitShaShort" >> $GITHUB_OUTPUT
fi
- echo "buildPath=$buildPath" >> $GITHUB_OUTPUT
- echo "refTag=$commitShaShort" >> $GITHUB_OUTPUT
+ echo "value=$command" >> $GITHUB_OUTPUT
fi
+ echo "buildPath=$buildPath" >> $GITHUB_OUTPUT
+ echo "refTag=$commitShaShort" >> $GITHUB_OUTPUT
env:
FIRST_MANAGER_TAG: ${{ steps.ci-cd-output.outputs.firstDockerTag }}
MANAGER_TAGS: ${{ steps.ci-cd-output.outputs.dockerTags }}
DEPLOYMENTS: ${{ steps.deployments.outputs.value }}
- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}
TEST_UI_CMD: ${{ steps.test-ui-command.outputs.value }}
- name: Define maven publish command
@@ -634,6 +565,7 @@ jobs:
echo 'Trigger event: ${{ github.event_name }}'
echo 'Is main repo: ${{ steps.is_main_repo.outputs.value == 'true' }}'
echo 'Has deployment dockerfile: ${{ steps.check_deployment_dockerfile.outputs.files_exists == 'true' }}'
+ echo 'OpenRemote version: ${{ steps.is_main_repo.outputs.ref }}'
echo 'Manager commit SHA: ${{ steps.manager-docker-command.outputs.refTag }}'
echo 'Deployment commit SHA: ${{ steps.deployment-docker-command.outputs.refTag }}'
echo 'Deployments: ${{ steps.deployments.outputs.value }}'
@@ -647,9 +579,6 @@ jobs:
echo "Java version: $(java --version)"
echo "Yarn version: $(yarn -v)"
echo "Node version: $(node -v)"
- echo 'Gradle cache: ${{ steps.java.outputs.cache-hit == 'true' }}'
- echo 'Yarn cache: ${{ steps.yarn-cache.outputs.cache-hit == 'true' }}'
- echo 'Manager tag cache: ${{ steps.manager-tag-cache.outputs.cache-hit == 'true' }}'
echo "************************************************************"
echo "************************************************************"
@@ -663,14 +592,9 @@ jobs:
composeProfile='profile/dev-ui.yml'
fi
- if [ $IS_MAIN_REPO == 'false' ]; then
- composeProfile="openremote/$composeProfile"
- fi
-
# Pull the images
docker compose -f $composeProfile pull
env:
- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}
TEST_UI_COMMAND: ${{ steps.test-ui-command.outputs.value }}
# Use docker layer caching to speed up image building
@@ -685,10 +609,6 @@ jobs:
run: |
composeProfile='profile/dev-testing.yml'
- if [ $IS_MAIN_REPO == 'false' ]; then
- composeProfile="openremote/$composeProfile"
- fi
-
# Make temp dir and set mask to 777 as docker seems to run as root
mkdir -p tmp
chmod 777 tmp
@@ -702,8 +622,6 @@ jobs:
# Run the tests
${{ steps.test-backend-command.outputs.value }}
- env:
- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}
timeout-minutes: 20
#continue-on-error: true
@@ -718,6 +636,26 @@ jobs:
if: ${{ steps.test-backend-command.outputs.value != '' && steps.test-ui-command.outputs.value != '' }}
run: ${{ steps.run-backend-tests.outputs.cleanup }}
+ - name: Run frontend tests
+ if: steps.test-ui-command.outputs.value != ''
+ run: |
+ composeProfile='profile/dev-ui.yml'
+
+ if [ $IS_MAIN_REPO == 'false' ]; then
+ composeProfile="openremote/$composeProfile"
+ fi
+
+ # Start the stack
+ MANAGER_VERSION=$MANAGER_TAG docker compose -f $composeProfile up -d --no-build
+
+ # Run the tests
+ ${{ steps.test-ui-command.outputs.value }}
+ env:
+ IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}
+ MANAGER_TAG: ${{ steps.manager-docker-command.outputs.refTag }}
+ timeout-minutes: 20
+ continue-on-error: true
+
- name: Run install dist
if: steps.install-command.outputs.value != ''
shell: python
@@ -789,26 +727,6 @@ jobs:
run: |
${{ steps.maven-publish-command.outputs.value }}
- - name: Run frontend tests
- if: steps.test-ui-command.outputs.value != ''
- run: |
- composeProfile='profile/dev-ui.yml'
-
- if [ $IS_MAIN_REPO == 'false' ]; then
- composeProfile="openremote/$composeProfile"
- fi
-
- # Start the stack
- MANAGER_VERSION=$MANAGER_TAG docker compose -f $composeProfile up -d --no-build
-
- # Run the tests
- ${{ steps.test-ui-command.outputs.value }}
- env:
- IS_MAIN_REPO: ${{ steps.is_main_repo.outputs.value }}
- MANAGER_TAG: ${{ steps.manager-docker-command.outputs.refTag }}
- timeout-minutes: 20
- continue-on-error: true
-
- name: Run npm publish command
if: ${{ steps.ci-cd-output.outputs.npmTag != '' }}
run: |
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index fc4ba75c6..a6c9289fd 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -34,11 +34,11 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
- - name: Set up JDK 17 and gradle cache
+ - name: Set up JDK 21 and gradle cache
uses: actions/setup-java@v4
with:
distribution: 'temurin'
- java-version: '17'
+ java-version: '21'
cache: 'gradle'
- name: Run release