100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Keycloak URL generation cleanup (#856)
Rich Turner6428014b292e
diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.ymlindex 4258462bf..341d4125d 100644--- a/.github/workflows/ci_cd.yml+++ b/.github/workflows/ci_cd.yml@@ -58,7 +58,7 @@ on:# Push on master excluding tagspush:branches:- - 'master'+ - 'master'tags-ignore:- '*.*'@@ -101,25 +101,25 @@ jobs:runs-on: ubuntu-lateststeps:-+- name: Cancel previous runsuses: styfle/[email protected]with:access_token: ${{ github.token }}-+- name: Get inputs and secretsid: inputs-and-secretsshell: pythonrun: |import osimport json-+# Overlay all inputs and secrets onto this jobs outputscallerInputs = os.getenv("CALLER_INPUTS")inputs = os.getenv("INPUTS")secrets = os.getenv("SECRETS")eventName = os.getenv("EVENT_NAME")-+if inputs is not None and inputs != '':inputs = json.loads(inputs)if secrets is not None and secrets != '':@@ -127,14 +127,14 @@ jobs:if eventName == 'workflow_call' and callerInputs is not None and callerInputs != 'null':os.system(f"echo 'Processing caller inputs'")inputs = json.loads(callerInputs)-+if inputs is not None and 'INPUTS' in inputs:os.system("echo 'Processing inputs from caller'")inputs = json.loads(inputs['INPUTS'])if 'SECRETS' in secrets:os.system("echo 'Processing secrets from caller'")secrets = json.loads(secrets['SECRETS'])-+# Iterate over secrets then inputs and assign them as outputs on this stepif secrets is not None and secrets != 'null':for key, value in secrets.items():@@ -143,7 +143,7 @@ jobs:value=value.replace('\n','%0A')value=value.replace('\r','%0D')os.system(f"echo '::set-output name={key}::{value}'")-+if inputs is not None and inputs != 'null':for key, value in inputs.items():os.system(f"echo 'Outputting input: {key}'")@@ -151,17 +151,17 @@ jobs:value=value.replace('\n','%0A')value=value.replace('\r','%0D')os.system(f"echo '::set-output name={key}::{value}'")-+env:CALLER_INPUTS: ${{ toJSON(inputs) }}SECRETS: ${{ toJSON(secrets) }}INPUTS: ${{ toJSON(github.event.inputs) }}EVENT_NAME: ${{ github.event_name }}-+- name: Public IPid: ip-addressuses: haythem/[email protected]-+- name: Checkoutuses: actions/checkout@v3with:@@ -246,7 +246,7 @@ jobs:uses: andstor/file-existence-action@v1with:files: ".ci_cd/ci_cd.json"-+- name: Check is custom project when scheduled triggerif: ${{ github.event_name == 'schedule' }}run: |@@ -266,17 +266,17 @@ jobs:run: |import jsonimport os-+eventName = os.getenv('EVENT_NAME')refName = os.getenv('REF_NAME')pushTag = NonedeployTag = NonedeployEnvironment = None-+f = open(".ci_cd/ci_cd.json")data = json.load(f)f.close()-+if "managerDockerPush" in data:managerPushConfig = data["managerDockerPush"]@@ -291,7 +291,7 @@ jobs:env:EVENT_NAME: ${{ github.event_name }}REF_NAME: ${{ github.ref_name }}-+- name: Load manager tag cacheif: ${{ steps.check_cicd_json.outputs.files_exists == 'true' && github.event_name == 'schedule' }}uses: actions/cache@v3@@ -310,7 +310,7 @@ jobs:run: |import jsonimport os-+eventName = os.getenv('EVENT_NAME')refName = os.getenv('REF_NAME')isMainRepo = os.getenv('IS_MAIN_REPO')@@ -318,7 +318,7 @@ jobs:dockerPublishTags = NonemavenPublishTag = NonedeployEnvironment = None-+f = open(".ci_cd/ci_cd.json")data = json.load(f)f.close()@@ -328,7 +328,7 @@ jobs:if data is not None and eventName in data:eventConfig = data[eventName]-+if eventName == "managerDockerPush":# Iterate through keys and check if that manager docker image has changed since last execution@@ -359,7 +359,7 @@ jobs:os.system(f"echo 'No deploy config in ci_cd.json for manager tag {key}'")else:os.system(f"echo 'Manager docker tag has not changed: {key}'")-+elif eventName == "push" and refName in eventConfig:eventConfig = eventConfig[refName]@@ -378,7 +378,7 @@ jobs:dockerPublishTags = eventConfig['distribute']['docker']if 'maven' in eventConfig['distribute']:mavenPublishTag = eventConfig['distribute']['maven']-+if dockerPublishTags is not None and isMainRepo == 'true':dockerPublishTags = dockerPublishTags.replace("$version", refName)firstDockerTag = dockerPublishTags.split(",")[0]@@ -386,19 +386,19 @@ jobs:os.system(f" echo 'Manager tags to push to docker: {dockerPublishTags}'")dockerPublishTags = " ".join(map(lambda t: f"-t openremote/manager:{t.strip()}", dockerPublishTags.split(",")))os.system(f"echo ::set-output name=dockerTags::{dockerPublishTags}")-+if mavenPublishTag is not None and isMainRepo == 'true':mavenPublishTag = mavenPublishTag.replace("$version", refName)os.system(f" echo 'Maven publish version: {mavenPublishTag}'")os.system(f"echo ::set-output name=mavenTag::{mavenPublishTag}")-+deployStr = Noneif deploys is not None:if not isinstance(deploys, list):deploys = [deploys]-+deployStr = ""-+for deploy in deploys:if 'environment' in deploy:deployStr += deploy['environment']@@ -408,11 +408,11 @@ jobs:else:os.system("echo 'Manager tag not specified so using commit SHA'")deployStr += '#ref'-+deployStr += ";"deployStr = deployStr.rstrip(";")-+if deployStr is not None and len(deployStr) > 0:print(f"Deployments to deploy: {deployStr}")os.system(f"echo ::set-output name=deploys::{deployStr}")@@ -426,12 +426,12 @@ jobs:run: |cd ~/manager-tags-newcp -r . ~/manager-tags-old/-+- name: Sanitize deployments valueid: deploymentsrun: |deployments=$DEPLOYMENTS-+if [ "$EVENT_NAME" == 'workflow_dispatch' ]; thentag="$INPUT_MANAGER_TAG"if [ -z "$INPUT_MANAGER_TAG" ]; then@@ -455,7 +455,7 @@ jobs:id: test-ui-commandif: ${{ steps.skip-ui-tests.outputs.value != 'true' }}run: echo ""-+- name: Define manager docker build commandid: manager-docker-commandshell: bash@@ -512,13 +512,13 @@ jobs:echo "Deployment must have a build.gradle file to prepare the deployment files in the deployment/build dir"exit 1fi-+buildPath="deployment/build"commitSha=$(git rev-parse HEAD)commitShaShort=$(git rev-parse --short HEAD)echo "::set-output name=buildPath::$buildPath"echo "::set-output name=refTag::$commitShaShort"-+if [ -n "$DEPLOYMENTS" ]; thencommand="docker build --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 -t openremote/deployment:$commitShaShort $buildPath"echo "::set-output name=value::$command"@@ -549,7 +549,7 @@ jobs:with:username: ${{ steps.inputs-and-secrets.outputs._TEMP_DOCKERHUB_USER || steps.inputs-and-secrets.outputs.DOCKERHUB_USER }}password: ${{ steps.inputs-and-secrets.outputs._TEMP_DOCKERHUB_PASSWORD || steps.inputs-and-secrets.outputs.DOCKERHUB_PASSWORD }}-+- name: set up QEMUif: ${{ steps.manager-docker-command.outputs.value != '' || steps.deployment-docker-command.outputs.value != '' }}uses: docker/setup-qemu-action@v2@@ -624,11 +624,11 @@ jobs:elsecomposeProfile='profile/dev-ui.yml'fi-+if [ $IS_CUSTOM_PROJECT == 'true' ]; thencomposeProfile="openremote/$composeProfile"fi-+# Pull the imagesdocker-compose -f $composeProfile pullenv:@@ -645,20 +645,19 @@ jobs:id: run-backend-testsif: ${{ steps.test-backend-command.outputs.value != '' }}run: |-composeProfile='profile/dev-testing.yml'-+if [ $IS_CUSTOM_PROJECT == 'true' ]; thencomposeProfile="openremote/$composeProfile"fi-+# Define cleanup commandecho "::set-output name=cleanup::docker-compose -f $composeProfile down"-+# Start the stackecho "docker-compose -f ${composeProfile} up -d --no-build"docker-compose -f ${composeProfile} up -d --no-build-+# Run the tests${{ steps.test-backend-command.outputs.value }}env:@@ -700,13 +699,13 @@ jobs:envFile = "temp.env"else:continue-+os.system(f"echo 'Setting environment variable {key}...'")os.putenv(key, value)-+buildCmd = os.getenv("CMD")result = subprocess.run(f"bash {buildCmd}", shell=True)-+if result.returncode != 0:os.system("echo 'installDist failed'")sys.exit(result.returncode)@@ -715,7 +714,7 @@ jobs:CMD: ${{ steps.install-command.outputs.value }}INPUTS_AND_SECRETS: ${{ toJSON(steps.inputs-and-secrets.outputs) }}timeout-minutes: 20-+- name: Run manager docker build commandif: steps.manager-docker-command.outputs.value != ''run: |@@ -730,14 +729,14 @@ jobs:if: steps.test-ui-command.outputs.value != ''run: |composeProfile='profile/dev-ui.yml'-+if [ $IS_CUSTOM_PROJECT == 'true' ]; thencomposeProfile="openremote/$composeProfile"fi# Start the stackMANAGER_VERSION=$MANAGER_TAG docker-compose -f $composeProfile up -d --no-build-+# Run the tests${{ steps.test-ui-command.outputs.value }}env:@@ -755,12 +754,11 @@ jobs:if: steps.deployments.outputs.value != ''shell: pythonrun: |-import jsonimport osimport sysimport subprocess-+deployments = os.getenv("DEPLOYMENTS")deployments = deployments.split(";")managerRef = os.getenv("MANAGER_REF")@@ -769,15 +767,15 @@ jobs:inputsAndSecrets = json.loads(os.getenv("INPUTS_AND_SECRETS"))ipv4 = os.getenv("IPV4")ipv6 = os.getenv("IPV6")-+failure = False-+# Determine deploy script to usedeployScript = ".ci_cd/deploy.sh"-+if not os.path.exists(deployScript) and isCustomProject == 'true':deployScript = "openremote/.ci_cd/deploy.sh"-+if not os.path.exists(deployScript):os.system(f"Deploy script not found '{deployScript}'")sys.exit(1)@@ -787,25 +785,25 @@ jobs:env = dep[0]managerTag = dep[1]managerTagFound = True-+os.putenv("MANAGER_TAG", managerTag)os.putenv("ENVIRONMENT", env)-+# Clean stale ssh credentials and temp filesos.system("rm temp.env 2>/dev/null")os.system("rm ssh.key 2>/dev/null")os.system("rm -r temp 2>/dev/null")os.system("mkdir temp")-+# ------------------------------------------------------# Output env variables to temp env file for POSIX shell# -------------------------------------------------------+# Output inputs and secrets (spacial handling for SSH_KEY and some other variables)# _$ENV_ prefixed keys are output last (to override any non env specific keys)environment = (env if env else "").upper()prefix = "_" + environment + "_"-+for key, value in inputsAndSecrets.items():if "." in key:continue@@ -857,12 +855,12 @@ jobs:# Uncomment this in combination with the SSH debug step afterwards to debug deployment script#sys.exit(0)result = subprocess.run(f"bash {deployScript}", shell=True)-+if result.returncode != 0:os.system(f"echo 'Deployment failed: managerTag={managerTag} deploymentTag={deploymentRef} environment={env}'")failure = Truecontinue-+if failure == True:os.system("echo 'One or more deployments failed'")sys.exit(1)diff --git a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.javaindex b54728df9..8d23e39cc 100644--- a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java+++ b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java@@ -121,8 +121,8 @@ public class ManagerKeycloakIdentityProvider extends KeycloakIdentityProvider imsetActiveCredentials(grant);}- String hostname = getString(container.getConfig(), Constants.OR_HOSTNAME, "localhost");- int port = getInteger(container.getConfig(), Constants.OR_SSL_PORT, 8080); // Should just be called port+ String hostname = getString(container.getConfig(), Constants.OR_HOSTNAME, null);+ int port = getInteger(container.getConfig(), Constants.OR_SSL_PORT, -1); // Should just be called portString keycloakHostname = getString(container.getConfig(), KC_HOSTNAME, hostname);int keycloakPort = getInteger(container.getConfig(), KC_HOSTNAME_PORT, port);String keycloakHostnamePath = getString(container.getConfig(), KC_HOSTNAME_PATH, "auth");diff --git a/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.javaindex 30ebd7c1c..3cd1372ed 100644--- a/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java@@ -20,6 +20,7 @@package org.openremote.manager.system;import com.fasterxml.jackson.databind.node.ObjectNode;+import io.grpc.netty.shaded.io.netty.internal.tcnative.SSL;import org.openremote.model.Constants;import org.openremote.model.Container;import org.openremote.model.ContainerService;@@ -57,6 +58,10 @@ public class SslHealthStatusProvider implements X509TrustManager, HealthStatusPrpublic void init(Container container) throws Exception {int SSLPort = getInteger(container.getConfig(), Constants.OR_SSL_PORT, -1);++ if (SSLPort < 0) {+ SSLPort = 443;+ }if (SSLPort > 0 && SSLPort <= 65536) {this.SSLPort = SSLPort;diff --git a/profile/dev-ui.yml b/profile/dev-ui.ymlindex 60bd50461..228338aaa 100644--- a/profile/dev-ui.yml+++ b/profile/dev-ui.yml@@ -56,6 +56,5 @@ services:environment:OR_SETUP_RUN_ON_RESTART: ${OR_SETUP_RUN_ON_RESTART:-true}OR_DEV_MODE: ${OR_DEV_MODE:-true}- KC_HOSTNAME_PORT: ${KC_HOSTNAME_PORT:-8080}ports:- "8080:8080"diff --git a/project.gradle b/project.gradleindex b4b10c73b..8e680547a 100644--- a/project.gradle+++ b/project.gradle@@ -103,7 +103,7 @@ if (project == rootProject) {envs = [OR_SETUP_TYPE: "demo",OR_WEBSERVER_LISTEN_HOST: "0.0.0.0",- OR_SSL_PORT: "443"+ OR_SSL_PORT: "-1"]}"Test Setup"(Application) {@@ -113,7 +113,7 @@ if (project == rootProject) {moduleName = getProject().idea.module.name + (isCustomProject() ? ".openremote.setup.integration" : ".setup.integration")envs = [OR_WEBSERVER_LISTEN_HOST: "0.0.0.0",- OR_SSL_PORT: "443"+ OR_SSL_PORT: "-1"]}"Empty"(Application) {@@ -123,7 +123,7 @@ if (project == rootProject) {moduleName = getProject().idea.module.name + (isCustomProject() ? ".openremote.manager.main" : ".manager.main")envs = [OR_WEBSERVER_LISTEN_HOST: "0.0.0.0",- OR_SSL_PORT: "443"+ OR_SSL_PORT: "-1"]}}diff --git a/test/src/test/groovy/org/openremote/test/provisioning/UserAndAssetProvisioningTest.groovy b/test/src/test/groovy/org/openremote/test/provisioning/UserAndAssetProvisioningTest.groovyindex 4f87b21cf..2f7dee89a 100644--- a/test/src/test/groovy/org/openremote/test/provisioning/UserAndAssetProvisioningTest.groovy+++ b/test/src/test/groovy/org/openremote/test/provisioning/UserAndAssetProvisioningTest.groovy@@ -445,6 +445,8 @@ class UserAndAssetProvisioningTest extends Specification implements ManagerContadeviceNUniqueId = "device4"deviceNRequestTopic = "$PROVISIONING_TOKEN/$deviceNUniqueId/$REQUEST_TOKEN".toString()deviceNResponseTopic = "$PROVISIONING_TOKEN/$deviceNUniqueId/$RESPONSE_TOKEN".toString()+ deviceNClient = new MQTT_IOClient(mqttDeviceNClientId, mqttHost, mqttPort, false, false, null, null, null)+ deviceNClient.setTopicSubscribeFailureConsumer(subscribeFailureCallback)deviceNClient.connect()then: "mqtt connection should exist"@@ -492,6 +494,8 @@ class UserAndAssetProvisioningTest extends Specification implements ManagerContadeviceNUniqueId = "device3"deviceNRequestTopic = "$PROVISIONING_TOKEN/$deviceNUniqueId/$REQUEST_TOKEN".toString()deviceNResponseTopic = "$PROVISIONING_TOKEN/$deviceNUniqueId/$RESPONSE_TOKEN".toString()+ deviceNClient = new MQTT_IOClient(mqttDeviceNClientId, mqttHost, mqttPort, false, false, null, null, null)+ deviceNClient.setTopicSubscribeFailureConsumer(subscribeFailureCallback)deviceNClient.connect()then: "mqtt connection should exist"@@ -573,6 +577,8 @@ class UserAndAssetProvisioningTest extends Specification implements ManagerContawhen: "another device re-connects"deviceNResponses.clear()+ deviceNClient = new MQTT_IOClient(mqttDeviceNClientId, mqttHost, mqttPort, false, false, null, null, null)+ deviceNClient.setTopicSubscribeFailureConsumer(subscribeFailureCallback)deviceNClient.connect()then: "mqtt connection should exist"@@ -671,6 +677,8 @@ class UserAndAssetProvisioningTest extends Specification implements ManagerContawhen: "a device with disabled user account connects"deviceNResponses.clear()+ deviceNClient = new MQTT_IOClient(mqttDeviceNClientId, mqttHost, mqttPort, false, false, null, null, null)+ deviceNClient.setTopicSubscribeFailureConsumer(subscribeFailureCallback)deviceNClient.connect()then: "mqtt connection should exist"diff --git a/ui/component/core/src/index.ts b/ui/component/core/src/index.tsindex 49cd4fc5d..0138d9952 100644--- a/ui/component/core/src/index.ts+++ b/ui/component/core/src/index.ts@@ -132,15 +132,8 @@ export function normaliseConfig(config: ManagerConfig): ManagerConfig {normalisedConfig.realm = "master";}- if (normalisedConfig.auth === Auth.KEYCLOAK) {- // Determine URL of keycloak server- if (!normalisedConfig.keycloakUrl || normalisedConfig.keycloakUrl === "") {- // Assume keycloak is running on same host as the manager- normalisedConfig.keycloakUrl = normalisedConfig.managerUrl + "/auth";- }-- // Normalise by stripping any trailing slashes- normalisedConfig.keycloakUrl = normalisedConfig.keycloakUrl.replace(/\/+$/, "");+ if (!normalisedConfig.auth) {+ normalisedConfig.auth = Auth.KEYCLOAK;}if (normalisedConfig.consoleAutoEnable === undefined) {@@ -356,12 +349,43 @@ export class Manager implements EventProviderFactory {// BASIC auth will likely require UI so lets init translation at leastsuccess = await this.doTranslateInit() && success;success = await this.doAuthInit();- } else {+ } else if (this._config.auth === Auth.KEYCLOAK) {++ // The info endpoint of the manager might return a relative URL (relative to the manager)+ if (!this._config.keycloakUrl && this._authServerUrl) {+ const managerURL = new URL(this._config.managerUrl!);+ let authServerURL: URL;++ try {+ authServerURL = new URL(this._authServerUrl);+ } catch (e) {+ // Could be a relative URL+ authServerURL = new URL(managerURL);+ authServerURL.pathname = this._authServerUrl;+ }++ // Use manager URL info+ if (!authServerURL.protocol) {+ authServerURL.protocol = managerURL.protocol;+ }+ if (!authServerURL.hostname) {+ authServerURL.hostname = managerURL.hostname;+ }+ if (!authServerURL.port) {+ authServerURL.port = managerURL.port;+ }+ this._config.keycloakUrl = authServerURL.toString();+ }++ // If we still don't know auth server URL then use manager URLif (!this._config.keycloakUrl) {- this.config.keycloakUrl = this._authServerUrl;+ this._config.keycloakUrl = this._config.managerUrl + "/auth";}+ // Normalise by stripping any trailing slashes+ this._config.keycloakUrl = this._config.keycloakUrl.replace(/\/+$/, "");+success = await this.doAuthInit();// If failed then we can assume keycloak auth requested but unavailable