Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Keycloak URL generation cleanup (#856)

Rich Turner6428014b292e
diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml
index 4258462bf..341d4125d 100644
--- a/.github/workflows/ci_cd.yml
+++ b/.github/workflows/ci_cd.yml
@@ -58,7 +58,7 @@ on:
# Push on master excluding tags
push:
branches:
- - 'master'
+ - 'master'
tags-ignore:
- '*.*'
@@ -101,25 +101,25 @@ jobs:
runs-on: ubuntu-latest
steps:
-
+
- name: Cancel previous runs
uses: styfle/[email protected]
with:
access_token: ${{ github.token }}
-
+
- name: Get inputs and secrets
id: inputs-and-secrets
shell: python
run: |
import os
import json
-
+
# Overlay all inputs and secrets onto this jobs outputs
callerInputs = os.getenv("CALLER_INPUTS")
inputs = os.getenv("INPUTS")
secrets = os.getenv("SECRETS")
eventName = os.getenv("EVENT_NAME")
-
+
if inputs is not None and inputs != '':
inputs = json.loads(inputs)
if secrets is not None and secrets != '':
@@ -127,14 +127,14 @@ jobs:
if eventName == 'workflow_call' and callerInputs is not None and callerInputs != 'null':
os.system(f"echo 'Processing caller inputs'")
inputs = json.loads(callerInputs)
-
+
if inputs is not None and 'INPUTS' in inputs:
os.system("echo 'Processing inputs from caller'")
inputs = json.loads(inputs['INPUTS'])
if 'SECRETS' in secrets:
os.system("echo 'Processing secrets from caller'")
secrets = json.loads(secrets['SECRETS'])
-
+
# Iterate over secrets then inputs and assign them as outputs on this step
if secrets is not None and secrets != 'null':
for key, value in secrets.items():
@@ -143,7 +143,7 @@ jobs:
value=value.replace('\n','%0A')
value=value.replace('\r','%0D')
os.system(f"echo '::set-output name={key}::{value}'")
-
+
if inputs is not None and inputs != 'null':
for key, value in inputs.items():
os.system(f"echo 'Outputting input: {key}'")
@@ -151,17 +151,17 @@ jobs:
value=value.replace('\n','%0A')
value=value.replace('\r','%0D')
os.system(f"echo '::set-output name={key}::{value}'")
-
+
env:
CALLER_INPUTS: ${{ toJSON(inputs) }}
SECRETS: ${{ toJSON(secrets) }}
INPUTS: ${{ toJSON(github.event.inputs) }}
EVENT_NAME: ${{ github.event_name }}
-
+
- name: Public IP
id: ip-address
uses: haythem/[email protected]
-
+
- name: Checkout
uses: actions/checkout@v3
with:
@@ -246,7 +246,7 @@ jobs:
uses: andstor/file-existence-action@v1
with:
files: ".ci_cd/ci_cd.json"
-
+
- name: Check is custom project when scheduled trigger
if: ${{ github.event_name == 'schedule' }}
run: |
@@ -266,17 +266,17 @@ jobs:
run: |
import json
import os
-
+
eventName = os.getenv('EVENT_NAME')
refName = os.getenv('REF_NAME')
pushTag = None
deployTag = None
deployEnvironment = None
-
+
f = open(".ci_cd/ci_cd.json")
data = json.load(f)
f.close()
-
+
if "managerDockerPush" in data:
managerPushConfig = data["managerDockerPush"]
@@ -291,7 +291,7 @@ jobs:
env:
EVENT_NAME: ${{ github.event_name }}
REF_NAME: ${{ github.ref_name }}
-
+
- name: Load manager tag cache
if: ${{ steps.check_cicd_json.outputs.files_exists == 'true' && github.event_name == 'schedule' }}
uses: actions/cache@v3
@@ -310,7 +310,7 @@ jobs:
run: |
import json
import os
-
+
eventName = os.getenv('EVENT_NAME')
refName = os.getenv('REF_NAME')
isMainRepo = os.getenv('IS_MAIN_REPO')
@@ -318,7 +318,7 @@ jobs:
dockerPublishTags = None
mavenPublishTag = None
deployEnvironment = None
-
+
f = open(".ci_cd/ci_cd.json")
data = json.load(f)
f.close()
@@ -328,7 +328,7 @@ jobs:
if data is not None and eventName in data:
eventConfig = data[eventName]
-
+
if eventName == "managerDockerPush":
# Iterate through keys and check if that manager docker image has changed since last execution
@@ -359,7 +359,7 @@ jobs:
os.system(f"echo 'No deploy config in ci_cd.json for manager tag {key}'")
else:
os.system(f"echo 'Manager docker tag has not changed: {key}'")
-
+
elif eventName == "push" and refName in eventConfig:
eventConfig = eventConfig[refName]
@@ -378,7 +378,7 @@ jobs:
dockerPublishTags = eventConfig['distribute']['docker']
if 'maven' in eventConfig['distribute']:
mavenPublishTag = eventConfig['distribute']['maven']
-
+
if dockerPublishTags is not None and isMainRepo == 'true':
dockerPublishTags = dockerPublishTags.replace("$version", refName)
firstDockerTag = dockerPublishTags.split(",")[0]
@@ -386,19 +386,19 @@ jobs:
os.system(f" echo 'Manager tags to push to docker: {dockerPublishTags}'")
dockerPublishTags = " ".join(map(lambda t: f"-t openremote/manager:{t.strip()}", dockerPublishTags.split(",")))
os.system(f"echo ::set-output name=dockerTags::{dockerPublishTags}")
-
+
if mavenPublishTag is not None and isMainRepo == 'true':
mavenPublishTag = mavenPublishTag.replace("$version", refName)
os.system(f" echo 'Maven publish version: {mavenPublishTag}'")
os.system(f"echo ::set-output name=mavenTag::{mavenPublishTag}")
-
+
deployStr = None
if deploys is not None:
if not isinstance(deploys, list):
deploys = [deploys]
-
+
deployStr = ""
-
+
for deploy in deploys:
if 'environment' in deploy:
deployStr += deploy['environment']
@@ -408,11 +408,11 @@ jobs:
else:
os.system("echo 'Manager tag not specified so using commit SHA'")
deployStr += '#ref'
-
+
deployStr += ";"
deployStr = deployStr.rstrip(";")
-
+
if deployStr is not None and len(deployStr) > 0:
print(f"Deployments to deploy: {deployStr}")
os.system(f"echo ::set-output name=deploys::{deployStr}")
@@ -426,12 +426,12 @@ jobs:
run: |
cd ~/manager-tags-new
cp -r . ~/manager-tags-old/
-
+
- name: Sanitize deployments value
id: deployments
run: |
deployments=$DEPLOYMENTS
-
+
if [ "$EVENT_NAME" == 'workflow_dispatch' ]; then
tag="$INPUT_MANAGER_TAG"
if [ -z "$INPUT_MANAGER_TAG" ]; then
@@ -455,7 +455,7 @@ jobs:
id: test-ui-command
if: ${{ steps.skip-ui-tests.outputs.value != 'true' }}
run: echo ""
-
+
- name: Define manager docker build command
id: manager-docker-command
shell: bash
@@ -512,13 +512,13 @@ jobs:
echo "Deployment must have a build.gradle file to prepare the deployment files in the deployment/build dir"
exit 1
fi
-
+
buildPath="deployment/build"
commitSha=$(git rev-parse HEAD)
commitShaShort=$(git rev-parse --short HEAD)
echo "::set-output name=buildPath::$buildPath"
echo "::set-output name=refTag::$commitShaShort"
-
+
if [ -n "$DEPLOYMENTS" ]; then
command="docker build --build-arg GIT_COMMIT=$commitSha --platform linux/amd64,linux/aarch64 -t openremote/deployment:$commitShaShort $buildPath"
echo "::set-output name=value::$command"
@@ -549,7 +549,7 @@ jobs:
with:
username: ${{ steps.inputs-and-secrets.outputs._TEMP_DOCKERHUB_USER || steps.inputs-and-secrets.outputs.DOCKERHUB_USER }}
password: ${{ steps.inputs-and-secrets.outputs._TEMP_DOCKERHUB_PASSWORD || steps.inputs-and-secrets.outputs.DOCKERHUB_PASSWORD }}
-
+
- name: set up QEMU
if: ${{ steps.manager-docker-command.outputs.value != '' || steps.deployment-docker-command.outputs.value != '' }}
uses: docker/setup-qemu-action@v2
@@ -624,11 +624,11 @@ jobs:
else
composeProfile='profile/dev-ui.yml'
fi
-
+
if [ $IS_CUSTOM_PROJECT == 'true' ]; then
composeProfile="openremote/$composeProfile"
fi
-
+
# Pull the images
docker-compose -f $composeProfile pull
env:
@@ -645,20 +645,19 @@ jobs:
id: run-backend-tests
if: ${{ steps.test-backend-command.outputs.value != '' }}
run: |
-
composeProfile='profile/dev-testing.yml'
-
+
if [ $IS_CUSTOM_PROJECT == 'true' ]; then
composeProfile="openremote/$composeProfile"
fi
-
+
# Define cleanup command
echo "::set-output name=cleanup::docker-compose -f $composeProfile down"
-
+
# Start the stack
echo "docker-compose -f ${composeProfile} up -d --no-build"
docker-compose -f ${composeProfile} up -d --no-build
-
+
# Run the tests
${{ steps.test-backend-command.outputs.value }}
env:
@@ -700,13 +699,13 @@ jobs:
envFile = "temp.env"
else:
continue
-
+
os.system(f"echo 'Setting environment variable {key}...'")
os.putenv(key, value)
-
+
buildCmd = os.getenv("CMD")
result = subprocess.run(f"bash {buildCmd}", shell=True)
-
+
if result.returncode != 0:
os.system("echo 'installDist failed'")
sys.exit(result.returncode)
@@ -715,7 +714,7 @@ jobs:
CMD: ${{ steps.install-command.outputs.value }}
INPUTS_AND_SECRETS: ${{ toJSON(steps.inputs-and-secrets.outputs) }}
timeout-minutes: 20
-
+
- name: Run manager docker build command
if: steps.manager-docker-command.outputs.value != ''
run: |
@@ -730,14 +729,14 @@ jobs:
if: steps.test-ui-command.outputs.value != ''
run: |
composeProfile='profile/dev-ui.yml'
-
+
if [ $IS_CUSTOM_PROJECT == 'true' ]; then
composeProfile="openremote/$composeProfile"
fi
# Start the stack
MANAGER_VERSION=$MANAGER_TAG docker-compose -f $composeProfile up -d --no-build
-
+
# Run the tests
${{ steps.test-ui-command.outputs.value }}
env:
@@ -755,12 +754,11 @@ jobs:
if: steps.deployments.outputs.value != ''
shell: python
run: |
-
import json
import os
import sys
import subprocess
-
+
deployments = os.getenv("DEPLOYMENTS")
deployments = deployments.split(";")
managerRef = os.getenv("MANAGER_REF")
@@ -769,15 +767,15 @@ jobs:
inputsAndSecrets = json.loads(os.getenv("INPUTS_AND_SECRETS"))
ipv4 = os.getenv("IPV4")
ipv6 = os.getenv("IPV6")
-
+
failure = False
-
+
# Determine deploy script to use
deployScript = ".ci_cd/deploy.sh"
-
+
if not os.path.exists(deployScript) and isCustomProject == 'true':
deployScript = "openremote/.ci_cd/deploy.sh"
-
+
if not os.path.exists(deployScript):
os.system(f"Deploy script not found '{deployScript}'")
sys.exit(1)
@@ -787,25 +785,25 @@ jobs:
env = dep[0]
managerTag = dep[1]
managerTagFound = True
-
+
os.putenv("MANAGER_TAG", managerTag)
os.putenv("ENVIRONMENT", env)
-
+
# Clean stale ssh credentials and temp files
os.system("rm temp.env 2>/dev/null")
os.system("rm ssh.key 2>/dev/null")
os.system("rm -r temp 2>/dev/null")
os.system("mkdir temp")
-
+
# ------------------------------------------------------
# Output env variables to temp env file for POSIX shell
# ------------------------------------------------------
-
+
# Output inputs and secrets (spacial handling for SSH_KEY and some other variables)
# _$ENV_ prefixed keys are output last (to override any non env specific keys)
environment = (env if env else "").upper()
prefix = "_" + environment + "_"
-
+
for key, value in inputsAndSecrets.items():
if "." in key:
continue
@@ -857,12 +855,12 @@ jobs:
# Uncomment this in combination with the SSH debug step afterwards to debug deployment script
#sys.exit(0)
result = subprocess.run(f"bash {deployScript}", shell=True)
-
+
if result.returncode != 0:
os.system(f"echo 'Deployment failed: managerTag={managerTag} deploymentTag={deploymentRef} environment={env}'")
failure = True
continue
-
+
if failure == True:
os.system("echo 'One or more deployments failed'")
sys.exit(1)
diff --git a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java
index b54728df9..8d23e39cc 100644
--- a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java
+++ b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java
@@ -121,8 +121,8 @@ public class ManagerKeycloakIdentityProvider extends KeycloakIdentityProvider im
setActiveCredentials(grant);
}
- String hostname = getString(container.getConfig(), Constants.OR_HOSTNAME, "localhost");
- int port = getInteger(container.getConfig(), Constants.OR_SSL_PORT, 8080); // Should just be called port
+ String hostname = getString(container.getConfig(), Constants.OR_HOSTNAME, null);
+ int port = getInteger(container.getConfig(), Constants.OR_SSL_PORT, -1); // Should just be called port
String keycloakHostname = getString(container.getConfig(), KC_HOSTNAME, hostname);
int keycloakPort = getInteger(container.getConfig(), KC_HOSTNAME_PORT, port);
String keycloakHostnamePath = getString(container.getConfig(), KC_HOSTNAME_PATH, "auth");
diff --git a/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java
index 30ebd7c1c..3cd1372ed 100644
--- a/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java
+++ b/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java
@@ -20,6 +20,7 @@
package org.openremote.manager.system;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.grpc.netty.shaded.io.netty.internal.tcnative.SSL;
import org.openremote.model.Constants;
import org.openremote.model.Container;
import org.openremote.model.ContainerService;
@@ -57,6 +58,10 @@ public class SslHealthStatusProvider implements X509TrustManager, HealthStatusPr
public void init(Container container) throws Exception {
int SSLPort = getInteger(container.getConfig(), Constants.OR_SSL_PORT, -1);
+
+ if (SSLPort < 0) {
+ SSLPort = 443;
+ }
if (SSLPort > 0 && SSLPort <= 65536) {
this.SSLPort = SSLPort;
diff --git a/profile/dev-ui.yml b/profile/dev-ui.yml
index 60bd50461..228338aaa 100644
--- a/profile/dev-ui.yml
+++ b/profile/dev-ui.yml
@@ -56,6 +56,5 @@ services:
environment:
OR_SETUP_RUN_ON_RESTART: ${OR_SETUP_RUN_ON_RESTART:-true}
OR_DEV_MODE: ${OR_DEV_MODE:-true}
- KC_HOSTNAME_PORT: ${KC_HOSTNAME_PORT:-8080}
ports:
- "8080:8080"
diff --git a/project.gradle b/project.gradle
index b4b10c73b..8e680547a 100644
--- a/project.gradle
+++ b/project.gradle
@@ -103,7 +103,7 @@ if (project == rootProject) {
envs = [
OR_SETUP_TYPE: "demo",
OR_WEBSERVER_LISTEN_HOST: "0.0.0.0",
- OR_SSL_PORT: "443"
+ OR_SSL_PORT: "-1"
]
}
"Test Setup"(Application) {
@@ -113,7 +113,7 @@ if (project == rootProject) {
moduleName = getProject().idea.module.name + (isCustomProject() ? ".openremote.setup.integration" : ".setup.integration")
envs = [
OR_WEBSERVER_LISTEN_HOST: "0.0.0.0",
- OR_SSL_PORT: "443"
+ OR_SSL_PORT: "-1"
]
}
"Empty"(Application) {
@@ -123,7 +123,7 @@ if (project == rootProject) {
moduleName = getProject().idea.module.name + (isCustomProject() ? ".openremote.manager.main" : ".manager.main")
envs = [
OR_WEBSERVER_LISTEN_HOST: "0.0.0.0",
- OR_SSL_PORT: "443"
+ OR_SSL_PORT: "-1"
]
}
}
diff --git a/test/src/test/groovy/org/openremote/test/provisioning/UserAndAssetProvisioningTest.groovy b/test/src/test/groovy/org/openremote/test/provisioning/UserAndAssetProvisioningTest.groovy
index 4f87b21cf..2f7dee89a 100644
--- a/test/src/test/groovy/org/openremote/test/provisioning/UserAndAssetProvisioningTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/provisioning/UserAndAssetProvisioningTest.groovy
@@ -445,6 +445,8 @@ class UserAndAssetProvisioningTest extends Specification implements ManagerConta
deviceNUniqueId = "device4"
deviceNRequestTopic = "$PROVISIONING_TOKEN/$deviceNUniqueId/$REQUEST_TOKEN".toString()
deviceNResponseTopic = "$PROVISIONING_TOKEN/$deviceNUniqueId/$RESPONSE_TOKEN".toString()
+ deviceNClient = new MQTT_IOClient(mqttDeviceNClientId, mqttHost, mqttPort, false, false, null, null, null)
+ deviceNClient.setTopicSubscribeFailureConsumer(subscribeFailureCallback)
deviceNClient.connect()
then: "mqtt connection should exist"
@@ -492,6 +494,8 @@ class UserAndAssetProvisioningTest extends Specification implements ManagerConta
deviceNUniqueId = "device3"
deviceNRequestTopic = "$PROVISIONING_TOKEN/$deviceNUniqueId/$REQUEST_TOKEN".toString()
deviceNResponseTopic = "$PROVISIONING_TOKEN/$deviceNUniqueId/$RESPONSE_TOKEN".toString()
+ deviceNClient = new MQTT_IOClient(mqttDeviceNClientId, mqttHost, mqttPort, false, false, null, null, null)
+ deviceNClient.setTopicSubscribeFailureConsumer(subscribeFailureCallback)
deviceNClient.connect()
then: "mqtt connection should exist"
@@ -573,6 +577,8 @@ class UserAndAssetProvisioningTest extends Specification implements ManagerConta
when: "another device re-connects"
deviceNResponses.clear()
+ deviceNClient = new MQTT_IOClient(mqttDeviceNClientId, mqttHost, mqttPort, false, false, null, null, null)
+ deviceNClient.setTopicSubscribeFailureConsumer(subscribeFailureCallback)
deviceNClient.connect()
then: "mqtt connection should exist"
@@ -671,6 +677,8 @@ class UserAndAssetProvisioningTest extends Specification implements ManagerConta
when: "a device with disabled user account connects"
deviceNResponses.clear()
+ deviceNClient = new MQTT_IOClient(mqttDeviceNClientId, mqttHost, mqttPort, false, false, null, null, null)
+ deviceNClient.setTopicSubscribeFailureConsumer(subscribeFailureCallback)
deviceNClient.connect()
then: "mqtt connection should exist"
diff --git a/ui/component/core/src/index.ts b/ui/component/core/src/index.ts
index 49cd4fc5d..0138d9952 100644
--- a/ui/component/core/src/index.ts
+++ b/ui/component/core/src/index.ts
@@ -132,15 +132,8 @@ export function normaliseConfig(config: ManagerConfig): ManagerConfig {
normalisedConfig.realm = "master";
}
- if (normalisedConfig.auth === Auth.KEYCLOAK) {
- // Determine URL of keycloak server
- if (!normalisedConfig.keycloakUrl || normalisedConfig.keycloakUrl === "") {
- // Assume keycloak is running on same host as the manager
- normalisedConfig.keycloakUrl = normalisedConfig.managerUrl + "/auth";
- }
-
- // Normalise by stripping any trailing slashes
- normalisedConfig.keycloakUrl = normalisedConfig.keycloakUrl.replace(/\/+$/, "");
+ if (!normalisedConfig.auth) {
+ normalisedConfig.auth = Auth.KEYCLOAK;
}
if (normalisedConfig.consoleAutoEnable === undefined) {
@@ -356,12 +349,43 @@ export class Manager implements EventProviderFactory {
// BASIC auth will likely require UI so lets init translation at least
success = await this.doTranslateInit() && success;
success = await this.doAuthInit();
- } else {
+ } else if (this._config.auth === Auth.KEYCLOAK) {
+
+ // The info endpoint of the manager might return a relative URL (relative to the manager)
+ if (!this._config.keycloakUrl && this._authServerUrl) {
+ const managerURL = new URL(this._config.managerUrl!);
+ let authServerURL: URL;
+
+ try {
+ authServerURL = new URL(this._authServerUrl);
+ } catch (e) {
+ // Could be a relative URL
+ authServerURL = new URL(managerURL);
+ authServerURL.pathname = this._authServerUrl;
+ }
+
+ // Use manager URL info
+ if (!authServerURL.protocol) {
+ authServerURL.protocol = managerURL.protocol;
+ }
+ if (!authServerURL.hostname) {
+ authServerURL.hostname = managerURL.hostname;
+ }
+ if (!authServerURL.port) {
+ authServerURL.port = managerURL.port;
+ }
+ this._config.keycloakUrl = authServerURL.toString();
+ }
+
+ // If we still don't know auth server URL then use manager URL
if (!this._config.keycloakUrl) {
- this.config.keycloakUrl = this._authServerUrl;
+ this._config.keycloakUrl = this._config.managerUrl + "/auth";
}
+ // Normalise by stripping any trailing slashes
+ this._config.keycloakUrl = this._config.keycloakUrl.replace(/\/+$/, "");
+
success = await this.doAuthInit();
// If failed then we can assume keycloak auth requested but unavailable