Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Update version of container images used by Helm charts for k8s deployment (#3338)

Eric Bariaux01a2754c0dd7
diff --git a/kubernetes/keycloak/Chart.yaml b/kubernetes/keycloak/Chart.yaml
index de0ade992..8729e9c53 100644
--- a/kubernetes/keycloak/Chart.yaml
+++ b/kubernetes/keycloak/Chart.yaml
@@ -6,4 +6,4 @@ type: application
version: 0.1.0
-appVersion: "23.0.7.6"
+appVersion: "26.7.3.0"
diff --git a/kubernetes/keycloak/templates/deployment.yaml b/kubernetes/keycloak/templates/deployment.yaml
index f7662ef08..693e59138 100644
--- a/kubernetes/keycloak/templates/deployment.yaml
+++ b/kubernetes/keycloak/templates/deployment.yaml
@@ -6,6 +6,17 @@ metadata:
{{- include "keycloak.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
+ {{- with .Values.strategy }}
+ strategy:
+ {{- if eq .type "Recreate" }}
+ type: Recreate
+ # Recreate cannot have rollingUpdate settings. Existing deployments may need
+ # a merge patch before a server-side apply upgrade (see test/load2-eks/README.md).
+ rollingUpdate: null
+ {{- else }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- end }}
selector:
matchLabels:
{{- include "keycloak.selectorLabels" . | nindent 6 }}
@@ -66,7 +77,7 @@ spec:
- name: KC_DB_POOL_MAX_SIZE
value: "{{ .Values.or.postgresql.poolMaxSize }}"
- name: KC_HOSTNAME
- value: "{{ .Values.or.hostname }}"
+ value: "https://{{ .Values.or.hostname }}/auth"
- name: KC_BOOTSTRAP_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
diff --git a/kubernetes/keycloak/values.yaml b/kubernetes/keycloak/values.yaml
index ab56e8c64..f0f776ad7 100644
--- a/kubernetes/keycloak/values.yaml
+++ b/kubernetes/keycloak/values.yaml
@@ -5,6 +5,10 @@
# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
replicaCount: 1
+# Optional Deployment strategy. Use type: Recreate when the cluster has no
+# spare capacity for overlapping old and new Pods; upgrades interrupt service.
+strategy: {}
+
# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
image:
repository: openremote/keycloak
diff --git a/kubernetes/manager/Chart.yaml b/kubernetes/manager/Chart.yaml
index 89446bbc9..1457f34cc 100644
--- a/kubernetes/manager/Chart.yaml
+++ b/kubernetes/manager/Chart.yaml
@@ -6,4 +6,4 @@ type: application
version: 0.1.0
-appVersion: "1.20.2"
+appVersion: "1.31.1"
diff --git a/kubernetes/manager/templates/deployment.yaml b/kubernetes/manager/templates/deployment.yaml
index d346cb1c9..bdec8e9f7 100644
--- a/kubernetes/manager/templates/deployment.yaml
+++ b/kubernetes/manager/templates/deployment.yaml
@@ -12,6 +12,17 @@ metadata:
{{- include "manager.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
+ {{- with .Values.strategy }}
+ strategy:
+ {{- if eq .type "Recreate" }}
+ type: Recreate
+ # Recreate cannot have rollingUpdate settings. Existing deployments may need
+ # a merge patch before a server-side apply upgrade (see test/load2-eks/README.md).
+ rollingUpdate: null
+ {{- else }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- end }}
selector:
matchLabels:
{{- include "manager.selectorLabels" . | nindent 6 }}
diff --git a/kubernetes/manager/values.yaml b/kubernetes/manager/values.yaml
index 2588c9c65..7c925911d 100644
--- a/kubernetes/manager/values.yaml
+++ b/kubernetes/manager/values.yaml
@@ -5,6 +5,10 @@
# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
replicaCount: 1
+# Optional Deployment strategy. Use type: Recreate when the cluster has no
+# spare capacity for overlapping old and new Pods; upgrades interrupt service.
+strategy: {}
+
# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
image:
repository: openremote/manager
diff --git a/kubernetes/postgresql/Chart.yaml b/kubernetes/postgresql/Chart.yaml
index e5062ffdb..0ff06a079 100644
--- a/kubernetes/postgresql/Chart.yaml
+++ b/kubernetes/postgresql/Chart.yaml
@@ -6,4 +6,4 @@ type: application
version: 0.1.0
-appVersion: "17.6.0.2-slim"
+appVersion: "17.9.0.1-slim"
diff --git a/kubernetes/proxy/Chart.yaml b/kubernetes/proxy/Chart.yaml
index 984fb744a..d689c61e2 100644
--- a/kubernetes/proxy/Chart.yaml
+++ b/kubernetes/proxy/Chart.yaml
@@ -6,4 +6,4 @@ type: application
version: 0.1.0
-appVersion: "2.9.15.1"
+appVersion: "3.2.19.0"
diff --git a/test/load1-eks/README.md b/test/load1-eks/README.md
index 90a614493..ea790dc81 100644
--- a/test/load1-eks/README.md
+++ b/test/load1-eks/README.md
@@ -11,25 +11,30 @@ see `load1` folder for the tools, scripts and documentation.
## Creating load test manager image
-There seems to be a problem in the setup gradle build file, and you need to comment out the
-
-```groovy
-tasks.register('demoJar', Jar) {
- base {
- archivesName = "openremote-demo-${project.name}"
- }
- from sourceSets.demo.output
-}
-```
-
-section from `setup/build.gradle` to properly build the load test image.
-
-After that, run
+Build from the repository root using the checkout you want to test. The current
+setup build supports `SETUP_JAR=load1` directly; no Gradle file edits are needed.
+The load profile uses the custom `load1` tag, so upgrading the Manager chart's
+`appVersion` does not upgrade this image. Rebuild and push it for each Manager
+version you want to test:
```
./gradlew -PSETUP_JAR=load1 clean installDist
+# The Manager Dockerfile copies lib, but not deployment/manager/extensions.
+# Include the custom setup JAR on the image's application classpath.
+cp manager/build/install/manager/deployment/manager/extensions/openremote-load1-setup-*.jar \
+ manager/build/install/manager/lib/
+
export AWS_DEVELOPERS_ACCOUNT_ID="dev-account-id"
aws ecr get-login-password --region eu-west-1 | docker login --username AWS --password-stdin $AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com
docker buildx build --push --platform linux/amd64,linux/arm64 -t $AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/manager:load1 manager/build/install/manager/
```
+
+The profile uses `image.pullPolicy: Always` so new Manager Pods pull the current
+`load1` image. Pushing the tag does not restart existing Pods. Keycloak and
+PostgreSQL use the image versions in their Kubernetes charts.
+
+The JAR copy above is required: without it, the image can initialize Keycloak
+and allow login, but the custom load-test setup provider is absent and no test
+users or assets are created. During a clean initialization, Manager logs should
+identify `org.openremote.setup.load1.SetupTasks` as a custom setup provider.
diff --git a/test/load1-eks/values-manager-eks-load.yaml b/test/load1-eks/values-manager-eks-load.yaml
index 29e525106..1abfed0a5 100644
--- a/test/load1-eks/values-manager-eks-load.yaml
+++ b/test/load1-eks/values-manager-eks-load.yaml
@@ -1,4 +1,5 @@
image:
+ pullPolicy: Always
tag: "load1"
livenessProbe:
diff --git a/test/load2-eks/README.md b/test/load2-eks/README.md
index 211dd73e8..9c5583cce 100644
--- a/test/load2-eks/README.md
+++ b/test/load2-eks/README.md
@@ -7,7 +7,7 @@ For the manager, JVM parameters are used to make use of the extra memory availab
There are different "profiles" available depending on the power required and the tests to be run:
- large: minimal set-up useful to test memory leaks and pressure condition
-- xlarge-minimal: doubles memory allocation for manager compared to large profile (4Gi vs 2Gi), with room for further increases
+- xlarge-minimal: gives Manager 2Gi more memory than large (3840Mi vs 1792Mi), with room for further increases
- xlarge: bigger setup using all capacity on 4 CPU/16 GB cluster
- 2xlarge: bigger setup using all capacity on 8 CPU/32 GB cluster
- 4xlarge: bigger setup using all capacity on 16 CPU/64 GB cluster
@@ -15,6 +15,14 @@ There are different "profiles" available depending on the power required and the
The profile is selected by setting the OR_PROFILE environment variable in the `eks-common.sh` script.
+The `large` and `xlarge-minimal` profiles now use 768Mi for Keycloak.
+The old 512Mi allocation caused Keycloak 26 to be OOMKilled
+during initial setup.
+If Manager has already created its schema when Keycloak fails, a subsequent
+Manager restart can skip setup (`clean install = false`) even though the
+`openremote` client and test dataset were never initialized. Stabilize Keycloak
+before deliberately resetting and rebuilding that incomplete test dataset.
+
This folder contains a different setup than load1 and includes different test scenarios.
The setup is controlled by the following parameters:
@@ -39,16 +47,33 @@ The setup creates:
- OR_SETUP_USERS restricted service users, linked to the corresponding Building asset,
with appropriate permission to push attribute values
-You build and push the custom manager image in a similar way than for load1, by running
+Build and push the custom Manager image from the repository root using the
+checkout you want to test. All profiles use the custom `load2` tag, so upgrading
+the Manager chart's `appVersion` does not upgrade this image. Rebuild and push
+it for each Manager version you want to test:
```
./gradlew -PSETUP_JAR=load2 clean installDist
+# The Manager Dockerfile copies lib, but not deployment/manager/extensions.
+# Include the custom setup JAR on the image's application classpath.
+cp manager/build/install/manager/deployment/manager/extensions/openremote-load2-setup-*.jar \
+ manager/build/install/manager/lib/
+
export AWS_DEVELOPERS_ACCOUNT_ID="dev-account-id"
aws ecr get-login-password --region eu-west-1 | docker login --username AWS --password-stdin $AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com
docker buildx build --push --platform linux/amd64,linux/arm64 -t $AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/manager:load2 manager/build/install/manager/
```
+All profiles use `image.pullPolicy: Always` so new Manager Pods pull the current
+`load2` image. Pushing the tag does not restart existing Pods. Keycloak and
+PostgreSQL use the image versions in their Kubernetes charts.
+
+The JAR copy above is required: without it, the image can initialize Keycloak
+and allow login, but the custom load-test setup provider is absent and no test
+users or assets are created. During a clean initialization, Manager logs should
+identify `org.openremote.setup.load2.SetupTasks` as a custom setup provider.
+
You might want to update the stack without re-creating the whole cluster, as re-creating the cluster takes time and
requires re-generating a new certificate (on which we have some limits).
To do so, keep the proxy running and uninstall the other charts
@@ -58,6 +83,66 @@ helm uninstall manager postgresql keycloak
```
then use the `eks-deploy-load.sh` script to properly re-deploy those charts.
+That script sets `or.setupRunOnRestart=true` to reset and rebuild the test
+dataset; this procedure does not preserve your existing test data.
+
+## Switching an existing Deployment to Recreate
+
+The Keycloak and Manager charts support an optional deployment strategy. To
+avoid needing spare capacity for overlapping old and new Pods during an
+upgrade, set this in the values file for each chart that should use it:
+
+```yaml
+strategy:
+ type: Recreate
+```
+
+With `Recreate`, a rollout terminates the old Pods before starting their
+replacements, so upgrades interrupt service. The chart default, `strategy: {}`,
+leaves Kubernetes to use its default `RollingUpdate` strategy on a fresh install.
+
+When changing an existing Deployment from `RollingUpdate` to `Recreate`, its
+`spec.strategy.rollingUpdate` settings must also be removed. Kubernetes rejects
+a Deployment that has both `type: Recreate` and a `rollingUpdate` block. The
+templates render `rollingUpdate: null` to clear that block, but if an upgrade
+(including one using server-side apply) retains the existing settings, it can
+fail with:
+
+```text
+spec.strategy.rollingUpdate: Forbidden: may not be specified when strategy `type` is 'Recreate'
+```
+
+For this error, use a JSON merge patch to set the strategy and remove the
+incompatible block in the same request, then retry the upgrade. A merge patch
+updates the specified fields while preserving omitted fields; a `null` value
+removes a field. Unlike server-side apply, it explicitly patches the live object
+without relying on apply's field ownership to remove previously managed settings.
+
+Set `K_CONTEXT` to the target cluster context and `K_NAMESPACE` to the namespace
+containing the release. The examples use the Deployment names created by the
+load-test scripts; adjust them if you use different release names or name overrides.
+Run only the patch for each Deployment you intend to switch:
+
+```shell
+kubectl --context "$K_CONTEXT" --namespace "$K_NAMESPACE" patch deployment keycloak \
+ --type=merge \
+ -p '{"spec":{"strategy":{"type":"Recreate","rollingUpdate":null}}}'
+
+kubectl --context "$K_CONTEXT" --namespace "$K_NAMESPACE" patch deployment manager \
+ --type=merge \
+ -p '{"spec":{"strategy":{"type":"Recreate","rollingUpdate":null}}}'
+```
+
+Keep `strategy.type: Recreate` in the Helm values used for subsequent upgrades.
+This workaround is only needed if the strategy change fails to remove the old
+settings. Fresh installations using `Recreate` and upgrades that keep `Recreate`
+do not need it. A Deployment created after this chart option was introduced can
+still encounter the issue if it starts with `RollingUpdate` and switches later.
+
+See the Kubernetes documentation on [patching API objects](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/)
+and [server-side apply](https://kubernetes.io/docs/reference/using-api/server-side-apply/).
+
+## Running load tests
Once deployed, running the load tests can be done using the same clients as for load testing a VM,
but using the scenarios present under `scenarios` in this folder instead.
diff --git a/test/load2-eks/profiles/large/values-keycloak-eks-load.yaml b/test/load2-eks/profiles/large/values-keycloak-eks-load.yaml
index 41bfda1b4..eeceaf6e6 100644
--- a/test/load2-eks/profiles/large/values-keycloak-eks-load.yaml
+++ b/test/load2-eks/profiles/large/values-keycloak-eks-load.yaml
@@ -1,7 +1,7 @@
resources:
limits:
cpu: 280m
- memory: 512Mi
+ memory: 768Mi
requests:
cpu: 280m
- memory: 512Mi
+ memory: 768Mi
diff --git a/test/load2-eks/profiles/large/values-manager-eks-load.yaml b/test/load2-eks/profiles/large/values-manager-eks-load.yaml
index 26232bb31..a5644c35d 100644
--- a/test/load2-eks/profiles/large/values-manager-eks-load.yaml
+++ b/test/load2-eks/profiles/large/values-manager-eks-load.yaml
@@ -26,10 +26,10 @@ service:
resources:
limits:
cpu: 400m
- memory: 2Gi
+ memory: 1792Mi
requests:
cpu: 400m
- memory: 2Gi
+ memory: 1792Mi
volumeSize: "16Gi"
diff --git a/test/load2-eks/profiles/xlarge-minimal/values-keycloak-eks-load.yaml b/test/load2-eks/profiles/xlarge-minimal/values-keycloak-eks-load.yaml
index 41bfda1b4..eeceaf6e6 100644
--- a/test/load2-eks/profiles/xlarge-minimal/values-keycloak-eks-load.yaml
+++ b/test/load2-eks/profiles/xlarge-minimal/values-keycloak-eks-load.yaml
@@ -1,7 +1,7 @@
resources:
limits:
cpu: 280m
- memory: 512Mi
+ memory: 768Mi
requests:
cpu: 280m
- memory: 512Mi
+ memory: 768Mi
diff --git a/test/load2-eks/profiles/xlarge-minimal/values-manager-eks-load.yaml b/test/load2-eks/profiles/xlarge-minimal/values-manager-eks-load.yaml
index f09aecd72..bf5a9b77f 100644
--- a/test/load2-eks/profiles/xlarge-minimal/values-manager-eks-load.yaml
+++ b/test/load2-eks/profiles/xlarge-minimal/values-manager-eks-load.yaml
@@ -26,10 +26,10 @@ service:
resources:
limits:
cpu: 400m
- memory: 4Gi
+ memory: 3840Mi
requests:
cpu: 400m
- memory: 4Gi
+ memory: 3840Mi
volumeSize: "16Gi"