100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Add concept of profiles when setting-up load2 cluster (#2348)
Eric Bariaux18fc04b6518c
diff --git a/test/load2-eks/README.md b/test/load2-eks/README.mdindex 831a9fb97..fd9a05162 100644--- a/test/load2-eks/README.md+++ b/test/load2-eks/README.md@@ -1,9 +1,15 @@# Load Tests-Scripts and configuration files to setup an OpenRemote stack running in an EKS cluster appropriate for load testing.+Scripts and configuration files to setup an OpenRemote stack running in an EKS cluster appropriate for load testing.This includes configuring the cluster to use a more powerful machineand adapting the values files to configure the memory usage on the different pods.For the manager, JVM parameters are used to make use of the extra memory available to the container.+There are different "profiles" available depending on the power required and the tests to be run:+- large: minimal set-up useful to test memory leaks and pressure condition+- xlarge: doubles memory allocation for manager compared to large profile (4Gi vs 2Gi), with room for further increases+- 2xlarge: much bigger setup to test system limits and get a feel for performance level we can reach++The profile is selected by setting the OR_PROFILE environment variable in the `eks-common.sh` script.This folder contains a different setup than load1 and includes different test scenarios.diff --git a/test/load2-eks/eks-common.sh b/test/load2-eks/eks-common.shindex a0367444a..eb316b4d1 100755--- a/test/load2-eks/eks-common.sh+++ b/test/load2-eks/eks-common.sh@@ -14,6 +14,7 @@ if [ -z "$AWS_DEVELOPERS_ACCOUNT_ID" ]; thenfiOR_KUBERNETES_PATH=../../kubernetes+export OR_PROFILE=large# Name of cluster, not exposed but must be unique within accountexport CLUSTER_NAME=loadtest-clusterdiff --git a/test/load2-eks/eks-setup-load.sh b/test/load2-eks/eks-setup-load.shindex 2b2503023..e53cef411 100755--- a/test/load2-eks/eks-setup-load.sh+++ b/test/load2-eks/eks-setup-load.sh@@ -2,7 +2,7 @@. ./eks-common.sh-envsubst < cluster.yaml | eksctl create cluster -f - --profile or+envsubst < profiles/$OR_PROFILE/cluster.yaml | eksctl create cluster -f - --profile or# [Installation Guide - AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/installation/)@@ -27,9 +27,9 @@ CLUSTER_DNS=$(kubectl get svc kube-dns -n kube-system -o jsonpath='{.spec.clustehelm install or-setup $OR_KUBERNETES_PATH/or-setup -f values-or-setup-eks-load.yaml --set aws.enabled=true --set aws.managerVolumeId=$MANAGER_VOLUMEID --set aws.psqlVolumeId=$PSQL_VOLUMEID-helm install proxy $OR_KUBERNETES_PATH/proxy -f $OR_KUBERNETES_PATH/proxy/values-eks.yaml -f values-proxy-eks-load.yaml --set or.nameserver=$CLUSTER_DNS --set or.hostname=$FQDN+helm install proxy $OR_KUBERNETES_PATH/proxy -f $OR_KUBERNETES_PATH/proxy/values-eks.yaml -f profiles/$OR_PROFILE/values-proxy-eks-load.yaml --set or.nameserver=$CLUSTER_DNS --set or.hostname=$FQDN-helm install postgresql $OR_KUBERNETES_PATH/postgresql -f $OR_KUBERNETES_PATH/postgresql/values-eks.yaml -f values-postgresql-eks-load.yaml+helm install postgresql $OR_KUBERNETES_PATH/postgresql -f $OR_KUBERNETES_PATH/postgresql/values-eks.yaml -f profiles/$OR_PROFILE/values-postgresql-eks-load.yaml# Waiting for the LB to be created# AWS LB Controller only creates an Network LB if there's a service@@ -55,9 +55,9 @@ aws route53 change-resource-record-sets \--profile dnschghelm install keycloak $OR_KUBERNETES_PATH/keycloak -f $OR_KUBERNETES_PATH/keycloak/values-haproxy.yaml \- -f values-keycloak-eks-load.yaml --set-string or.hostname=$FQDN+ -f profiles/$OR_PROFILE/values-keycloak-eks-load.yaml --set-string or.hostname=$FQDNhelm install manager $OR_KUBERNETES_PATH/manager -f $OR_KUBERNETES_PATH/manager/values-haproxy-eks.yaml \- -f values-manager-eks-load.yaml --set-string or.hostname=$FQDN \+ -f profiles/$OR_PROFILE/values-manager-eks-load.yaml --set-string or.hostname=$FQDN \--set-string image.repository=$AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/managerwhile ! dig +short $FQDN | grep -qE '^[0-9]'; dodiff --git a/test/load2-eks/profiles/2xlarge/README.md b/test/load2-eks/profiles/2xlarge/README.mdnew file mode 100644index 000000000..a80d08fcf--- /dev/null+++ b/test/load2-eks/profiles/2xlarge/README.md@@ -0,0 +1,2 @@+Cluster using 1 m8g.2xlarge VM+Running the stack with more CPU and memory (up to 13GB for manager)diff --git a/test/load2-eks/cluster.yaml b/test/load2-eks/profiles/2xlarge/cluster.yamlsimilarity index 100%rename from test/load2-eks/cluster.yamlrename to test/load2-eks/profiles/2xlarge/cluster.yamldiff --git a/test/load2-eks/values-keycloak-eks-load.yaml b/test/load2-eks/profiles/2xlarge/values-keycloak-eks-load.yamlsimilarity index 100%rename from test/load2-eks/values-keycloak-eks-load.yamlrename to test/load2-eks/profiles/2xlarge/values-keycloak-eks-load.yamldiff --git a/test/load2-eks/values-manager-eks-load.yaml b/test/load2-eks/profiles/2xlarge/values-manager-eks-load.yamlsimilarity index 61%rename from test/load2-eks/values-manager-eks-load.yamlrename to test/load2-eks/profiles/2xlarge/values-manager-eks-load.yamlindex 47b87c17a..87f795e84 100644--- a/test/load2-eks/values-manager-eks-load.yaml+++ b/test/load2-eks/profiles/2xlarge/values-manager-eks-load.yaml@@ -1,4 +1,5 @@image:+ pullPolicy: Alwaystag: "load2"livenessProbe:@@ -31,10 +32,12 @@ volumeSize: "16Gi"or:env:- name: OR_SETUP_USERS- value: "5000"+ value: "1000"- name: OR_SETUP_ASSETSvalue: "5"- name: OR_SETUP_STORE_DATA_POINTSvalue: "false"- name: OR_JAVA_OPTS- value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof"+ value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof -Xlog:gc*,gc+ref=debug,gc+phases=debug,gc+age=debug,safepoint:file=gc.log"+ - name: JAVA_TOOL_OPTIONS+ value: "-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=8085 -Dcom.sun.management.jmxremote.rmi.port=8085 -Djava.rmi.server.hostname=localhost"diff --git a/test/load2-eks/values-postgresql-eks-load.yaml b/test/load2-eks/profiles/2xlarge/values-postgresql-eks-load.yamlsimilarity index 100%rename from test/load2-eks/values-postgresql-eks-load.yamlrename to test/load2-eks/profiles/2xlarge/values-postgresql-eks-load.yamldiff --git a/test/load2-eks/values-proxy-eks-load.yaml b/test/load2-eks/profiles/2xlarge/values-proxy-eks-load.yamlsimilarity index 100%rename from test/load2-eks/values-proxy-eks-load.yamlrename to test/load2-eks/profiles/2xlarge/values-proxy-eks-load.yamldiff --git a/test/load2-eks/profiles/large/README.md b/test/load2-eks/profiles/large/README.mdnew file mode 100644index 000000000..3152396b4--- /dev/null+++ b/test/load2-eks/profiles/large/README.md@@ -0,0 +1,3 @@+Cluster using 1 m8g.large VM+Running the stack with limited CPU and memory, mainly used to investigate bottlenecks and memory leaks+as it's easier to hit the limits running some load tests against it.diff --git a/test/load2-eks/profiles/large/cluster.yaml b/test/load2-eks/profiles/large/cluster.yamlnew file mode 100644index 000000000..9912f09f6--- /dev/null+++ b/test/load2-eks/profiles/large/cluster.yaml@@ -0,0 +1,35 @@+apiVersion: eksctl.io/v1alpha5+kind: ClusterConfig++metadata:+ name: ${CLUSTER_NAME}+ region: ${AWS_REGION}+ # Kubernetes version to use for cluster, good idea to keep up to date+ version: "1.34"++availabilityZones:+ - ${AWS_REGION}a+ - ${AWS_REGION}b++iam:+ withOIDC: true+ serviceAccounts:+ - metadata:+ name: aws-load-balancer-controller+ namespace: kube-system+ wellKnownPolicies:+ awsLoadBalancerController: true++nodeGroups:+ - name: ng-1+ instanceType: m8g.large+ volumeSize: 32+ desiredCapacity: 1+ availabilityZones:+ - ${AWS_REGION}a++addons:+ - name: aws-ebs-csi-driver+ version: latest+ wellKnownPolicies:+ ebsCSIController: truediff --git a/test/load2-eks/profiles/large/values-keycloak-eks-load.yaml b/test/load2-eks/profiles/large/values-keycloak-eks-load.yamlnew file mode 100644index 000000000..41bfda1b4--- /dev/null+++ b/test/load2-eks/profiles/large/values-keycloak-eks-load.yaml@@ -0,0 +1,7 @@+resources:+ limits:+ cpu: 280m+ memory: 512Mi+ requests:+ cpu: 280m+ memory: 512Midiff --git a/test/load2-eks/profiles/large/values-manager-eks-load.yaml b/test/load2-eks/profiles/large/values-manager-eks-load.yamlnew file mode 100644index 000000000..f41feb010--- /dev/null+++ b/test/load2-eks/profiles/large/values-manager-eks-load.yaml@@ -0,0 +1,43 @@+image:+ pullPolicy: Always+ tag: "load2"++livenessProbe:+ httpGet:+ path: /+ port: 8080+ periodSeconds: 5+readinessProbe:+ httpGet:+ path: /+ port: 8080+ periodSeconds: 5+startupProbe:+ httpGet:+ path: /+ port: 8080+ failureThreshold: 500+ periodSeconds: 10++resources:+ limits:+ cpu: 400m+ memory: 2Gi+ requests:+ cpu: 400m+ memory: 2Gi++volumeSize: "16Gi"++or:+ env:+ - name: OR_SETUP_USERS+ value: "1000"+ - name: OR_SETUP_ASSETS+ value: "5"+ - name: OR_SETUP_STORE_DATA_POINTS+ value: "false"+ - name: OR_JAVA_OPTS+ value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof -Xlog:gc*,gc+ref=debug,gc+phases=debug,gc+age=debug,safepoint:file=gc.log"+ - name: JAVA_TOOL_OPTIONS+ value: "-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=8085 -Dcom.sun.management.jmxremote.rmi.port=8085 -Djava.rmi.server.hostname=localhost"diff --git a/test/load2-eks/profiles/large/values-postgresql-eks-load.yaml b/test/load2-eks/profiles/large/values-postgresql-eks-load.yamlnew file mode 100644index 000000000..ce6c97c90--- /dev/null+++ b/test/load2-eks/profiles/large/values-postgresql-eks-load.yaml@@ -0,0 +1,9 @@+volumeSize: "16Gi"++resources:+ limits:+ cpu: 350m+ memory: 1Gi+ requests:+ cpu: 350m+ memory: 1Gidiff --git a/test/load2-eks/profiles/large/values-proxy-eks-load.yaml b/test/load2-eks/profiles/large/values-proxy-eks-load.yamlnew file mode 100644index 000000000..83704321d--- /dev/null+++ b/test/load2-eks/profiles/large/values-proxy-eks-load.yaml@@ -0,0 +1,7 @@+resources:+ limits:+ cpu: 200m+ memory: 512Mi+ requests:+ cpu: 200m+ memory: 512Midiff --git a/test/load2-eks/profiles/xlarge/README.md b/test/load2-eks/profiles/xlarge/README.mdnew file mode 100644index 000000000..c7f052274--- /dev/null+++ b/test/load2-eks/profiles/xlarge/README.md@@ -0,0 +1,3 @@+Cluster using 1 m8g.xlarge VM+Still running the stack with limited CPU and memory.+Mostly slightly increasing memory over large version to check what the impact is on MQTT queue backlog.diff --git a/test/load2-eks/profiles/xlarge/cluster.yaml b/test/load2-eks/profiles/xlarge/cluster.yamlnew file mode 100644index 000000000..f5d60c9f4--- /dev/null+++ b/test/load2-eks/profiles/xlarge/cluster.yaml@@ -0,0 +1,35 @@+apiVersion: eksctl.io/v1alpha5+kind: ClusterConfig++metadata:+ name: ${CLUSTER_NAME}+ region: ${AWS_REGION}+ # Kubernetes version to use for cluster, good idea to keep up to date+ version: "1.34"++availabilityZones:+ - ${AWS_REGION}a+ - ${AWS_REGION}b++iam:+ withOIDC: true+ serviceAccounts:+ - metadata:+ name: aws-load-balancer-controller+ namespace: kube-system+ wellKnownPolicies:+ awsLoadBalancerController: true++nodeGroups:+ - name: ng-1+ instanceType: m8g.xlarge+ volumeSize: 32+ desiredCapacity: 1+ availabilityZones:+ - ${AWS_REGION}a++addons:+ - name: aws-ebs-csi-driver+ version: latest+ wellKnownPolicies:+ ebsCSIController: truediff --git a/test/load2-eks/profiles/xlarge/values-keycloak-eks-load.yaml b/test/load2-eks/profiles/xlarge/values-keycloak-eks-load.yamlnew file mode 100644index 000000000..41bfda1b4--- /dev/null+++ b/test/load2-eks/profiles/xlarge/values-keycloak-eks-load.yaml@@ -0,0 +1,7 @@+resources:+ limits:+ cpu: 280m+ memory: 512Mi+ requests:+ cpu: 280m+ memory: 512Midiff --git a/test/load2-eks/profiles/xlarge/values-manager-eks-load.yaml b/test/load2-eks/profiles/xlarge/values-manager-eks-load.yamlnew file mode 100644index 000000000..ffbc49423--- /dev/null+++ b/test/load2-eks/profiles/xlarge/values-manager-eks-load.yaml@@ -0,0 +1,43 @@+image:+ pullPolicy: Always+ tag: "load2"++livenessProbe:+ httpGet:+ path: /+ port: 8080+ periodSeconds: 5+readinessProbe:+ httpGet:+ path: /+ port: 8080+ periodSeconds: 5+startupProbe:+ httpGet:+ path: /+ port: 8080+ failureThreshold: 500+ periodSeconds: 10++resources:+ limits:+ cpu: 400m+ memory: 4Gi+ requests:+ cpu: 400m+ memory: 4Gi++volumeSize: "16Gi"++or:+ env:+ - name: OR_SETUP_USERS+ value: "1000"+ - name: OR_SETUP_ASSETS+ value: "5"+ - name: OR_SETUP_STORE_DATA_POINTS+ value: "false"+ - name: OR_JAVA_OPTS+ value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof -Xlog:gc*,gc+ref=debug,gc+phases=debug,gc+age=debug,safepoint:file=gc.log"+ - name: JAVA_TOOL_OPTIONS+ value: "-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=8085 -Dcom.sun.management.jmxremote.rmi.port=8085 -Djava.rmi.server.hostname=localhost"diff --git a/test/load2-eks/profiles/xlarge/values-postgresql-eks-load.yaml b/test/load2-eks/profiles/xlarge/values-postgresql-eks-load.yamlnew file mode 100644index 000000000..ce6c97c90--- /dev/null+++ b/test/load2-eks/profiles/xlarge/values-postgresql-eks-load.yaml@@ -0,0 +1,9 @@+volumeSize: "16Gi"++resources:+ limits:+ cpu: 350m+ memory: 1Gi+ requests:+ cpu: 350m+ memory: 1Gidiff --git a/test/load2-eks/profiles/xlarge/values-proxy-eks-load.yaml b/test/load2-eks/profiles/xlarge/values-proxy-eks-load.yamlnew file mode 100644index 000000000..83704321d--- /dev/null+++ b/test/load2-eks/profiles/xlarge/values-proxy-eks-load.yaml@@ -0,0 +1,7 @@+resources:+ limits:+ cpu: 200m+ memory: 512Mi+ requests:+ cpu: 200m+ memory: 512Mi