Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Add concept of profiles when setting-up load2 cluster (#2348)

Eric Bariaux18fc04b6518c
diff --git a/test/load2-eks/README.md b/test/load2-eks/README.md
index 831a9fb97..fd9a05162 100644
--- a/test/load2-eks/README.md
+++ b/test/load2-eks/README.md
@@ -1,9 +1,15 @@
# Load Tests
-Scripts and configuration files to setup an OpenRemote stack running in an EKS cluster appropriate for load testing.
+Scripts and configuration files to setup an OpenRemote stack running in an EKS cluster appropriate for load testing.
This includes configuring the cluster to use a more powerful machine
and adapting the values files to configure the memory usage on the different pods.
For the manager, JVM parameters are used to make use of the extra memory available to the container.
+There are different "profiles" available depending on the power required and the tests to be run:
+- large: minimal set-up useful to test memory leaks and pressure condition
+- xlarge: doubles memory allocation for manager compared to large profile (4Gi vs 2Gi), with room for further increases
+- 2xlarge: much bigger setup to test system limits and get a feel for performance level we can reach
+
+The profile is selected by setting the OR_PROFILE environment variable in the `eks-common.sh` script.
This folder contains a different setup than load1 and includes different test scenarios.
diff --git a/test/load2-eks/eks-common.sh b/test/load2-eks/eks-common.sh
index a0367444a..eb316b4d1 100755
--- a/test/load2-eks/eks-common.sh
+++ b/test/load2-eks/eks-common.sh
@@ -14,6 +14,7 @@ if [ -z "$AWS_DEVELOPERS_ACCOUNT_ID" ]; then
fi
OR_KUBERNETES_PATH=../../kubernetes
+export OR_PROFILE=large
# Name of cluster, not exposed but must be unique within account
export CLUSTER_NAME=loadtest-cluster
diff --git a/test/load2-eks/eks-setup-load.sh b/test/load2-eks/eks-setup-load.sh
index 2b2503023..e53cef411 100755
--- a/test/load2-eks/eks-setup-load.sh
+++ b/test/load2-eks/eks-setup-load.sh
@@ -2,7 +2,7 @@
. ./eks-common.sh
-envsubst < cluster.yaml | eksctl create cluster -f - --profile or
+envsubst < profiles/$OR_PROFILE/cluster.yaml | eksctl create cluster -f - --profile or
# [Installation Guide - AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/installation/)
@@ -27,9 +27,9 @@ CLUSTER_DNS=$(kubectl get svc kube-dns -n kube-system -o jsonpath='{.spec.cluste
helm install or-setup $OR_KUBERNETES_PATH/or-setup -f values-or-setup-eks-load.yaml --set aws.enabled=true --set aws.managerVolumeId=$MANAGER_VOLUMEID --set aws.psqlVolumeId=$PSQL_VOLUMEID
-helm install proxy $OR_KUBERNETES_PATH/proxy -f $OR_KUBERNETES_PATH/proxy/values-eks.yaml -f values-proxy-eks-load.yaml --set or.nameserver=$CLUSTER_DNS --set or.hostname=$FQDN
+helm install proxy $OR_KUBERNETES_PATH/proxy -f $OR_KUBERNETES_PATH/proxy/values-eks.yaml -f profiles/$OR_PROFILE/values-proxy-eks-load.yaml --set or.nameserver=$CLUSTER_DNS --set or.hostname=$FQDN
-helm install postgresql $OR_KUBERNETES_PATH/postgresql -f $OR_KUBERNETES_PATH/postgresql/values-eks.yaml -f values-postgresql-eks-load.yaml
+helm install postgresql $OR_KUBERNETES_PATH/postgresql -f $OR_KUBERNETES_PATH/postgresql/values-eks.yaml -f profiles/$OR_PROFILE/values-postgresql-eks-load.yaml
# Waiting for the LB to be created
# AWS LB Controller only creates an Network LB if there's a service
@@ -55,9 +55,9 @@ aws route53 change-resource-record-sets \
--profile dnschg
helm install keycloak $OR_KUBERNETES_PATH/keycloak -f $OR_KUBERNETES_PATH/keycloak/values-haproxy.yaml \
- -f values-keycloak-eks-load.yaml --set-string or.hostname=$FQDN
+ -f profiles/$OR_PROFILE/values-keycloak-eks-load.yaml --set-string or.hostname=$FQDN
helm install manager $OR_KUBERNETES_PATH/manager -f $OR_KUBERNETES_PATH/manager/values-haproxy-eks.yaml \
- -f values-manager-eks-load.yaml --set-string or.hostname=$FQDN \
+ -f profiles/$OR_PROFILE/values-manager-eks-load.yaml --set-string or.hostname=$FQDN \
--set-string image.repository=$AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/manager
while ! dig +short $FQDN | grep -qE '^[0-9]'; do
diff --git a/test/load2-eks/profiles/2xlarge/README.md b/test/load2-eks/profiles/2xlarge/README.md
new file mode 100644
index 000000000..a80d08fcf
--- /dev/null
+++ b/test/load2-eks/profiles/2xlarge/README.md
@@ -0,0 +1,2 @@
+Cluster using 1 m8g.2xlarge VM
+Running the stack with more CPU and memory (up to 13GB for manager)
diff --git a/test/load2-eks/cluster.yaml b/test/load2-eks/profiles/2xlarge/cluster.yaml
similarity index 100%
rename from test/load2-eks/cluster.yaml
rename to test/load2-eks/profiles/2xlarge/cluster.yaml
diff --git a/test/load2-eks/values-keycloak-eks-load.yaml b/test/load2-eks/profiles/2xlarge/values-keycloak-eks-load.yaml
similarity index 100%
rename from test/load2-eks/values-keycloak-eks-load.yaml
rename to test/load2-eks/profiles/2xlarge/values-keycloak-eks-load.yaml
diff --git a/test/load2-eks/values-manager-eks-load.yaml b/test/load2-eks/profiles/2xlarge/values-manager-eks-load.yaml
similarity index 61%
rename from test/load2-eks/values-manager-eks-load.yaml
rename to test/load2-eks/profiles/2xlarge/values-manager-eks-load.yaml
index 47b87c17a..87f795e84 100644
--- a/test/load2-eks/values-manager-eks-load.yaml
+++ b/test/load2-eks/profiles/2xlarge/values-manager-eks-load.yaml
@@ -1,4 +1,5 @@
image:
+ pullPolicy: Always
tag: "load2"
livenessProbe:
@@ -31,10 +32,12 @@ volumeSize: "16Gi"
or:
env:
- name: OR_SETUP_USERS
- value: "5000"
+ value: "1000"
- name: OR_SETUP_ASSETS
value: "5"
- name: OR_SETUP_STORE_DATA_POINTS
value: "false"
- name: OR_JAVA_OPTS
- value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof"
+ value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof -Xlog:gc*,gc+ref=debug,gc+phases=debug,gc+age=debug,safepoint:file=gc.log"
+ - name: JAVA_TOOL_OPTIONS
+ value: "-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=8085 -Dcom.sun.management.jmxremote.rmi.port=8085 -Djava.rmi.server.hostname=localhost"
diff --git a/test/load2-eks/values-postgresql-eks-load.yaml b/test/load2-eks/profiles/2xlarge/values-postgresql-eks-load.yaml
similarity index 100%
rename from test/load2-eks/values-postgresql-eks-load.yaml
rename to test/load2-eks/profiles/2xlarge/values-postgresql-eks-load.yaml
diff --git a/test/load2-eks/values-proxy-eks-load.yaml b/test/load2-eks/profiles/2xlarge/values-proxy-eks-load.yaml
similarity index 100%
rename from test/load2-eks/values-proxy-eks-load.yaml
rename to test/load2-eks/profiles/2xlarge/values-proxy-eks-load.yaml
diff --git a/test/load2-eks/profiles/large/README.md b/test/load2-eks/profiles/large/README.md
new file mode 100644
index 000000000..3152396b4
--- /dev/null
+++ b/test/load2-eks/profiles/large/README.md
@@ -0,0 +1,3 @@
+Cluster using 1 m8g.large VM
+Running the stack with limited CPU and memory, mainly used to investigate bottlenecks and memory leaks
+as it's easier to hit the limits running some load tests against it.
diff --git a/test/load2-eks/profiles/large/cluster.yaml b/test/load2-eks/profiles/large/cluster.yaml
new file mode 100644
index 000000000..9912f09f6
--- /dev/null
+++ b/test/load2-eks/profiles/large/cluster.yaml
@@ -0,0 +1,35 @@
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+metadata:
+ name: ${CLUSTER_NAME}
+ region: ${AWS_REGION}
+ # Kubernetes version to use for cluster, good idea to keep up to date
+ version: "1.34"
+
+availabilityZones:
+ - ${AWS_REGION}a
+ - ${AWS_REGION}b
+
+iam:
+ withOIDC: true
+ serviceAccounts:
+ - metadata:
+ name: aws-load-balancer-controller
+ namespace: kube-system
+ wellKnownPolicies:
+ awsLoadBalancerController: true
+
+nodeGroups:
+ - name: ng-1
+ instanceType: m8g.large
+ volumeSize: 32
+ desiredCapacity: 1
+ availabilityZones:
+ - ${AWS_REGION}a
+
+addons:
+ - name: aws-ebs-csi-driver
+ version: latest
+ wellKnownPolicies:
+ ebsCSIController: true
diff --git a/test/load2-eks/profiles/large/values-keycloak-eks-load.yaml b/test/load2-eks/profiles/large/values-keycloak-eks-load.yaml
new file mode 100644
index 000000000..41bfda1b4
--- /dev/null
+++ b/test/load2-eks/profiles/large/values-keycloak-eks-load.yaml
@@ -0,0 +1,7 @@
+resources:
+ limits:
+ cpu: 280m
+ memory: 512Mi
+ requests:
+ cpu: 280m
+ memory: 512Mi
diff --git a/test/load2-eks/profiles/large/values-manager-eks-load.yaml b/test/load2-eks/profiles/large/values-manager-eks-load.yaml
new file mode 100644
index 000000000..f41feb010
--- /dev/null
+++ b/test/load2-eks/profiles/large/values-manager-eks-load.yaml
@@ -0,0 +1,43 @@
+image:
+ pullPolicy: Always
+ tag: "load2"
+
+livenessProbe:
+ httpGet:
+ path: /
+ port: 8080
+ periodSeconds: 5
+readinessProbe:
+ httpGet:
+ path: /
+ port: 8080
+ periodSeconds: 5
+startupProbe:
+ httpGet:
+ path: /
+ port: 8080
+ failureThreshold: 500
+ periodSeconds: 10
+
+resources:
+ limits:
+ cpu: 400m
+ memory: 2Gi
+ requests:
+ cpu: 400m
+ memory: 2Gi
+
+volumeSize: "16Gi"
+
+or:
+ env:
+ - name: OR_SETUP_USERS
+ value: "1000"
+ - name: OR_SETUP_ASSETS
+ value: "5"
+ - name: OR_SETUP_STORE_DATA_POINTS
+ value: "false"
+ - name: OR_JAVA_OPTS
+ value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof -Xlog:gc*,gc+ref=debug,gc+phases=debug,gc+age=debug,safepoint:file=gc.log"
+ - name: JAVA_TOOL_OPTIONS
+ value: "-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=8085 -Dcom.sun.management.jmxremote.rmi.port=8085 -Djava.rmi.server.hostname=localhost"
diff --git a/test/load2-eks/profiles/large/values-postgresql-eks-load.yaml b/test/load2-eks/profiles/large/values-postgresql-eks-load.yaml
new file mode 100644
index 000000000..ce6c97c90
--- /dev/null
+++ b/test/load2-eks/profiles/large/values-postgresql-eks-load.yaml
@@ -0,0 +1,9 @@
+volumeSize: "16Gi"
+
+resources:
+ limits:
+ cpu: 350m
+ memory: 1Gi
+ requests:
+ cpu: 350m
+ memory: 1Gi
diff --git a/test/load2-eks/profiles/large/values-proxy-eks-load.yaml b/test/load2-eks/profiles/large/values-proxy-eks-load.yaml
new file mode 100644
index 000000000..83704321d
--- /dev/null
+++ b/test/load2-eks/profiles/large/values-proxy-eks-load.yaml
@@ -0,0 +1,7 @@
+resources:
+ limits:
+ cpu: 200m
+ memory: 512Mi
+ requests:
+ cpu: 200m
+ memory: 512Mi
diff --git a/test/load2-eks/profiles/xlarge/README.md b/test/load2-eks/profiles/xlarge/README.md
new file mode 100644
index 000000000..c7f052274
--- /dev/null
+++ b/test/load2-eks/profiles/xlarge/README.md
@@ -0,0 +1,3 @@
+Cluster using 1 m8g.xlarge VM
+Still running the stack with limited CPU and memory.
+Mostly slightly increasing memory over large version to check what the impact is on MQTT queue backlog.
diff --git a/test/load2-eks/profiles/xlarge/cluster.yaml b/test/load2-eks/profiles/xlarge/cluster.yaml
new file mode 100644
index 000000000..f5d60c9f4
--- /dev/null
+++ b/test/load2-eks/profiles/xlarge/cluster.yaml
@@ -0,0 +1,35 @@
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+metadata:
+ name: ${CLUSTER_NAME}
+ region: ${AWS_REGION}
+ # Kubernetes version to use for cluster, good idea to keep up to date
+ version: "1.34"
+
+availabilityZones:
+ - ${AWS_REGION}a
+ - ${AWS_REGION}b
+
+iam:
+ withOIDC: true
+ serviceAccounts:
+ - metadata:
+ name: aws-load-balancer-controller
+ namespace: kube-system
+ wellKnownPolicies:
+ awsLoadBalancerController: true
+
+nodeGroups:
+ - name: ng-1
+ instanceType: m8g.xlarge
+ volumeSize: 32
+ desiredCapacity: 1
+ availabilityZones:
+ - ${AWS_REGION}a
+
+addons:
+ - name: aws-ebs-csi-driver
+ version: latest
+ wellKnownPolicies:
+ ebsCSIController: true
diff --git a/test/load2-eks/profiles/xlarge/values-keycloak-eks-load.yaml b/test/load2-eks/profiles/xlarge/values-keycloak-eks-load.yaml
new file mode 100644
index 000000000..41bfda1b4
--- /dev/null
+++ b/test/load2-eks/profiles/xlarge/values-keycloak-eks-load.yaml
@@ -0,0 +1,7 @@
+resources:
+ limits:
+ cpu: 280m
+ memory: 512Mi
+ requests:
+ cpu: 280m
+ memory: 512Mi
diff --git a/test/load2-eks/profiles/xlarge/values-manager-eks-load.yaml b/test/load2-eks/profiles/xlarge/values-manager-eks-load.yaml
new file mode 100644
index 000000000..ffbc49423
--- /dev/null
+++ b/test/load2-eks/profiles/xlarge/values-manager-eks-load.yaml
@@ -0,0 +1,43 @@
+image:
+ pullPolicy: Always
+ tag: "load2"
+
+livenessProbe:
+ httpGet:
+ path: /
+ port: 8080
+ periodSeconds: 5
+readinessProbe:
+ httpGet:
+ path: /
+ port: 8080
+ periodSeconds: 5
+startupProbe:
+ httpGet:
+ path: /
+ port: 8080
+ failureThreshold: 500
+ periodSeconds: 10
+
+resources:
+ limits:
+ cpu: 400m
+ memory: 4Gi
+ requests:
+ cpu: 400m
+ memory: 4Gi
+
+volumeSize: "16Gi"
+
+or:
+ env:
+ - name: OR_SETUP_USERS
+ value: "1000"
+ - name: OR_SETUP_ASSETS
+ value: "5"
+ - name: OR_SETUP_STORE_DATA_POINTS
+ value: "false"
+ - name: OR_JAVA_OPTS
+ value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof -Xlog:gc*,gc+ref=debug,gc+phases=debug,gc+age=debug,safepoint:file=gc.log"
+ - name: JAVA_TOOL_OPTIONS
+ value: "-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=8085 -Dcom.sun.management.jmxremote.rmi.port=8085 -Djava.rmi.server.hostname=localhost"
diff --git a/test/load2-eks/profiles/xlarge/values-postgresql-eks-load.yaml b/test/load2-eks/profiles/xlarge/values-postgresql-eks-load.yaml
new file mode 100644
index 000000000..ce6c97c90
--- /dev/null
+++ b/test/load2-eks/profiles/xlarge/values-postgresql-eks-load.yaml
@@ -0,0 +1,9 @@
+volumeSize: "16Gi"
+
+resources:
+ limits:
+ cpu: 350m
+ memory: 1Gi
+ requests:
+ cpu: 350m
+ memory: 1Gi
diff --git a/test/load2-eks/profiles/xlarge/values-proxy-eks-load.yaml b/test/load2-eks/profiles/xlarge/values-proxy-eks-load.yaml
new file mode 100644
index 000000000..83704321d
--- /dev/null
+++ b/test/load2-eks/profiles/xlarge/values-proxy-eks-load.yaml
@@ -0,0 +1,7 @@
+resources:
+ limits:
+ cpu: 200m
+ memory: 512Mi
+ requests:
+ cpu: 200m
+ memory: 512Mi