100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Setup for load testing OR stack running in EKS (#2134)
Eric Bariaux53ac41ac4e3c
diff --git a/kubernetes/eks-cleanup-haproxy.sh b/kubernetes/eks-cleanup-haproxy.shindex 928d109f5..60c9af558 100755--- a/kubernetes/eks-cleanup-haproxy.sh+++ b/kubernetes/eks-cleanup-haproxy.sh@@ -2,8 +2,14 @@. ./eks-common.sh-DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[0].DNSName")-HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[0].CanonicalHostedZoneId")+CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."+ exit 1+fi++DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")echo "Delete DNS record $FQDN"@@ -20,11 +26,6 @@ helm uninstall proxyecho "Delete VPC peerings"-CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)-if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then- echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."- exit 1-fiVPC_PEERING_IDS=$(aws ec2 describe-vpc-peering-connections \--filters "Name=accepter-vpc-info.vpc-id,Values=$CLUSTER_VPC_ID" "Name=status-code,Values=active" \--query 'VpcPeeringConnections[*].VpcPeeringConnectionId' \diff --git a/kubernetes/eks-setup-haproxy.sh b/kubernetes/eks-setup-haproxy.shindex e2ad7b65d..8d9399f63 100755--- a/kubernetes/eks-setup-haproxy.sh+++ b/kubernetes/eks-setup-haproxy.sh@@ -38,13 +38,13 @@ if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; thenexit 1fi-while ! aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network']" 2>/dev/null | grep '"Code": "active"'; do+while ! aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing']" 2>/dev/null | grep '"Code": "active"'; doecho "Waiting for load balancer to be created..."sleep 10done-DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network'].DNSName | [0]")-HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network'].CanonicalHostedZoneId | [0]")+DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")aws route53 change-resource-record-sets \--hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \@@ -65,7 +65,7 @@ done# Now that DNS is in place, HAProxy can properly create the certificatekubectl exec $(kubectl get pod -l "app.kubernetes.io/name=proxy" -o name) -- sh -c "/entrypoint.sh add $FQDN" || {echo "Certificate generation failed, please wait a moment and manually execute the following command:"- echo 'kubectl exec $(kubectl get pod -l "app.kubernetes.io/name=proxy" -o name) -- sh -c "/entrypoint.sh add <fully qualified hostname>"'+ echo "kubectl exec \$(kubectl get pod -l \"app.kubernetes.io/name=proxy\" -o name) -- sh -c \"/entrypoint.sh add $FQDN\""}echo "Access the manager at https://$FQDN"diff --git a/test/load1-eks/README.md b/test/load1-eks/README.mdnew file mode 100644index 000000000..e284446b3--- /dev/null+++ b/test/load1-eks/README.md@@ -0,0 +1,32 @@+# Load Tests++Scripts and configuration files to setup an OpenRemote stack running in an EKS cluster appropriate for load testing.+This includes configuring the cluster to use a more powerful machine+and adapting the values files to configure the memory usage on the different pods.+For the manager, JVM parameters are used to make use of the extra memory available to the container.+It also uses a custom manager image (available in the ECR) that includes setup code to create assets and users used during the tests.++Once deployed, running the load tests can be done using the same clients as for load testing a VM,+see `load1` folder for the tools, scripts and documentation.++## Creating load test manager image++There seems to be a problem in the setup gradle build file, and you need to comment out the+```groovy+tasks.register('demoJar', Jar) {+ base {+ archivesName = "openremote-demo-${project.name}"+ }+ from sourceSets.demo.output+}+```+section from `setup/build.gradle` to properly build the load test image.++After that, run+```+./gradlew -PSETUP_JAR=load1 clean installDist++export AWS_DEVELOPERS_ACCOUNT_ID="dev-account-id"+aws ecr get-login-password --region eu-west-1 | docker login --username AWS --password-stdin $AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com+docker buildx build --push --platform linux/amd64,linux/arm64 -t $AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/manager:load1 manager/build/install/manager/+```diff --git a/test/load1-eks/cluster.yaml b/test/load1-eks/cluster.yamlnew file mode 100644index 000000000..28a99f3aa--- /dev/null+++ b/test/load1-eks/cluster.yaml@@ -0,0 +1,33 @@+apiVersion: eksctl.io/v1alpha5+kind: ClusterConfig++metadata:+ name: ${CLUSTER_NAME}+ region: ${AWS_REGION}+ # Kubernetes version to use for cluster, good idea to keep up to date+ version: "1.33"++availabilityZones:+ - ${AWS_REGION}a+ - ${AWS_REGION}b++iam:+ withOIDC: true+ serviceAccounts:+ - metadata:+ name: aws-load-balancer-controller+ namespace: kube-system+ wellKnownPolicies:+ awsLoadBalancerController: true++nodeGroups:+ - name: ng-1+ instanceType: m8g.2xlarge+ volumeSize: 32+ desiredCapacity: 1++addons:+ - name: aws-ebs-csi-driver+ version: latest+ wellKnownPolicies:+ ebsCSIController: truediff --git a/test/load1-eks/eks-cleanup-load.sh b/test/load1-eks/eks-cleanup-load.shnew file mode 100755index 000000000..e0b2f811c--- /dev/null+++ b/test/load1-eks/eks-cleanup-load.sh@@ -0,0 +1,66 @@+#!/bin/bash++. ./eks-common.sh++CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."+ exit 1+fi++DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")++echo "Delete DNS record $FQDN"++aws route53 change-resource-record-sets \+ --hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \+ --change-batch \+ '{"Changes": [ { "Action": "DELETE", "ResourceRecordSet": { "Name": "'$FQDN'", "Type": "A", "AliasTarget":{ "HostedZoneId": '$HOSTED_ZONE_ID',"DNSName": '$DNS_NAME',"EvaluateTargetHealth": false} } } ]}' \+ --profile dnschg++helm uninstall manager+helm uninstall keycloak+helm uninstall postgresql+helm uninstall proxy++echo "Delete VPC peerings"++VPC_PEERING_IDS=$(aws ec2 describe-vpc-peering-connections \+ --filters "Name=accepter-vpc-info.vpc-id,Values=$CLUSTER_VPC_ID" "Name=status-code,Values=active" \+ --query 'VpcPeeringConnections[*].VpcPeeringConnectionId' \+ --profile or \+ --output text)++for peering_id in $VPC_PEERING_IDS; do+ echo "Deleting peering $peering_id"+ aws ec2 delete-vpc-peering-connection --vpc-peering-connection-id $peering_id --profile or+done++# Give the AWS LB Controller time to delete the NLBs after the Service is deleted+while aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network']" 2>/dev/null | grep '"Code": "active"'; do+ echo "Waiting for load balancers to be deleted..."+ sleep 10+done+helm uninstall aws-load-balancer-controller -n kube-system++MANAGER_VOLUMEID=$(kubectl get pv manager-data-pv -o=jsonpath='{.spec.awsElasticBlockStore.volumeID}')+PSQL_VOLUMEID=$(kubectl get pv postgresql-data-pv -o=jsonpath='{.spec.awsElasticBlockStore.volumeID}')+kubectl delete pv manager-data-pv+kubectl delete pv postgresql-data-pv+aws ec2 delete-volume --volume-id $MANAGER_VOLUMEID+aws ec2 delete-volume --volume-id $PSQL_VOLUMEID+helm uninstall or-setup++# Manually deleting all addons, this should not be required but otherwise the delete cluster fails+# with "2 pods are unevictable from node ..." error messages+eksctl delete addon --cluster $CLUSTER_NAME --name aws-ebs-csi-driver+eksctl delete addon --cluster $CLUSTER_NAME --name vpc-cni+eksctl delete addon --cluster $CLUSTER_NAME --name metrics-server+eksctl delete addon --cluster $CLUSTER_NAME --name coredns+eksctl delete addon --cluster $CLUSTER_NAME --name kube-proxy++eksctl delete cluster --wait --name $CLUSTER_NAME || {+ echo "Clean-up has failed, please check the status in AWS CloudFormation console at https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks?filteringText=&filteringStatus=active&viewNested=true"+ echo "Failed clean-ups are often caused by a dangling VPC. Manually delete the VPC and then retry the cluster stack delete operation."+}diff --git a/test/load1-eks/eks-common.sh b/test/load1-eks/eks-common.shnew file mode 100755index 000000000..153a513cd--- /dev/null+++ b/test/load1-eks/eks-common.sh@@ -0,0 +1,47 @@+#!/bin/bash++set -eo pipefail++if [ -z "$AWS_ACCOUNT_ID" ]; then+ echo "Error: AWS_ACCOUNT_ID environment variable is not set"+ echo "Please set it to the ID of the account you want to deploy the cluster into with: export AWS_ACCOUNT_ID=your-account-id"+ exit 1+fi+if [ -z "$AWS_DEVELOPERS_ACCOUNT_ID" ]; then+ echo "Error: $AWS_DEVELOPERS_ACCOUNT_ID environment variable is not set"+ echo "Please set it to the ID of the OpenRemote developers root account with: export AWS_DEVELOPERS_ACCOUNT_ID=dev-account-id"+ exit 1+fi++OR_KUBERNETES_PATH=../../kubernetes++# Name of cluster, not exposed but must be unique within account+export CLUSTER_NAME=loadtest-cluster++# Hostname to use for public access to this instance, always under the openremote.app domain+HOSTNAME=load1+FQDN=$HOSTNAME.openremote.app++export AWS_REGION="eu-west-1"++DNSCHG_ROLE_ARN="arn:aws:iam::$AWS_DEVELOPERS_ACCOUNT_ID:role/route53-full-access"++aws configure set region $AWS_REGION+aws configure --profile or set region $AWS_REGION+aws configure --profile or set output json+aws configure --profile dnschg set source_profile or+aws configure --profile dnschg set role_arn $DNSCHG_ROLE_ARN++if [ -n "$AWS_ACCESS_KEY_ID" ]; then+ aws configure --profile or set aws_access_key_id $AWS_ACCESS_KEY_ID+fi++if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then+ aws configure --profile or set aws_secret_access_key $AWS_SECRET_ACCESS_KEY+fi++if [ -n "$AWS_SESSION_TOKEN" ]; then+ aws configure --profile or set aws_session_token $AWS_SESSION_TOKEN+fi++aws sts assume-role --role-arn $DNSCHG_ROLE_ARN --role-session-name dnschg --profile or --query "Credentials.[AccessKeyId, SecretAccessKey, SessionToken]" --output text | awk -F'\t' '{print "aws_access_key_id "$1"\naws_secret_access_key "$2"\naws_session_token "$3 }' | xargs -L 1 aws configure --profile dnschg setdiff --git a/test/load1-eks/eks-setup-load.sh b/test/load1-eks/eks-setup-load.shnew file mode 100755index 000000000..972aa43b7--- /dev/null+++ b/test/load1-eks/eks-setup-load.sh@@ -0,0 +1,72 @@+#!/bin/bash++. ./eks-common.sh++envsubst < cluster.yaml | eksctl create cluster -f - --profile or++# [Installation Guide - AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/installation/)++helm repo add eks https://aws.github.io/eks-charts+helm repo update eks++helm install aws-load-balancer-controller eks/aws-load-balancer-controller \+ -n kube-system \+ --set clusterName=$CLUSTER_NAME \+ --set serviceAccount.create=false \+ --set serviceAccount.name=aws-load-balancer-controller++PSQL_VOLUMEID=$(aws ec2 create-volume --size 1 --availability-zone eu-west-1a --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=psql-data}]" --query VolumeId)+MANAGER_VOLUMEID=$(aws ec2 create-volume --size 1 --availability-zone eu-west-1a --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=manager-data}]" --query VolumeId)++# Wait for AWS LB ctrl to be ready+kubectl rollout status deployment aws-load-balancer-controller -n kube-system --timeout=300s++CLUSTER_DNS=$(kubectl get svc kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}:{.spec.ports[?(@.name=="dns")].port}')++helm install or-setup $OR_KUBERNETES_PATH/or-setup -f values-or-setup-eks-load.yaml --set aws.enabled=true --set aws.managerVolumeId=$MANAGER_VOLUMEID --set aws.psqlVolumeId=$PSQL_VOLUMEID++helm install proxy $OR_KUBERNETES_PATH/proxy -f $OR_KUBERNETES_PATH/proxy/values-eks.yaml -f values-proxy-eks-load.yaml --set or.nameserver=$CLUSTER_DNS --set or.hostname=$FQDN++helm install postgresql $OR_KUBERNETES_PATH/postgresql -f $OR_KUBERNETES_PATH/postgresql/values-eks.yaml -f values-postgresql-eks-load.yaml++# Waiting for the LB to be created+# AWS LB Controller only creates an Network LB if there's a service++CLUSTER_VPC_ID=$(aws eks describe-cluster --profile or --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text)+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."+ exit 1+fi++while ! aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing']" 2>/dev/null | grep '"Code": "active"'; do+ echo "Waiting for load balancer to be created..."+ sleep 10+done++DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")++aws route53 change-resource-record-sets \+ --hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \+ --change-batch \+ '{"Changes": [ { "Action": "UPSERT", "ResourceRecordSet": { "Name": "'$FQDN'", "Type": "A", "AliasTarget":{ "HostedZoneId": '$HOSTED_ZONE_ID',"DNSName": '$DNS_NAME',"EvaluateTargetHealth": false} } } ]}' \+ --profile dnschg++helm install keycloak $OR_KUBERNETES_PATH/keycloak -f $OR_KUBERNETES_PATH/keycloak/values-haproxy.yaml \+ -f values-keycloak-eks-load.yaml --set-string or.hostname=$FQDN+helm install manager $OR_KUBERNETES_PATH/manager -f $OR_KUBERNETES_PATH/manager/values-haproxy-eks.yaml \+ -f values-manager-eks-load.yaml --set-string or.hostname=$FQDN \+ --set-string image.repository=$AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/manager++while ! dig +short $FQDN | grep -qE '^[0-9]'; do+ echo "Waiting for DNS resolution..."+ sleep 5+done++# Now that DNS is in place, HAProxy can properly create the certificate+kubectl exec $(kubectl get pod -l "app.kubernetes.io/name=proxy" -o name) -- sh -c "/entrypoint.sh add $FQDN" || {+ echo "Certificate generation failed, please wait a moment and manually execute the following command:"+ echo "kubectl exec \$(kubectl get pod -l \"app.kubernetes.io/name=proxy\" -o name) -- sh -c \"/entrypoint.sh add $FQDN\""+}++echo "Access the manager at https://$FQDN"diff --git a/test/load1-eks/values-keycloak-eks-load.yaml b/test/load1-eks/values-keycloak-eks-load.yamlnew file mode 100644index 000000000..ed0ca53cc--- /dev/null+++ b/test/load1-eks/values-keycloak-eks-load.yaml@@ -0,0 +1,7 @@+resources:+ limits:+ cpu: 1500m+ memory: 5Gi+ requests:+ cpu: 1200m+ memory: 5Gidiff --git a/test/load1-eks/values-manager-eks-load.yaml b/test/load1-eks/values-manager-eks-load.yamlnew file mode 100644index 000000000..9f4b6696a--- /dev/null+++ b/test/load1-eks/values-manager-eks-load.yaml@@ -0,0 +1,40 @@+image:+ tag: "load1"++livenessProbe:+ httpGet:+ path: /+ port: 8080+ periodSeconds: 5+readinessProbe:+ httpGet:+ path: /+ port: 8080+ periodSeconds: 5+startupProbe:+ httpGet:+ path: /+ port: 8080+ failureThreshold: 500+ periodSeconds: 10++resources:+ limits:+ cpu: 3200m+ memory: 13Gi+ requests:+ cpu: 3000m+ memory: 13Gi++volumeSize: "16Gi"++or:+ env:+ - name: OR_SETUP_REGULAR_USERS+ value: "1000"+ - name: OR_SETUP_DEVICES+ value: "10000"+ - name: OR_SETUP_DEVICES_SKIP_COUNT+ value: "100"+ - name: OR_JAVA_OPTS+ value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof"diff --git a/test/load1-eks/values-or-setup-eks-load.yaml b/test/load1-eks/values-or-setup-eks-load.yamlnew file mode 100644index 000000000..8dcadffa0--- /dev/null+++ b/test/load1-eks/values-or-setup-eks-load.yaml@@ -0,0 +1,6 @@+managerVolumeSize: "16Gi"+psqlVolumeSize: "16Gi"++aws:+ # Set enabled to true if deployed under AWS+ enabled: truediff --git a/test/load1-eks/values-postgresql-eks-load.yaml b/test/load1-eks/values-postgresql-eks-load.yamlnew file mode 100644index 000000000..7b8df849e--- /dev/null+++ b/test/load1-eks/values-postgresql-eks-load.yaml@@ -0,0 +1,9 @@+volumeSize: "16Gi"++resources:+ limits:+ cpu: 1500m+ memory: 5Gi+ requests:+ cpu: 1400m+ memory: 5Gidiff --git a/test/load1-eks/values-proxy-eks-load.yaml b/test/load1-eks/values-proxy-eks-load.yamlnew file mode 100644index 000000000..971a2db22--- /dev/null+++ b/test/load1-eks/values-proxy-eks-load.yaml@@ -0,0 +1,7 @@+resources:+ limits:+ cpu: 1000m+ memory: 5Gi+ requests:+ cpu: 1000m+ memory: 5Gi