Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Setup for load testing OR stack running in EKS (#2134)

Eric Bariaux53ac41ac4e3c
diff --git a/kubernetes/eks-cleanup-haproxy.sh b/kubernetes/eks-cleanup-haproxy.sh
index 928d109f5..60c9af558 100755
--- a/kubernetes/eks-cleanup-haproxy.sh
+++ b/kubernetes/eks-cleanup-haproxy.sh
@@ -2,8 +2,14 @@
. ./eks-common.sh
-DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[0].DNSName")
-HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[0].CanonicalHostedZoneId")
+CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)
+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then
+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."
+ exit 1
+fi
+
+DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")
+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")
echo "Delete DNS record $FQDN"
@@ -20,11 +26,6 @@ helm uninstall proxy
echo "Delete VPC peerings"
-CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)
-if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then
- echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."
- exit 1
-fi
VPC_PEERING_IDS=$(aws ec2 describe-vpc-peering-connections \
--filters "Name=accepter-vpc-info.vpc-id,Values=$CLUSTER_VPC_ID" "Name=status-code,Values=active" \
--query 'VpcPeeringConnections[*].VpcPeeringConnectionId' \
diff --git a/kubernetes/eks-setup-haproxy.sh b/kubernetes/eks-setup-haproxy.sh
index e2ad7b65d..8d9399f63 100755
--- a/kubernetes/eks-setup-haproxy.sh
+++ b/kubernetes/eks-setup-haproxy.sh
@@ -38,13 +38,13 @@ if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then
exit 1
fi
-while ! aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network']" 2>/dev/null | grep '"Code": "active"'; do
+while ! aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing']" 2>/dev/null | grep '"Code": "active"'; do
echo "Waiting for load balancer to be created..."
sleep 10
done
-DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network'].DNSName | [0]")
-HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network'].CanonicalHostedZoneId | [0]")
+DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")
+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")
aws route53 change-resource-record-sets \
--hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \
@@ -65,7 +65,7 @@ done
# Now that DNS is in place, HAProxy can properly create the certificate
kubectl exec $(kubectl get pod -l "app.kubernetes.io/name=proxy" -o name) -- sh -c "/entrypoint.sh add $FQDN" || {
echo "Certificate generation failed, please wait a moment and manually execute the following command:"
- echo 'kubectl exec $(kubectl get pod -l "app.kubernetes.io/name=proxy" -o name) -- sh -c "/entrypoint.sh add <fully qualified hostname>"'
+ echo "kubectl exec \$(kubectl get pod -l \"app.kubernetes.io/name=proxy\" -o name) -- sh -c \"/entrypoint.sh add $FQDN\""
}
echo "Access the manager at https://$FQDN"
diff --git a/test/load1-eks/README.md b/test/load1-eks/README.md
new file mode 100644
index 000000000..e284446b3
--- /dev/null
+++ b/test/load1-eks/README.md
@@ -0,0 +1,32 @@
+# Load Tests
+
+Scripts and configuration files to setup an OpenRemote stack running in an EKS cluster appropriate for load testing.
+This includes configuring the cluster to use a more powerful machine
+and adapting the values files to configure the memory usage on the different pods.
+For the manager, JVM parameters are used to make use of the extra memory available to the container.
+It also uses a custom manager image (available in the ECR) that includes setup code to create assets and users used during the tests.
+
+Once deployed, running the load tests can be done using the same clients as for load testing a VM,
+see `load1` folder for the tools, scripts and documentation.
+
+## Creating load test manager image
+
+There seems to be a problem in the setup gradle build file, and you need to comment out the
+```groovy
+tasks.register('demoJar', Jar) {
+ base {
+ archivesName = "openremote-demo-${project.name}"
+ }
+ from sourceSets.demo.output
+}
+```
+section from `setup/build.gradle` to properly build the load test image.
+
+After that, run
+```
+./gradlew -PSETUP_JAR=load1 clean installDist
+
+export AWS_DEVELOPERS_ACCOUNT_ID="dev-account-id"
+aws ecr get-login-password --region eu-west-1 | docker login --username AWS --password-stdin $AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com
+docker buildx build --push --platform linux/amd64,linux/arm64 -t $AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/manager:load1 manager/build/install/manager/
+```
diff --git a/test/load1-eks/cluster.yaml b/test/load1-eks/cluster.yaml
new file mode 100644
index 000000000..28a99f3aa
--- /dev/null
+++ b/test/load1-eks/cluster.yaml
@@ -0,0 +1,33 @@
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+metadata:
+ name: ${CLUSTER_NAME}
+ region: ${AWS_REGION}
+ # Kubernetes version to use for cluster, good idea to keep up to date
+ version: "1.33"
+
+availabilityZones:
+ - ${AWS_REGION}a
+ - ${AWS_REGION}b
+
+iam:
+ withOIDC: true
+ serviceAccounts:
+ - metadata:
+ name: aws-load-balancer-controller
+ namespace: kube-system
+ wellKnownPolicies:
+ awsLoadBalancerController: true
+
+nodeGroups:
+ - name: ng-1
+ instanceType: m8g.2xlarge
+ volumeSize: 32
+ desiredCapacity: 1
+
+addons:
+ - name: aws-ebs-csi-driver
+ version: latest
+ wellKnownPolicies:
+ ebsCSIController: true
diff --git a/test/load1-eks/eks-cleanup-load.sh b/test/load1-eks/eks-cleanup-load.sh
new file mode 100755
index 000000000..e0b2f811c
--- /dev/null
+++ b/test/load1-eks/eks-cleanup-load.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+. ./eks-common.sh
+
+CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)
+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then
+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."
+ exit 1
+fi
+
+DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")
+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")
+
+echo "Delete DNS record $FQDN"
+
+aws route53 change-resource-record-sets \
+ --hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \
+ --change-batch \
+ '{"Changes": [ { "Action": "DELETE", "ResourceRecordSet": { "Name": "'$FQDN'", "Type": "A", "AliasTarget":{ "HostedZoneId": '$HOSTED_ZONE_ID',"DNSName": '$DNS_NAME',"EvaluateTargetHealth": false} } } ]}' \
+ --profile dnschg
+
+helm uninstall manager
+helm uninstall keycloak
+helm uninstall postgresql
+helm uninstall proxy
+
+echo "Delete VPC peerings"
+
+VPC_PEERING_IDS=$(aws ec2 describe-vpc-peering-connections \
+ --filters "Name=accepter-vpc-info.vpc-id,Values=$CLUSTER_VPC_ID" "Name=status-code,Values=active" \
+ --query 'VpcPeeringConnections[*].VpcPeeringConnectionId' \
+ --profile or \
+ --output text)
+
+for peering_id in $VPC_PEERING_IDS; do
+ echo "Deleting peering $peering_id"
+ aws ec2 delete-vpc-peering-connection --vpc-peering-connection-id $peering_id --profile or
+done
+
+# Give the AWS LB Controller time to delete the NLBs after the Service is deleted
+while aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network']" 2>/dev/null | grep '"Code": "active"'; do
+ echo "Waiting for load balancers to be deleted..."
+ sleep 10
+done
+helm uninstall aws-load-balancer-controller -n kube-system
+
+MANAGER_VOLUMEID=$(kubectl get pv manager-data-pv -o=jsonpath='{.spec.awsElasticBlockStore.volumeID}')
+PSQL_VOLUMEID=$(kubectl get pv postgresql-data-pv -o=jsonpath='{.spec.awsElasticBlockStore.volumeID}')
+kubectl delete pv manager-data-pv
+kubectl delete pv postgresql-data-pv
+aws ec2 delete-volume --volume-id $MANAGER_VOLUMEID
+aws ec2 delete-volume --volume-id $PSQL_VOLUMEID
+helm uninstall or-setup
+
+# Manually deleting all addons, this should not be required but otherwise the delete cluster fails
+# with "2 pods are unevictable from node ..." error messages
+eksctl delete addon --cluster $CLUSTER_NAME --name aws-ebs-csi-driver
+eksctl delete addon --cluster $CLUSTER_NAME --name vpc-cni
+eksctl delete addon --cluster $CLUSTER_NAME --name metrics-server
+eksctl delete addon --cluster $CLUSTER_NAME --name coredns
+eksctl delete addon --cluster $CLUSTER_NAME --name kube-proxy
+
+eksctl delete cluster --wait --name $CLUSTER_NAME || {
+ echo "Clean-up has failed, please check the status in AWS CloudFormation console at https://eu-west-1.console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks?filteringText=&filteringStatus=active&viewNested=true"
+ echo "Failed clean-ups are often caused by a dangling VPC. Manually delete the VPC and then retry the cluster stack delete operation."
+}
diff --git a/test/load1-eks/eks-common.sh b/test/load1-eks/eks-common.sh
new file mode 100755
index 000000000..153a513cd
--- /dev/null
+++ b/test/load1-eks/eks-common.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -eo pipefail
+
+if [ -z "$AWS_ACCOUNT_ID" ]; then
+ echo "Error: AWS_ACCOUNT_ID environment variable is not set"
+ echo "Please set it to the ID of the account you want to deploy the cluster into with: export AWS_ACCOUNT_ID=your-account-id"
+ exit 1
+fi
+if [ -z "$AWS_DEVELOPERS_ACCOUNT_ID" ]; then
+ echo "Error: $AWS_DEVELOPERS_ACCOUNT_ID environment variable is not set"
+ echo "Please set it to the ID of the OpenRemote developers root account with: export AWS_DEVELOPERS_ACCOUNT_ID=dev-account-id"
+ exit 1
+fi
+
+OR_KUBERNETES_PATH=../../kubernetes
+
+# Name of cluster, not exposed but must be unique within account
+export CLUSTER_NAME=loadtest-cluster
+
+# Hostname to use for public access to this instance, always under the openremote.app domain
+HOSTNAME=load1
+FQDN=$HOSTNAME.openremote.app
+
+export AWS_REGION="eu-west-1"
+
+DNSCHG_ROLE_ARN="arn:aws:iam::$AWS_DEVELOPERS_ACCOUNT_ID:role/route53-full-access"
+
+aws configure set region $AWS_REGION
+aws configure --profile or set region $AWS_REGION
+aws configure --profile or set output json
+aws configure --profile dnschg set source_profile or
+aws configure --profile dnschg set role_arn $DNSCHG_ROLE_ARN
+
+if [ -n "$AWS_ACCESS_KEY_ID" ]; then
+ aws configure --profile or set aws_access_key_id $AWS_ACCESS_KEY_ID
+fi
+
+if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
+ aws configure --profile or set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+fi
+
+if [ -n "$AWS_SESSION_TOKEN" ]; then
+ aws configure --profile or set aws_session_token $AWS_SESSION_TOKEN
+fi
+
+aws sts assume-role --role-arn $DNSCHG_ROLE_ARN --role-session-name dnschg --profile or --query "Credentials.[AccessKeyId, SecretAccessKey, SessionToken]" --output text | awk -F'\t' '{print "aws_access_key_id "$1"\naws_secret_access_key "$2"\naws_session_token "$3 }' | xargs -L 1 aws configure --profile dnschg set
diff --git a/test/load1-eks/eks-setup-load.sh b/test/load1-eks/eks-setup-load.sh
new file mode 100755
index 000000000..972aa43b7
--- /dev/null
+++ b/test/load1-eks/eks-setup-load.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+. ./eks-common.sh
+
+envsubst < cluster.yaml | eksctl create cluster -f - --profile or
+
+# [Installation Guide - AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/installation/)
+
+helm repo add eks https://aws.github.io/eks-charts
+helm repo update eks
+
+helm install aws-load-balancer-controller eks/aws-load-balancer-controller \
+ -n kube-system \
+ --set clusterName=$CLUSTER_NAME \
+ --set serviceAccount.create=false \
+ --set serviceAccount.name=aws-load-balancer-controller
+
+PSQL_VOLUMEID=$(aws ec2 create-volume --size 1 --availability-zone eu-west-1a --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=psql-data}]" --query VolumeId)
+MANAGER_VOLUMEID=$(aws ec2 create-volume --size 1 --availability-zone eu-west-1a --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=manager-data}]" --query VolumeId)
+
+# Wait for AWS LB ctrl to be ready
+kubectl rollout status deployment aws-load-balancer-controller -n kube-system --timeout=300s
+
+CLUSTER_DNS=$(kubectl get svc kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}:{.spec.ports[?(@.name=="dns")].port}')
+
+helm install or-setup $OR_KUBERNETES_PATH/or-setup -f values-or-setup-eks-load.yaml --set aws.enabled=true --set aws.managerVolumeId=$MANAGER_VOLUMEID --set aws.psqlVolumeId=$PSQL_VOLUMEID
+
+helm install proxy $OR_KUBERNETES_PATH/proxy -f $OR_KUBERNETES_PATH/proxy/values-eks.yaml -f values-proxy-eks-load.yaml --set or.nameserver=$CLUSTER_DNS --set or.hostname=$FQDN
+
+helm install postgresql $OR_KUBERNETES_PATH/postgresql -f $OR_KUBERNETES_PATH/postgresql/values-eks.yaml -f values-postgresql-eks-load.yaml
+
+# Waiting for the LB to be created
+# AWS LB Controller only creates an Network LB if there's a service
+
+CLUSTER_VPC_ID=$(aws eks describe-cluster --profile or --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text)
+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then
+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."
+ exit 1
+fi
+
+while ! aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing']" 2>/dev/null | grep '"Code": "active"'; do
+ echo "Waiting for load balancer to be created..."
+ sleep 10
+done
+
+DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")
+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")
+
+aws route53 change-resource-record-sets \
+ --hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \
+ --change-batch \
+ '{"Changes": [ { "Action": "UPSERT", "ResourceRecordSet": { "Name": "'$FQDN'", "Type": "A", "AliasTarget":{ "HostedZoneId": '$HOSTED_ZONE_ID',"DNSName": '$DNS_NAME',"EvaluateTargetHealth": false} } } ]}' \
+ --profile dnschg
+
+helm install keycloak $OR_KUBERNETES_PATH/keycloak -f $OR_KUBERNETES_PATH/keycloak/values-haproxy.yaml \
+ -f values-keycloak-eks-load.yaml --set-string or.hostname=$FQDN
+helm install manager $OR_KUBERNETES_PATH/manager -f $OR_KUBERNETES_PATH/manager/values-haproxy-eks.yaml \
+ -f values-manager-eks-load.yaml --set-string or.hostname=$FQDN \
+ --set-string image.repository=$AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/manager
+
+while ! dig +short $FQDN | grep -qE '^[0-9]'; do
+ echo "Waiting for DNS resolution..."
+ sleep 5
+done
+
+# Now that DNS is in place, HAProxy can properly create the certificate
+kubectl exec $(kubectl get pod -l "app.kubernetes.io/name=proxy" -o name) -- sh -c "/entrypoint.sh add $FQDN" || {
+ echo "Certificate generation failed, please wait a moment and manually execute the following command:"
+ echo "kubectl exec \$(kubectl get pod -l \"app.kubernetes.io/name=proxy\" -o name) -- sh -c \"/entrypoint.sh add $FQDN\""
+}
+
+echo "Access the manager at https://$FQDN"
diff --git a/test/load1-eks/values-keycloak-eks-load.yaml b/test/load1-eks/values-keycloak-eks-load.yaml
new file mode 100644
index 000000000..ed0ca53cc
--- /dev/null
+++ b/test/load1-eks/values-keycloak-eks-load.yaml
@@ -0,0 +1,7 @@
+resources:
+ limits:
+ cpu: 1500m
+ memory: 5Gi
+ requests:
+ cpu: 1200m
+ memory: 5Gi
diff --git a/test/load1-eks/values-manager-eks-load.yaml b/test/load1-eks/values-manager-eks-load.yaml
new file mode 100644
index 000000000..9f4b6696a
--- /dev/null
+++ b/test/load1-eks/values-manager-eks-load.yaml
@@ -0,0 +1,40 @@
+image:
+ tag: "load1"
+
+livenessProbe:
+ httpGet:
+ path: /
+ port: 8080
+ periodSeconds: 5
+readinessProbe:
+ httpGet:
+ path: /
+ port: 8080
+ periodSeconds: 5
+startupProbe:
+ httpGet:
+ path: /
+ port: 8080
+ failureThreshold: 500
+ periodSeconds: 10
+
+resources:
+ limits:
+ cpu: 3200m
+ memory: 13Gi
+ requests:
+ cpu: 3000m
+ memory: 13Gi
+
+volumeSize: "16Gi"
+
+or:
+ env:
+ - name: OR_SETUP_REGULAR_USERS
+ value: "1000"
+ - name: OR_SETUP_DEVICES
+ value: "10000"
+ - name: OR_SETUP_DEVICES_SKIP_COUNT
+ value: "100"
+ - name: OR_JAVA_OPTS
+ value: "-Dhibernate.generate_statistics=true -XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -XX:NativeMemoryTracking=summary -Xlog:all=warning:stdout:uptime,level,tags -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof"
diff --git a/test/load1-eks/values-or-setup-eks-load.yaml b/test/load1-eks/values-or-setup-eks-load.yaml
new file mode 100644
index 000000000..8dcadffa0
--- /dev/null
+++ b/test/load1-eks/values-or-setup-eks-load.yaml
@@ -0,0 +1,6 @@
+managerVolumeSize: "16Gi"
+psqlVolumeSize: "16Gi"
+
+aws:
+ # Set enabled to true if deployed under AWS
+ enabled: true
diff --git a/test/load1-eks/values-postgresql-eks-load.yaml b/test/load1-eks/values-postgresql-eks-load.yaml
new file mode 100644
index 000000000..7b8df849e
--- /dev/null
+++ b/test/load1-eks/values-postgresql-eks-load.yaml
@@ -0,0 +1,9 @@
+volumeSize: "16Gi"
+
+resources:
+ limits:
+ cpu: 1500m
+ memory: 5Gi
+ requests:
+ cpu: 1400m
+ memory: 5Gi
diff --git a/test/load1-eks/values-proxy-eks-load.yaml b/test/load1-eks/values-proxy-eks-load.yaml
new file mode 100644
index 000000000..971a2db22
--- /dev/null
+++ b/test/load1-eks/values-proxy-eks-load.yaml
@@ -0,0 +1,7 @@
+resources:
+ limits:
+ cpu: 1000m
+ memory: 5Gi
+ requests:
+ cpu: 1000m
+ memory: 5Gi