100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
load2 support for multiple HAProxy pod replicas (#2657)
Eric Bariauxf425ed17049a
diff --git a/kubernetes/proxy/templates/service.yaml b/kubernetes/proxy/templates/service.yamlindex 252ef8835..aed7dc5b3 100644--- a/kubernetes/proxy/templates/service.yaml+++ b/kubernetes/proxy/templates/service.yaml@@ -20,7 +20,7 @@ spec:protocol: TCPname: https- port: 8883- targetPort: 8883+ targetPort: {{ .Values.or.mqttPort }}protocol: TCPname: mqttsselector:diff --git a/kubernetes/proxy/values.yaml b/kubernetes/proxy/values.yamlindex 8c5a52b29..c1c7a99b6 100644--- a/kubernetes/proxy/values.yaml+++ b/kubernetes/proxy/values.yaml@@ -161,6 +161,9 @@ or:# The path prefix used for Keycloak HTTP requests (default not set, example: `/keycloak`)# keycloakPathPrefix:+ # The container binds to this port for handling MQTT(S) requests, must be a non-privileged port (>= 1024) when running inside the cluster+ mqttPort: 8883+# The container binds to this port for handling HTTP requests, must be a non-privileged port (>= 1024) when running inside the clusterhttpPort: 8080# The container binds to this port for handling HTTPS requests, must be a non-privileged port (>= 1024) when running inside the clusterdiff --git a/test/load2-eks/README.md b/test/load2-eks/README.mdindex 987e3fe46..0f043163c 100644--- a/test/load2-eks/README.md+++ b/test/load2-eks/README.md@@ -84,3 +84,10 @@ MANAGER_HOSTNAME: Hostname of the manager to be tested. Default is localhost.THREAD_COUNT: Number of parallel accounts that will connect and publish in parallel. Default is 1000.RAMP_RATE: Number of thread to add per second during ramp-up. Default is 50.MILLIS_BETWEEN_PUBLISHES: Delay between each publishing iteration. Default is 30000.++## Multi-proxy test++As part of [Test using multiple HAProxy replicas under EKS · Issue #2568 · openremote/openremote](https://github.com/openremote/openremote/issues/2568),+a setup terminating TLS at the AWS NLB level and using ACM to manage certificates, while still using HAProxy pod(s) has been implemented.+This can be installed using the `eks-setup-load-acm.sh` script.+This is really for testing purposes and clarification on how we deploy our stack is required to move further.diff --git a/test/load2-eks/eks-setup-load-acm.sh b/test/load2-eks/eks-setup-load-acm.shnew file mode 100755index 000000000..4e5ea0d0a--- /dev/null+++ b/test/load2-eks/eks-setup-load-acm.sh@@ -0,0 +1,90 @@+#!/bin/bash++. ./eks-common.sh++envsubst < profiles/$OR_PROFILE/cluster.yaml | eksctl create cluster -f - --profile or++# [Installation Guide - AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/installation/)++helm repo add eks https://aws.github.io/eks-charts+helm repo update eks++helm install aws-load-balancer-controller eks/aws-load-balancer-controller \+ -n kube-system \+ --set clusterName=$CLUSTER_NAME \+ --set serviceAccount.create=false \+ --set serviceAccount.name=aws-load-balancer-controller++PSQL_VOLUMEID=$(aws ec2 create-volume --size $(grep "psqlVolumeSize:" values-or-setup-eks-load.yaml | awk '{print $2}' | tr -d '"Gi') \+ --availability-zone eu-west-1a --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=psql-data}]" --query VolumeId)+MANAGER_VOLUMEID=$(aws ec2 create-volume --size $(grep "managerVolumeSize:" values-or-setup-eks-load.yaml | awk '{print $2}' | tr -d '"Gi') \+ --availability-zone eu-west-1a --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=manager-data}]" --query VolumeId)++# Wait for AWS LB ctrl to be ready+kubectl rollout status deployment aws-load-balancer-controller -n kube-system --timeout=300s++CLUSTER_DNS=$(kubectl get svc kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}:{.spec.ports[?(@.name=="dns")].port}')++helm install or-setup $OR_KUBERNETES_PATH/or-setup -f values-or-setup-eks-load.yaml --set aws.enabled=true --set aws.managerVolumeId=$MANAGER_VOLUMEID --set aws.psqlVolumeId=$PSQL_VOLUMEID++CERTIFICATE_ARN=$(aws acm request-certificate --domain-name $FQDN --validation-method DNS --profile or --query "CertificateArn" --output text)++DNS_RECORD_NAME=""+DNS_RECORD_VALUE=""+until [ -n "$DNS_RECORD_NAME" ] && [ "$DNS_RECORD_NAME" != "None" ] && [ -n "$DNS_RECORD_VALUE" ] && [ "$DNS_RECORD_VALUE" != "None" ]; do+ echo "Waiting for ACM DNS validation record for $FQDN..."+ DNS_RECORD_NAME=$(aws acm describe-certificate --certificate-arn $CERTIFICATE_ARN --profile or --query "Certificate.DomainValidationOptions[0].ResourceRecord.Name" --output text 2>/dev/null)+ DNS_RECORD_VALUE=$(aws acm describe-certificate --certificate-arn $CERTIFICATE_ARN --profile or --query "Certificate.DomainValidationOptions[0].ResourceRecord.Value" --output text 2>/dev/null)+ [ -n "$DNS_RECORD_NAME" ] && [ "$DNS_RECORD_NAME" != "None" ] && [ -n "$DNS_RECORD_VALUE" ] && [ "$DNS_RECORD_VALUE" != "None" ] && break+ sleep 5+done++aws route53 change-resource-record-sets \+ --hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \+ --change-batch \+ '{"Changes": [ { "Action": "UPSERT", "ResourceRecordSet": { "Name": "'$DNS_RECORD_NAME'", "Type": "CNAME", "TTL": 300, "ResourceRecords" : [ { "Value": "'$DNS_RECORD_VALUE'" } ] } } ]}' \+ --profile dnschg++helm install proxy $OR_KUBERNETES_PATH/proxy \+ -f $OR_KUBERNETES_PATH/proxy/values-eks.yaml -f profiles/$OR_PROFILE/values-proxy-eks-load.yaml -f values-proxy-acm-load.yaml \+ --set or.nameserver=$CLUSTER_DNS --set or.hostname=$FQDN \+ --set-string 'service.http.annotations.service\.beta\.kubernetes\.io\/aws-load-balancer-ssl-cert'=$CERTIFICATE_ARN \+ --set-string image.repository=$AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/proxy++helm install postgresql $OR_KUBERNETES_PATH/postgresql -f $OR_KUBERNETES_PATH/postgresql/values-eks.yaml -f profiles/$OR_PROFILE/values-postgresql-eks-load.yaml++# Waiting for the LB to be created+# AWS LB Controller only creates an Network LB if there's a service++CLUSTER_VPC_ID=$(aws eks describe-cluster --profile or --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text)+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."+ exit 1+fi++while ! aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing']" 2>/dev/null | grep '"Code": "active"'; do+ echo "Waiting for load balancer to be created..."+ sleep 10+done++DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")++aws route53 change-resource-record-sets \+ --hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \+ --change-batch \+ '{"Changes": [ { "Action": "UPSERT", "ResourceRecordSet": { "Name": "'$FQDN'", "Type": "A", "AliasTarget":{ "HostedZoneId": '$HOSTED_ZONE_ID',"DNSName": '$DNS_NAME',"EvaluateTargetHealth": false} } } ]}' \+ --profile dnschg++helm install keycloak $OR_KUBERNETES_PATH/keycloak -f $OR_KUBERNETES_PATH/keycloak/values-haproxy.yaml \+ -f profiles/$OR_PROFILE/values-keycloak-eks-load.yaml --set-string or.hostname=$FQDN+helm install manager $OR_KUBERNETES_PATH/manager -f $OR_KUBERNETES_PATH/manager/values-haproxy-eks.yaml \+ -f profiles/$OR_PROFILE/values-manager-eks-load.yaml --set-string or.hostname=$FQDN \+ --set-string image.repository=$AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/manager++while ! dig +short $FQDN | grep -qE '^[0-9]'; do+ echo "Waiting for DNS resolution..."+ sleep 5+done++echo "Access the manager at https://$FQDN"diff --git a/test/load2-eks/values-proxy-acm-load.yaml b/test/load2-eks/values-proxy-acm-load.yamlnew file mode 100644index 000000000..34b63abbf--- /dev/null+++ b/test/load2-eks/values-proxy-acm-load.yaml@@ -0,0 +1,16 @@+service:+ http:+ annotations:+ service.beta.kubernetes.io/aws-load-balancer-type: "external"+ service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"+ service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"++ service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:eu-west-1:<openremote account id>:certificate/..."+ service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443,8883"+or:+ haproxyConfig: /etc/haproxy/haproxy-edge-terminated-tls.cfg+ httpsPort: 8080+ mqttPort: 1883+ env:+ - name: DISABLE_ACME+ value: "true"