Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

load2 support for multiple HAProxy pod replicas (#2657)

Eric Bariauxf425ed17049a
diff --git a/kubernetes/proxy/templates/service.yaml b/kubernetes/proxy/templates/service.yaml
index 252ef8835..aed7dc5b3 100644
--- a/kubernetes/proxy/templates/service.yaml
+++ b/kubernetes/proxy/templates/service.yaml
@@ -20,7 +20,7 @@ spec:
protocol: TCP
name: https
- port: 8883
- targetPort: 8883
+ targetPort: {{ .Values.or.mqttPort }}
protocol: TCP
name: mqtts
selector:
diff --git a/kubernetes/proxy/values.yaml b/kubernetes/proxy/values.yaml
index 8c5a52b29..c1c7a99b6 100644
--- a/kubernetes/proxy/values.yaml
+++ b/kubernetes/proxy/values.yaml
@@ -161,6 +161,9 @@ or:
# The path prefix used for Keycloak HTTP requests (default not set, example: `/keycloak`)
# keycloakPathPrefix:
+ # The container binds to this port for handling MQTT(S) requests, must be a non-privileged port (>= 1024) when running inside the cluster
+ mqttPort: 8883
+
# The container binds to this port for handling HTTP requests, must be a non-privileged port (>= 1024) when running inside the cluster
httpPort: 8080
# The container binds to this port for handling HTTPS requests, must be a non-privileged port (>= 1024) when running inside the cluster
diff --git a/test/load2-eks/README.md b/test/load2-eks/README.md
index 987e3fe46..0f043163c 100644
--- a/test/load2-eks/README.md
+++ b/test/load2-eks/README.md
@@ -84,3 +84,10 @@ MANAGER_HOSTNAME: Hostname of the manager to be tested. Default is localhost.
THREAD_COUNT: Number of parallel accounts that will connect and publish in parallel. Default is 1000.
RAMP_RATE: Number of thread to add per second during ramp-up. Default is 50.
MILLIS_BETWEEN_PUBLISHES: Delay between each publishing iteration. Default is 30000.
+
+## Multi-proxy test
+
+As part of [Test using multiple HAProxy replicas under EKS · Issue #2568 · openremote/openremote](https://github.com/openremote/openremote/issues/2568),
+a setup terminating TLS at the AWS NLB level and using ACM to manage certificates, while still using HAProxy pod(s) has been implemented.
+This can be installed using the `eks-setup-load-acm.sh` script.
+This is really for testing purposes and clarification on how we deploy our stack is required to move further.
diff --git a/test/load2-eks/eks-setup-load-acm.sh b/test/load2-eks/eks-setup-load-acm.sh
new file mode 100755
index 000000000..4e5ea0d0a
--- /dev/null
+++ b/test/load2-eks/eks-setup-load-acm.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+. ./eks-common.sh
+
+envsubst < profiles/$OR_PROFILE/cluster.yaml | eksctl create cluster -f - --profile or
+
+# [Installation Guide - AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/deploy/installation/)
+
+helm repo add eks https://aws.github.io/eks-charts
+helm repo update eks
+
+helm install aws-load-balancer-controller eks/aws-load-balancer-controller \
+ -n kube-system \
+ --set clusterName=$CLUSTER_NAME \
+ --set serviceAccount.create=false \
+ --set serviceAccount.name=aws-load-balancer-controller
+
+PSQL_VOLUMEID=$(aws ec2 create-volume --size $(grep "psqlVolumeSize:" values-or-setup-eks-load.yaml | awk '{print $2}' | tr -d '"Gi') \
+ --availability-zone eu-west-1a --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=psql-data}]" --query VolumeId)
+MANAGER_VOLUMEID=$(aws ec2 create-volume --size $(grep "managerVolumeSize:" values-or-setup-eks-load.yaml | awk '{print $2}' | tr -d '"Gi') \
+ --availability-zone eu-west-1a --tag-specifications "ResourceType=volume,Tags=[{Key=Name,Value=manager-data}]" --query VolumeId)
+
+# Wait for AWS LB ctrl to be ready
+kubectl rollout status deployment aws-load-balancer-controller -n kube-system --timeout=300s
+
+CLUSTER_DNS=$(kubectl get svc kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}:{.spec.ports[?(@.name=="dns")].port}')
+
+helm install or-setup $OR_KUBERNETES_PATH/or-setup -f values-or-setup-eks-load.yaml --set aws.enabled=true --set aws.managerVolumeId=$MANAGER_VOLUMEID --set aws.psqlVolumeId=$PSQL_VOLUMEID
+
+CERTIFICATE_ARN=$(aws acm request-certificate --domain-name $FQDN --validation-method DNS --profile or --query "CertificateArn" --output text)
+
+DNS_RECORD_NAME=""
+DNS_RECORD_VALUE=""
+until [ -n "$DNS_RECORD_NAME" ] && [ "$DNS_RECORD_NAME" != "None" ] && [ -n "$DNS_RECORD_VALUE" ] && [ "$DNS_RECORD_VALUE" != "None" ]; do
+ echo "Waiting for ACM DNS validation record for $FQDN..."
+ DNS_RECORD_NAME=$(aws acm describe-certificate --certificate-arn $CERTIFICATE_ARN --profile or --query "Certificate.DomainValidationOptions[0].ResourceRecord.Name" --output text 2>/dev/null)
+ DNS_RECORD_VALUE=$(aws acm describe-certificate --certificate-arn $CERTIFICATE_ARN --profile or --query "Certificate.DomainValidationOptions[0].ResourceRecord.Value" --output text 2>/dev/null)
+ [ -n "$DNS_RECORD_NAME" ] && [ "$DNS_RECORD_NAME" != "None" ] && [ -n "$DNS_RECORD_VALUE" ] && [ "$DNS_RECORD_VALUE" != "None" ] && break
+ sleep 5
+done
+
+aws route53 change-resource-record-sets \
+ --hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \
+ --change-batch \
+ '{"Changes": [ { "Action": "UPSERT", "ResourceRecordSet": { "Name": "'$DNS_RECORD_NAME'", "Type": "CNAME", "TTL": 300, "ResourceRecords" : [ { "Value": "'$DNS_RECORD_VALUE'" } ] } } ]}' \
+ --profile dnschg
+
+helm install proxy $OR_KUBERNETES_PATH/proxy \
+ -f $OR_KUBERNETES_PATH/proxy/values-eks.yaml -f profiles/$OR_PROFILE/values-proxy-eks-load.yaml -f values-proxy-acm-load.yaml \
+ --set or.nameserver=$CLUSTER_DNS --set or.hostname=$FQDN \
+ --set-string 'service.http.annotations.service\.beta\.kubernetes\.io\/aws-load-balancer-ssl-cert'=$CERTIFICATE_ARN \
+ --set-string image.repository=$AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/proxy
+
+helm install postgresql $OR_KUBERNETES_PATH/postgresql -f $OR_KUBERNETES_PATH/postgresql/values-eks.yaml -f profiles/$OR_PROFILE/values-postgresql-eks-load.yaml
+
+# Waiting for the LB to be created
+# AWS LB Controller only creates an Network LB if there's a service
+
+CLUSTER_VPC_ID=$(aws eks describe-cluster --profile or --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text)
+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then
+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."
+ exit 1
+fi
+
+while ! aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing']" 2>/dev/null | grep '"Code": "active"'; do
+ echo "Waiting for load balancer to be created..."
+ sleep 10
+done
+
+DNS_NAME=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].DNSName | [0]")
+HOSTED_ZONE_ID=$(aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?VpcId=='$CLUSTER_VPC_ID' && Type=='network' && Scheme=='internet-facing'].CanonicalHostedZoneId | [0]")
+
+aws route53 change-resource-record-sets \
+ --hosted-zone-id /hostedzone/Z08751721JH0NB6LLCB4V \
+ --change-batch \
+ '{"Changes": [ { "Action": "UPSERT", "ResourceRecordSet": { "Name": "'$FQDN'", "Type": "A", "AliasTarget":{ "HostedZoneId": '$HOSTED_ZONE_ID',"DNSName": '$DNS_NAME',"EvaluateTargetHealth": false} } } ]}' \
+ --profile dnschg
+
+helm install keycloak $OR_KUBERNETES_PATH/keycloak -f $OR_KUBERNETES_PATH/keycloak/values-haproxy.yaml \
+ -f profiles/$OR_PROFILE/values-keycloak-eks-load.yaml --set-string or.hostname=$FQDN
+helm install manager $OR_KUBERNETES_PATH/manager -f $OR_KUBERNETES_PATH/manager/values-haproxy-eks.yaml \
+ -f profiles/$OR_PROFILE/values-manager-eks-load.yaml --set-string or.hostname=$FQDN \
+ --set-string image.repository=$AWS_DEVELOPERS_ACCOUNT_ID.dkr.ecr.eu-west-1.amazonaws.com/openremote/manager
+
+while ! dig +short $FQDN | grep -qE '^[0-9]'; do
+ echo "Waiting for DNS resolution..."
+ sleep 5
+done
+
+echo "Access the manager at https://$FQDN"
diff --git a/test/load2-eks/values-proxy-acm-load.yaml b/test/load2-eks/values-proxy-acm-load.yaml
new file mode 100644
index 000000000..34b63abbf
--- /dev/null
+++ b/test/load2-eks/values-proxy-acm-load.yaml
@@ -0,0 +1,16 @@
+service:
+ http:
+ annotations:
+ service.beta.kubernetes.io/aws-load-balancer-type: "external"
+ service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
+ service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
+
+ service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:eu-west-1:<openremote account id>:certificate/..."
+ service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443,8883"
+or:
+ haproxyConfig: /etc/haproxy/haproxy-edge-terminated-tls.cfg
+ httpsPort: 8080
+ mqttPort: 1883
+ env:
+ - name: DISABLE_ACME
+ value: "true"