Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Review services to expose metrics under k8s (#2122)

Eric Bariauxf4476918f13c
diff --git a/kubernetes/README.md b/kubernetes/README.md
index c91956047..01a46ce86 100644
--- a/kubernetes/README.md
+++ b/kubernetes/README.md
@@ -101,6 +101,15 @@ or:
value: "false"
```
+### Metrics
+
+Both the manager and HAProxy expose Prometheus metrics.
+By defaults the metrics are exposed on a dedicated ClusterIP service.
+The manager configuration has 2 different flags that related to metrics:
+- `or.metricsEnabled` indicating if the manager container exposes metrics
+- `service.metrics.enabled` indicating if a metrics service for the manager should be exposed
+ This is only effective if the manager exposes metrics i.e. both flags must be true for the service to be created.
+
### When using HAProxy
#### Accessing MQTT
diff --git a/kubernetes/eks-cleanup-haproxy.sh b/kubernetes/eks-cleanup-haproxy.sh
index 3f8e15714..928d109f5 100755
--- a/kubernetes/eks-cleanup-haproxy.sh
+++ b/kubernetes/eks-cleanup-haproxy.sh
@@ -18,6 +18,24 @@ helm uninstall keycloak
helm uninstall postgresql
helm uninstall proxy
+echo "Delete VPC peerings"
+
+CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)
+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then
+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."
+ exit 1
+fi
+VPC_PEERING_IDS=$(aws ec2 describe-vpc-peering-connections \
+ --filters "Name=accepter-vpc-info.vpc-id,Values=$CLUSTER_VPC_ID" "Name=status-code,Values=active" \
+ --query 'VpcPeeringConnections[*].VpcPeeringConnectionId' \
+ --profile or \
+ --output text)
+
+for peering_id in $VPC_PEERING_IDS; do
+ echo "Deleting peering $peering_id"
+ aws ec2 delete-vpc-peering-connection --vpc-peering-connection-id $peering_id --profile or
+done
+
# Give the AWS LB Controller time to delete the NLB after the Service is deleted
while aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?Type=='network']" 2>/dev/null | grep '"Code": "active"'; do
echo "Waiting for load balancer to be deleted..."
diff --git a/kubernetes/eks-cleanup.sh b/kubernetes/eks-cleanup.sh
index b4eae1951..85b4ef0c3 100755
--- a/kubernetes/eks-cleanup.sh
+++ b/kubernetes/eks-cleanup.sh
@@ -61,6 +61,24 @@ helm uninstall manager
helm uninstall keycloak
helm uninstall postgresql
+echo "Delete VPC peerings"
+
+CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)
+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then
+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."
+ exit 1
+fi
+VPC_PEERING_IDS=$(aws ec2 describe-vpc-peering-connections \
+ --filters "Name=accepter-vpc-info.vpc-id,Values=$CLUSTER_VPC_ID" "Name=status-code,Values=active" \
+ --query 'VpcPeeringConnections[*].VpcPeeringConnectionId' \
+ --profile or \
+ --output text)
+
+for peering_id in $VPC_PEERING_IDS; do
+ echo "Deleting peering $peering_id"
+ aws ec2 delete-vpc-peering-connection --vpc-peering-connection-id $peering_id --profile or
+done
+
# Give the AWS LB Controller time to delete the ALB and NLB after the Service is deleted
while aws elbv2 describe-load-balancers --profile or --query "LoadBalancers" --output text 2>/dev/null | grep -q .; do
echo "Waiting for load balancer to be deleted..."
diff --git a/kubernetes/manager/Chart.yaml b/kubernetes/manager/Chart.yaml
index b50600adc..f8480ae7a 100644
--- a/kubernetes/manager/Chart.yaml
+++ b/kubernetes/manager/Chart.yaml
@@ -6,4 +6,4 @@ type: application
version: 0.1.0
-appVersion: "1.7.0"
+appVersion: "1.8.0"
diff --git a/kubernetes/manager/templates/service.yaml b/kubernetes/manager/templates/service.yaml
index f9a6f9256..fff8a53fc 100644
--- a/kubernetes/manager/templates/service.yaml
+++ b/kubernetes/manager/templates/service.yaml
@@ -17,12 +17,6 @@ spec:
targetPort: 1883
protocol: TCP
{{- end }}
- {{- if .Values.or.metricsEnabled }}
- - name: metrics
- port: {{ .Values.or.metricsPort }}
- targetPort: {{ .Values.or.metricsPort }}
- protocol: TCP
- {{- end }}
selector:
{{- include "manager.selectorLabels" . | nindent 4 }}
---
@@ -71,3 +65,23 @@ spec:
targetPort: 1883
protocol: TCP
{{- end }}
+---
+{{- if and (.Values.or.metricsEnabled) (.Values.service.metrics.enabled) }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: manager-metrics
+ {{- with .Values.service.metrics.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ type: {{ .Values.service.metrics.type }}
+ ports:
+ - name: metrics
+ port: {{ .Values.service.metrics.port }}
+ targetPort: {{ .Values.or.metricsPort }}
+ protocol: TCP
+ selector:
+ {{- include "manager.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/kubernetes/manager/values-eks.yaml b/kubernetes/manager/values-eks.yaml
index 03707d163..0939beab3 100644
--- a/kubernetes/manager/values-eks.yaml
+++ b/kubernetes/manager/values-eks.yaml
@@ -41,6 +41,14 @@ service:
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:eu-west-1:463235666115:certificate/...
+ metrics:
+ enabled: true
+ type: LoadBalancer
+ port: 8405
+ annotations:
+ service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
+ service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+ service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
diff --git a/kubernetes/manager/values-haproxy-eks.yaml b/kubernetes/manager/values-haproxy-eks.yaml
index f36adcba2..d7b3c19e9 100644
--- a/kubernetes/manager/values-haproxy-eks.yaml
+++ b/kubernetes/manager/values-haproxy-eks.yaml
@@ -1,5 +1,14 @@
ingress:
enabled: false
+service:
+ metrics:
+ enabled: true
+ type: LoadBalancer
+ port: 8405
+ annotations:
+ service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
+ service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+ service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
affinity:
nodeAffinity:
diff --git a/kubernetes/manager/values.yaml b/kubernetes/manager/values.yaml
index b2a4cdecf..ef9650114 100644
--- a/kubernetes/manager/values.yaml
+++ b/kubernetes/manager/values.yaml
@@ -62,6 +62,12 @@ service:
mqtts:
enabled: false
annotations: {}
+ metrics:
+ enabled: true
+ # Service type, typically use ClusterIP for local deployment and LoadBalancer for cloud deployments
+ type: ClusterIP
+ port: 8405
+ annotations: {}
# Expose an Ingress for direct HTTP(S) access to the manager, disable when using HA Proxy
ingress:
diff --git a/kubernetes/proxy/templates/service.yaml b/kubernetes/proxy/templates/service.yaml
index 239ce8a1a..252ef8835 100644
--- a/kubernetes/proxy/templates/service.yaml
+++ b/kubernetes/proxy/templates/service.yaml
@@ -4,14 +4,14 @@ metadata:
name: {{ include "proxy.fullname" . }}
labels:
{{- include "proxy.labels" . | nindent 4 }}
- {{- with .Values.service.annotations }}
+ {{- with .Values.service.http.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
- type: {{ .Values.service.type }}
+ type: {{ .Values.service.http.type }}
ports:
- - port: {{ .Values.service.port }}
+ - port: {{ .Values.service.http.port }}
targetPort: {{ .Values.or.httpPort }}
protocol: TCP
name: http
@@ -23,9 +23,25 @@ spec:
targetPort: 8883
protocol: TCP
name: mqtts
- - port: 8404
+ selector:
+ {{- include "proxy.selectorLabels" . | nindent 4 }}
+---
+{{- if .Values.service.metrics.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: proxy-metrics
+ {{- with .Values.service.metrics.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ type: {{ .Values.service.metrics.type }}
+ ports:
+ - name: metrics
+ port: {{ .Values.service.metrics.port }}
targetPort: 8404
protocol: TCP
- name: metrics
selector:
{{- include "proxy.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/kubernetes/proxy/values-eks.yaml b/kubernetes/proxy/values-eks.yaml
index dad8438fd..edab6dceb 100644
--- a/kubernetes/proxy/values-eks.yaml
+++ b/kubernetes/proxy/values-eks.yaml
@@ -1,8 +1,17 @@
service:
- annotations:
- service.beta.kubernetes.io/aws-load-balancer-type: "external"
- service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
- service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
+ http:
+ annotations:
+ service.beta.kubernetes.io/aws-load-balancer-type: "external"
+ service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
+ service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
+ metrics:
+ enabled: true
+ type: LoadBalancer
+ port: 8404
+ annotations:
+ service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
+ service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+ service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
or:
hostname: "testmanager.openremote.app"
diff --git a/kubernetes/proxy/values.yaml b/kubernetes/proxy/values.yaml
index b63c10cd5..8c5a52b29 100644
--- a/kubernetes/proxy/values.yaml
+++ b/kubernetes/proxy/values.yaml
@@ -51,11 +51,18 @@ securityContext: {}
# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/
service:
- # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
- type: LoadBalancer
- # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
- port: 80
- annotations: {}
+ http:
+ # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types
+ type: LoadBalancer
+ # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports
+ port: 80
+ annotations: {}
+ metrics:
+ enabled: true
+ # Service type, typically use ClusterIP for local deployment and LoadBalancer for cloud deployments
+ type: ClusterIP
+ port: 8404
+ annotations: { }
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious