100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Review services to expose metrics under k8s (#2122)
Eric Bariauxf4476918f13c
diff --git a/kubernetes/README.md b/kubernetes/README.mdindex c91956047..01a46ce86 100644--- a/kubernetes/README.md+++ b/kubernetes/README.md@@ -101,6 +101,15 @@ or:value: "false"```+### Metrics++Both the manager and HAProxy expose Prometheus metrics.+By defaults the metrics are exposed on a dedicated ClusterIP service.+The manager configuration has 2 different flags that related to metrics:+- `or.metricsEnabled` indicating if the manager container exposes metrics+- `service.metrics.enabled` indicating if a metrics service for the manager should be exposed+ This is only effective if the manager exposes metrics i.e. both flags must be true for the service to be created.+### When using HAProxy#### Accessing MQTTdiff --git a/kubernetes/eks-cleanup-haproxy.sh b/kubernetes/eks-cleanup-haproxy.shindex 3f8e15714..928d109f5 100755--- a/kubernetes/eks-cleanup-haproxy.sh+++ b/kubernetes/eks-cleanup-haproxy.sh@@ -18,6 +18,24 @@ helm uninstall keycloakhelm uninstall postgresqlhelm uninstall proxy+echo "Delete VPC peerings"++CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."+ exit 1+fi+VPC_PEERING_IDS=$(aws ec2 describe-vpc-peering-connections \+ --filters "Name=accepter-vpc-info.vpc-id,Values=$CLUSTER_VPC_ID" "Name=status-code,Values=active" \+ --query 'VpcPeeringConnections[*].VpcPeeringConnectionId' \+ --profile or \+ --output text)++for peering_id in $VPC_PEERING_IDS; do+ echo "Deleting peering $peering_id"+ aws ec2 delete-vpc-peering-connection --vpc-peering-connection-id $peering_id --profile or+done+# Give the AWS LB Controller time to delete the NLB after the Service is deletedwhile aws elbv2 describe-load-balancers --profile or --query "LoadBalancers[?Type=='network']" 2>/dev/null | grep '"Code": "active"'; doecho "Waiting for load balancer to be deleted..."diff --git a/kubernetes/eks-cleanup.sh b/kubernetes/eks-cleanup.shindex b4eae1951..85b4ef0c3 100755--- a/kubernetes/eks-cleanup.sh+++ b/kubernetes/eks-cleanup.sh@@ -61,6 +61,24 @@ helm uninstall managerhelm uninstall keycloakhelm uninstall postgresql+echo "Delete VPC peerings"++CLUSTER_VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query 'cluster.resourcesVpcConfig.vpcId' --output text --profile or)+if [ -z "$CLUSTER_VPC_ID" ] || [ "$CLUSTER_VPC_ID" = "None" ]; then+ echo "Error: Failed to retrieve VPC ID for cluster '$CLUSTER_NAME'. Aborting."+ exit 1+fi+VPC_PEERING_IDS=$(aws ec2 describe-vpc-peering-connections \+ --filters "Name=accepter-vpc-info.vpc-id,Values=$CLUSTER_VPC_ID" "Name=status-code,Values=active" \+ --query 'VpcPeeringConnections[*].VpcPeeringConnectionId' \+ --profile or \+ --output text)++for peering_id in $VPC_PEERING_IDS; do+ echo "Deleting peering $peering_id"+ aws ec2 delete-vpc-peering-connection --vpc-peering-connection-id $peering_id --profile or+done+# Give the AWS LB Controller time to delete the ALB and NLB after the Service is deletedwhile aws elbv2 describe-load-balancers --profile or --query "LoadBalancers" --output text 2>/dev/null | grep -q .; doecho "Waiting for load balancer to be deleted..."diff --git a/kubernetes/manager/Chart.yaml b/kubernetes/manager/Chart.yamlindex b50600adc..f8480ae7a 100644--- a/kubernetes/manager/Chart.yaml+++ b/kubernetes/manager/Chart.yaml@@ -6,4 +6,4 @@ type: applicationversion: 0.1.0-appVersion: "1.7.0"+appVersion: "1.8.0"diff --git a/kubernetes/manager/templates/service.yaml b/kubernetes/manager/templates/service.yamlindex f9a6f9256..fff8a53fc 100644--- a/kubernetes/manager/templates/service.yaml+++ b/kubernetes/manager/templates/service.yaml@@ -17,12 +17,6 @@ spec:targetPort: 1883protocol: TCP{{- end }}- {{- if .Values.or.metricsEnabled }}- - name: metrics- port: {{ .Values.or.metricsPort }}- targetPort: {{ .Values.or.metricsPort }}- protocol: TCP- {{- end }}selector:{{- include "manager.selectorLabels" . | nindent 4 }}---@@ -71,3 +65,23 @@ spec:targetPort: 1883protocol: TCP{{- end }}+---+{{- if and (.Values.or.metricsEnabled) (.Values.service.metrics.enabled) }}+apiVersion: v1+kind: Service+metadata:+ name: manager-metrics+ {{- with .Values.service.metrics.annotations }}+ annotations:+ {{- toYaml . | nindent 4 }}+ {{- end }}+spec:+ type: {{ .Values.service.metrics.type }}+ ports:+ - name: metrics+ port: {{ .Values.service.metrics.port }}+ targetPort: {{ .Values.or.metricsPort }}+ protocol: TCP+ selector:+ {{- include "manager.selectorLabels" . | nindent 4 }}+{{- end }}diff --git a/kubernetes/manager/values-eks.yaml b/kubernetes/manager/values-eks.yamlindex 03707d163..0939beab3 100644--- a/kubernetes/manager/values-eks.yaml+++ b/kubernetes/manager/values-eks.yaml@@ -41,6 +41,14 @@ service:service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:eu-west-1:463235666115:certificate/...+ metrics:+ enabled: true+ type: LoadBalancer+ port: 8405+ annotations:+ service.beta.kubernetes.io/aws-load-balancer-type: "nlb"+ service.beta.kubernetes.io/aws-load-balancer-internal: "true"+ service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"affinity:nodeAffinity:requiredDuringSchedulingIgnoredDuringExecution:diff --git a/kubernetes/manager/values-haproxy-eks.yaml b/kubernetes/manager/values-haproxy-eks.yamlindex f36adcba2..d7b3c19e9 100644--- a/kubernetes/manager/values-haproxy-eks.yaml+++ b/kubernetes/manager/values-haproxy-eks.yaml@@ -1,5 +1,14 @@ingress:enabled: false+service:+ metrics:+ enabled: true+ type: LoadBalancer+ port: 8405+ annotations:+ service.beta.kubernetes.io/aws-load-balancer-type: "nlb"+ service.beta.kubernetes.io/aws-load-balancer-internal: "true"+ service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"affinity:nodeAffinity:diff --git a/kubernetes/manager/values.yaml b/kubernetes/manager/values.yamlindex b2a4cdecf..ef9650114 100644--- a/kubernetes/manager/values.yaml+++ b/kubernetes/manager/values.yaml@@ -62,6 +62,12 @@ service:mqtts:enabled: falseannotations: {}+ metrics:+ enabled: true+ # Service type, typically use ClusterIP for local deployment and LoadBalancer for cloud deployments+ type: ClusterIP+ port: 8405+ annotations: {}# Expose an Ingress for direct HTTP(S) access to the manager, disable when using HA Proxyingress:diff --git a/kubernetes/proxy/templates/service.yaml b/kubernetes/proxy/templates/service.yamlindex 239ce8a1a..252ef8835 100644--- a/kubernetes/proxy/templates/service.yaml+++ b/kubernetes/proxy/templates/service.yaml@@ -4,14 +4,14 @@ metadata:name: {{ include "proxy.fullname" . }}labels:{{- include "proxy.labels" . | nindent 4 }}- {{- with .Values.service.annotations }}+ {{- with .Values.service.http.annotations }}annotations:{{- toYaml . | nindent 4 }}{{- end }}spec:- type: {{ .Values.service.type }}+ type: {{ .Values.service.http.type }}ports:- - port: {{ .Values.service.port }}+ - port: {{ .Values.service.http.port }}targetPort: {{ .Values.or.httpPort }}protocol: TCPname: http@@ -23,9 +23,25 @@ spec:targetPort: 8883protocol: TCPname: mqtts- - port: 8404+ selector:+ {{- include "proxy.selectorLabels" . | nindent 4 }}+---+{{- if .Values.service.metrics.enabled }}+apiVersion: v1+kind: Service+metadata:+ name: proxy-metrics+ {{- with .Values.service.metrics.annotations }}+ annotations:+ {{- toYaml . | nindent 4 }}+ {{- end }}+spec:+ type: {{ .Values.service.metrics.type }}+ ports:+ - name: metrics+ port: {{ .Values.service.metrics.port }}targetPort: 8404protocol: TCP- name: metricsselector:{{- include "proxy.selectorLabels" . | nindent 4 }}+{{- end }}diff --git a/kubernetes/proxy/values-eks.yaml b/kubernetes/proxy/values-eks.yamlindex dad8438fd..edab6dceb 100644--- a/kubernetes/proxy/values-eks.yaml+++ b/kubernetes/proxy/values-eks.yaml@@ -1,8 +1,17 @@service:- annotations:- service.beta.kubernetes.io/aws-load-balancer-type: "external"- service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"- service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"+ http:+ annotations:+ service.beta.kubernetes.io/aws-load-balancer-type: "external"+ service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"+ service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"+ metrics:+ enabled: true+ type: LoadBalancer+ port: 8404+ annotations:+ service.beta.kubernetes.io/aws-load-balancer-type: "nlb"+ service.beta.kubernetes.io/aws-load-balancer-internal: "true"+ service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"or:hostname: "testmanager.openremote.app"diff --git a/kubernetes/proxy/values.yaml b/kubernetes/proxy/values.yamlindex b63c10cd5..8c5a52b29 100644--- a/kubernetes/proxy/values.yaml+++ b/kubernetes/proxy/values.yaml@@ -51,11 +51,18 @@ securityContext: {}# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/service:- # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types- type: LoadBalancer- # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports- port: 80- annotations: {}+ http:+ # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types+ type: LoadBalancer+ # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports+ port: 80+ annotations: {}+ metrics:+ enabled: true+ # Service type, typically use ClusterIP for local deployment and LoadBalancer for cloud deployments+ type: ClusterIP+ port: 8404+ annotations: { }resources: {}# We usually recommend not to specify default resources and to leave this as a conscious