100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Update CloudFormation templates to send query-exporter metrics to CloudWatch (#3315)
Dennis Kuijsdf0f4c222c10
diff --git a/.ci_cd/aws/cloudformation-create-dashboard.yml b/.ci_cd/aws/cloudformation-create-dashboard.ymlindex e5acd7667..039d593a8 100644--- a/.ci_cd/aws/cloudformation-create-dashboard.yml+++ b/.ci_cd/aws/cloudformation-create-dashboard.yml@@ -291,6 +291,174 @@ Resources:}}}+ },+ {+ "type": "metric",+ "x": 0,+ "y": 18,+ "width": 8,+ "height": 6,+ "properties": {+ "metrics": [+ [ "CWAgent-Prometheus", "pg_connections_used", "InstanceName", "a", { "id": "m1", "yAxis": "left", "label": "DB Connections Used" } ],+ [ "CWAgent-Prometheus", "pg_connections_free", "InstanceName", "a", { "id": "m2", "yAxis": "left", "label": "DB Connections Free" } ]+ ],+ "view": "timeSeries",+ "stacked": true,+ "region": "eu-west-1",+ "stat": "Sum",+ "period": 300,+ "title": "PostgreSQL Connections",+ "yAxis": {+ "left": {+ "showUnits": false,+ "label": "Connections",+ "min": 0+ }+ }+ }+ },+ {+ "type": "metric",+ "x": 8,+ "y": 18,+ "width": 8,+ "height": 6,+ "properties": {+ "view": "timeSeries",+ "metrics": [+ [ "CWAgent-Prometheus", "pg_long_running_queries_count", "InstanceName", "a", { "id": "m1", "yAxis": "left", "label": "Long Running Queries" } ]+ ],+ "region": "eu-west-1",+ "stat": "Sum",+ "period": 300,+ "title": "Long Running Queries"+ }+ },+ {+ "type": "metric",+ "x": 4,+ "y": 24,+ "width": 4,+ "height": 6,+ "properties": {+ "sparkline": true,+ "view": "singleValue",+ "metrics": [+ [ "CWAgent-Prometheus", "pg_index_bloat_count", "InstanceName", "a", { "id": "m1", "yAxis": "left", "label": "Index Bloat Count" } ],+ [ "CWAgent-Prometheus", "pg_table_bloat_count", "InstanceName", "a", { "id": "m2", "yAxis": "left", "label": "Table Bloat Count" } ]+ ],+ "region": "eu-west-1",+ "stat": "Sum",+ "period": 300,+ "title": "Bloat Count"+ }+ },+ {+ "type": "metric",+ "x": 0,+ "y": 24,+ "width": 4,+ "height": 6,+ "properties": {+ "sparkline": true,+ "view": "singleValue",+ "metrics": [+ [ "CWAgent-Prometheus", "pg_datapoint_chunks_needing_compression", "InstanceName", "a", { "id": "m1", "yAxis": "left", "label": "Chunks Needing Compression" } ]+ ],+ "region": "eu-west-1",+ "stat": "Maximum",+ "period": 86400,+ "start": "-P1D",+ "end": "P0D",+ "title": "Chunks Needing Compression"+ }+ },+ {+ "type": "metric",+ "x": 16,+ "y": 18,+ "width": 8,+ "height": 6,+ "properties": {+ "view": "gauge",+ "metrics": [+ [ "CWAgent-Prometheus", "pg_cache_hit_percentage", "InstanceName", "a", { "id": "m1", "yAxis": "left", "label": "Cache Hit Percentage" } ],+ [ "CWAgent-Prometheus", "pg_datapoint_uncompressed_cache_hit_ratio", "InstanceName", "a", { "id": "m2", "yAxis": "left", "label": "Uncompressed Cache Hit Ratio" } ]+ ],+ "region": "eu-west-1",+ "stat": "Average",+ "period": 3600,+ "title": "Cache Hit",+ "yAxis": {+ "left": {+ "min": 95,+ "max": 100+ }+ }+ }+ },+ {+ "type": "metric",+ "x": 16,+ "y": 24,+ "width": 8,+ "height": 6,+ "properties": {+ "view": "timeSeries",+ "metrics": [+ [ "CWAgent-Prometheus", "pg_timescale_job_total_failures_total", "InstanceName", "a", "proc_name", "_timescaledb_functions.policy_compression", "job_id", "1000", { "id": "m1", "yAxis": "left", "label": "Total Failures" } ]+ ],+ "region": "eu-west-1",+ "stat": "Average",+ "period": 3600,+ "title": "Timescale Total Failures",+ "yAxis": {+ "left": {+ "showUnits": false,+ "label": "Count",+ "min": 0+ }+ }+ }+ },+ {+ "type": "metric",+ "x": 8,+ "y": 24,+ "width": 8,+ "height": 6,+ "properties": {+ "view": "timeSeries",+ "metrics": [+ [ "CWAgent-Prometheus", "pg_dead_tuple_percent", "InstanceName", "a", "table_name", "dashboard", { "id": "m1", "yAxis": "left", "label": "Dashboard" } ],+ [ "...", "user_asset_link", { "id": "m2", "yAxis": "left", "label": "User Asset Link" } ],+ [ "...", "global_ruleset", { "id": "m3", "yAxis": "left", "label": "Global Ruleset" } ],+ [ "...", "realm_ruleset", { "id": "m4", "yAxis": "left", "label": "Realm Ruleset" } ],+ [ "...", "syslog_event", { "id": "m5", "yAxis": "left", "label": "Syslog Event" } ],+ [ "...", "asset", { "id": "m6", "yAxis": "left", "label": "Asset" } ],+ [ "...", "flyway_schema_history", { "id": "m7", "yAxis": "left", "label": "Flyway Schema History" } ],+ [ "...", "asset_datapoint", { "id": "m8", "yAxis": "left", "label": "Asset Datapoint" } ],+ [ "...", "provisioning_config", { "id": "m9", "yAxis": "left", "label": "Provisioning Config" } ],+ [ "...", "notification", { "id": "m10", "yAxis": "left", "label": "Notification" } ],+ [ "...", "alarm_asset_link", { "id": "m11", "yAxis": "left", "label": "Alarm Asset Link" } ],+ [ "...", "asset_ruleset", { "id": "m12", "yAxis": "left", "label": "Asset Ruleset" } ],+ [ "...", "alarm", { "id": "m13", "yAxis": "left", "label": "Alarm" } ],+ [ "...", "gateway_connection", { "id": "m14", "yAxis": "left", "label": "Gateway Connection" } ],+ [ "...", "asset_predicted_datapoint", { "id": "m15", "yAxis": "left", "label": "Asset Predicted Datapoint" } ]+ ],+ "region": "eu-west-1",+ "stat": "Average",+ "period": 300,+ "title": "Dead Tuple",+ "yAxis": {+ "left": {+ "showUnits": false,+ "label": "Percent",+ "min": 0+ }+ }+ }}]}diff --git a/.ci_cd/aws/cloudformation-create-ec2.yml b/.ci_cd/aws/cloudformation-create-ec2.ymlindex a37b76963..ecb3955b4 100644--- a/.ci_cd/aws/cloudformation-create-ec2.yml+++ b/.ci_cd/aws/cloudformation-create-ec2.yml@@ -261,7 +261,7 @@ Resources:- |{"agent":{- "metrics_collection_interval": 300+ "metrics_collection_interval": 900},"metrics": {"append_dimensions": {@@ -270,9 +270,8 @@ Resources:"metrics_collected": {"mem": {"measurement": [- "mem_used_percent"- ],- "metrics_collection_interval": 900+ "used_percent"+ ]},"disk": {"drop_device": true,@@ -282,8 +281,7 @@ Resources:"resources": ["/","/var/lib/docker/volumes"- ],- "metrics_collection_interval": 900+ ]}}},@@ -319,7 +317,17 @@ Resources:"haproxy_server_status": "Count","haproxy_server_http_responses_total": "Count","haproxy_server_max_session_rate": "Count/Second",- "haproxy_server_total_time_average_seconds": "Seconds"+ "haproxy_server_total_time_average_seconds": "Seconds",+ "pg_connections_used": "Count",+ "pg_connections_free": "Count",+ "pg_long_running_queries_count": "Count",+ "pg_cache_hit_percentage": "Percent",+ "pg_datapoint_chunks_needing_compression": "Count",+ "pg_datapoint_uncompressed_cache_hit_ratio": "Count",+ "pg_timescale_job_total_failures_total": "Count",+ "pg_dead_tuple_percent": "Percent",+ "pg_table_bloat_count": "Count",+ "pg_index_bloat_count": "Count"},"metric_declaration": [{@@ -340,6 +348,61 @@ Resources:"^or_provisioning_seconds_max$"]},+ {+ "source_labels": [ "job" ],+ "label_matcher": "^query-exporter$",+ "dimensions": [+ [ "InstanceName" ]+ ],+ "metric_selectors": [+ "^pg_connections_used$",+ "^pg_connections_free$",+ "^pg_long_running_queries_count$",+ "^pg_table_bloat_count$",+ "^pg_index_bloat_count$"+ ]+ },+ {+ "source_labels": [ "job" ],+ "label_matcher": "^query-exporter$",+ "dimensions": [+ [ "InstanceName", "table_name" ]+ ],+ "metric_selectors": [+ "^pg_dead_tuple_percent$"+ ]+ },+ {+ "source_labels": [ "job" ],+ "label_matcher": "^query-exporter$",+ "dimensions": [+ [ "InstanceName", "job_id", "proc_name" ]+ ],+ "metric_selectors": [+ "^pg_timescale_job_total_failures_total$"+ ]+ },+ {+ "source_labels": [ "job" ],+ "label_matcher": "^query-exporter-hourly$",+ "dimensions": [+ [ "InstanceName" ]+ ],+ "metric_selectors": [+ "^pg_cache_hit_percentage$",+ "^pg_datapoint_uncompressed_cache_hit_ratio$"+ ]+ },+ {+ "source_labels": [ "job" ],+ "label_matcher": "^query-exporter-daily$",+ "dimensions": [+ [ "InstanceName" ]+ ],+ "metric_selectors": [+ "^pg_datapoint_chunks_needing_compression$"+ ]+ },{"source_labels": [ "job", "source" ],"label_matcher": "manager;.*",@@ -455,6 +518,60 @@ Resources:regex: '(.*)'replacement: '${Host}'target_label: InstanceName++ - job_name: query-exporter+ sample_limit: 10000+ scrape_interval: 5m+ static_configs:+ - targets:+ - localhost:8406+ - labels:+ InstanceName: '${Host}' # DOESN'T WORK https://github.com/aws/amazon-cloudwatch-agent/issues/751+ metric_relabel_configs:+ - action: keep+ source_labels: [__name__]+ regex: '(pg_connections_used|pg_connections_free|pg_long_running_queries_count|pg_table_bloat_count|pg_index_bloat_count|pg_dead_tuple_percent|pg_timescale_job_total_failures_total)'+ - action: replace # THIS IS A WORKAROUND FOR BROKEN LABELS ABOVE+ source_labels: [ InstanceName ]+ regex: '(.*)'+ replacement: '${Host}'+ target_label: InstanceName++ - job_name: query-exporter-hourly+ sample_limit: 10000+ scrape_interval: 1h+ static_configs:+ - targets:+ - localhost:8406+ - labels:+ InstanceName: '${Host}' # DOESN'T WORK https://github.com/aws/amazon-cloudwatch-agent/issues/751+ metric_relabel_configs:+ - action: keep+ source_labels: [__name__]+ regex: '(pg_cache_hit_percentage|pg_datapoint_uncompressed_cache_hit_ratio)'+ - action: replace # THIS IS A WORKAROUND FOR BROKEN LABELS ABOVE+ source_labels: [ InstanceName ]+ regex: '(.*)'+ replacement: '${Host}'+ target_label: InstanceName++ - job_name: query-exporter-daily+ sample_limit: 10000+ scrape_interval: 1d+ static_configs:+ - targets:+ - localhost:8406+ - labels:+ InstanceName: '${Host}' # DOESN'T WORK https://github.com/aws/amazon-cloudwatch-agent/issues/751+ metric_relabel_configs:+ - action: keep+ source_labels: [__name__]+ regex: '(pg_datapoint_chunks_needing_compression)'+ - action: replace # THIS IS A WORKAROUND FOR BROKEN LABELS ABOVE+ source_labels: [ InstanceName ]+ regex: '(.*)'+ replacement: '${Host}'+ target_label: InstanceName- "# Prometheus scrape config file"daily_cleanup:files:@@ -560,7 +677,7 @@ Resources:if [ -n "${DNSHostedZoneRoleArn}" ]; thenecho "Assuming role '${DNSHostedZoneRoleArn}' for DNS A record update"- # Assume DNS role in TLD account+ # Assume DNS role in TLD accountCREDENTIALS=$(aws sts assume-role --role-arn ${DNSHostedZoneRoleArn} --role-session-name ${Host}-dns)if [ $? -ne 0 ]; thenecho "Failed to get assume role credentials"@@ -745,7 +862,7 @@ Resources:fi02_efs_dns:command: !Sub |- # Output EFS_DNS to /etc/environment+ # Output EFS_DNS to /etc/environmentif [ -n "${EFSDNS}" ]; thenecho "EFS Filesystem DNS: ${EFSDNS}"echo "EFS_DNS=${EFSDNS}" >> /etc/environment@@ -1057,7 +1174,7 @@ Resources:Namespace: CWAgentMetricName: mem_used_percentStatistic: Average- Period: 600+ Period: 900EvaluationPeriods: 3ComparisonOperator: GreaterThanThresholdThreshold: 90@@ -1148,3 +1265,197 @@ Resources:- !Ref SnsTopicOKActions:- !Ref SnsTopic++ PostgreSQLConnectionsAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ AlarmDescription: Available PostgreSQL Connections less then 15% of limit+ Metrics:+ - Id: e1+ ReturnData: true+ Expression: IF(m1 < (m1 + m2) * 0.15, 1, 0)+ - Id: m1+ ReturnData: false+ MetricStat:+ Metric:+ Namespace: CWAgent-Prometheus+ MetricName: pg_connections_free+ Dimensions:+ - Name: InstanceName+ Value: !Ref Host+ Period: 300+ Stat: Average+ - Id: m2+ ReturnData: false+ MetricStat:+ Metric:+ Namespace: CWAgent-Prometheus+ MetricName: pg_connections_used+ Dimensions:+ - Name: InstanceName+ Value: !Ref Host+ Period: 300+ Stat: Average+ EvaluationPeriods: 1+ ComparisonOperator: GreaterThanThreshold+ Threshold: 0+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic++ LongRunningQueriesAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ Namespace: CWAgent-Prometheus+ MetricName: pg_long_running_queries_count+ Statistic: Average+ Period: 300+ EvaluationPeriods: 1+ ComparisonOperator: GreaterThanThreshold+ Threshold: 0+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic+ Dimensions:+ - Name: InstanceName+ Value: !Ref Host++ CacheHitAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ Namespace: CWAgent-Prometheus+ MetricName: pg_cache_hit_percentage+ Statistic: Average+ Period: 3600+ EvaluationPeriods: 1+ ComparisonOperator: LessThanOrEqualToThreshold+ Threshold: 95+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic+ Dimensions:+ - Name: InstanceName+ Value: !Ref Host++ UncompressedCacheHitRatioAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ Namespace: CWAgent-Prometheus+ MetricName: pg_datapoint_uncompressed_cache_hit_ratio+ Statistic: Average+ Period: 3600+ EvaluationPeriods: 1+ ComparisonOperator: LessThanOrEqualToThreshold+ Threshold: 95+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic+ Dimensions:+ - Name: InstanceName+ Value: !Ref Host++ ChunksNeedsCompressionAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ Namespace: CWAgent-Prometheus+ MetricName: pg_datapoint_chunks_needing_compression+ Statistic: Average+ Period: 86400+ EvaluationPeriods: 1+ ComparisonOperator: GreaterThanThreshold+ Threshold: 0+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic+ Dimensions:+ - Name: InstanceName+ Value: !Ref Host++ TableBloatCountAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ Namespace: CWAgent-Prometheus+ MetricName: pg_table_bloat_count+ Statistic: Average+ Period: 300+ EvaluationPeriods: 1+ ComparisonOperator: GreaterThanThreshold+ Threshold: 0+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic+ Dimensions:+ - Name: InstanceName+ Value: !Ref Host++ IndexBloatCountAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ Namespace: CWAgent-Prometheus+ MetricName: pg_index_bloat_count+ Statistic: Average+ Period: 300+ EvaluationPeriods: 1+ ComparisonOperator: GreaterThanThreshold+ Threshold: 0+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic+ Dimensions:+ - Name: InstanceName+ Value: !Ref Host++ DeadTupleAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ Metrics:+ - Id: q1+ Label: pg_dead_tuple_percent+ ReturnData: true+ Expression: SELECT AVG(pg_dead_tuple_percent) FROM SCHEMA("CWAgent-Prometheus", InstanceName,table_name) GROUP BY table_name ORDER BY AVG() DESC+ Period: 300+ EvaluationPeriods: 1+ ComparisonOperator: GreaterThanThreshold+ Threshold: 20+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic++ TimescaleTotalFailureAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ Namespace: CWAgent-Prometheus+ MetricName: pg_timescale_job_total_failures_total+ Statistic: Average+ Period: 3600+ EvaluationPeriods: 1+ ComparisonOperator: GreaterThanThreshold+ Threshold: 0+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic+ Dimensions:+ - Name: InstanceName+ Value: !Ref Host+ - Name: job_id+ Value: "1000"+ - Name: proc_name+ Value: "_timescaledb_functions.policy_compression"