100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
CI/CD: Separate EBS volume for docker volumes (#1790)
Dennis Kuijsac941b3d5808
diff --git a/.ci_cd/aws/cloudformation-create-dashboard.yml b/.ci_cd/aws/cloudformation-create-dashboard.ymlindex 31bd590b0..614eb656e 100644--- a/.ci_cd/aws/cloudformation-create-dashboard.yml+++ b/.ci_cd/aws/cloudformation-create-dashboard.yml@@ -1,12 +1,10 @@# ----------------------------------------------------------------------------------------------------------------------# Provisions a cloudwatch dashboard with variables to select which instance to show metrics for, should be one per+# Provisions a CloudWatch dashboard with variables to select which instance to show metrics for, should be one per# region.# ---------------------------------------------------------------------------------------------------------------------AWSTemplateFormatVersion: '2010-09-09'Description: 'AWS Linux 2, docker compose, nfs support, cron support, cloud watch logging, route53 updating'-Resources:-CloudWatchDashboard:Type: AWS::CloudWatch::DashboardProperties:@@ -241,7 +239,8 @@ Resources:"properties": {"metrics": [[ "CWAgent", "mem_used_percent", "InstanceId", "a", { "label": "Memory Usage %" } ],- [ "CWAgent", "disk_used_percent", ".", ".", "path", "/", "fstype", "xfs", { "label": "Disk Usage %" } ]+ [ "CWAgent", "disk_used_percent", ".", ".", "path", "/", "fstype", "xfs", { "label": "Root Disk Usage %" } ],+ [ "CWAgent", "disk_used_percent", ".", ".", "path", "/var/lib/docker/volumes", "fstype", "xfs", { "label": "Data Disk Usage %" } ]],"view": "timeSeries","stacked": false,diff --git a/.ci_cd/aws/cloudformation-create-ec2.yml b/.ci_cd/aws/cloudformation-create-ec2.ymlindex 54387bd7c..c8a76e79f 100644--- a/.ci_cd/aws/cloudformation-create-ec2.yml+++ b/.ci_cd/aws/cloudformation-create-ec2.yml@@ -77,6 +77,13 @@ Parameters:Description: Role ARN that can be assumed for updating DNS A record (if Hosted Zone in another account)Type: StringDefault: ""+ DLMExecutionRoleArn:+ Description: Role ARN for executing the DLM operations+ Type: String+ SnapshotId:+ Description: Snapshot ID to create the EBS data volume based of an existing snapshot+ Type: String+ Default: ""ElasticIP:Description: Assign elastic IP addressType: String@@ -91,10 +98,14 @@ Parameters:- true- falseDefault: false- DiskSize:+ RootDiskSize:Description: Root EBS volume size GBType: NumberDefault: 30+ DataDiskSize:+ Description: Amount of storage you want to provision for this EBS data volume+ Type: Number+ Default: 16HealthChecks:Description: Create health checksType: String@@ -128,6 +139,7 @@ Conditions:AssignElasticIP: !Equals [!Ref ElasticIP, "true"]MetricsEnabled: !Equals [!Ref Metrics, "true"]CreateHealthChecks: !Equals [!Ref HealthChecks, "true"]+ SnapshotProvided: !Not [!Equals [!Ref SnapshotId, ""]]UseAMIID: !Equals [!Ref ImageId, ""]Resources:@@ -216,9 +228,9 @@ Resources:default:- setupCfnHup- create_swapfile+ - docker_setup- cloudwatch_config- restart_amazon-cloudwatch-agent- - docker_setup- cron_setup- env_vars- dns_updater@@ -269,7 +281,8 @@ Resources:"used_percent"],"resources": [- "/"+ "/",+ "/var/lib/docker/volumes"],"metrics_collection_interval": 900}@@ -447,7 +460,7 @@ Resources:daily_cleanup:files:"/usr/local/bin/daily-cleanup":- content: !Sub |+ content: |#!/bin/bashjournalctl --vacuum-time=7d"/etc/cron.d/cleanup":@@ -766,7 +779,7 @@ Resources:docker:enabled: "true"ensureRunning: "true"-+cron_setup:packages:yum:@@ -828,7 +841,7 @@ Resources:BlockDeviceMappings:- DeviceName: /dev/xvdaEbs:- VolumeSize: !Ref DiskSize+ VolumeSize: !Ref RootDiskSizeVolumeType: gp3DeleteOnTermination: trueIpv6AddressCount: 1@@ -866,7 +879,51 @@ Resources:InstanceId: !Ref EC2InstanceEIP: !Ref IPAddress- # Telemetry and Alarms+ EBSDataVolume:+ Type: AWS::EC2::Volume+ Properties:+ AvailabilityZone: !GetAtt EC2Instance.AvailabilityZone+ Size: !Ref DataDiskSize+ VolumeType: gp3+ SnapshotId: !If [SnapshotProvided, !Ref SnapshotId, !Ref 'AWS::NoValue']+ Tags:+ - Key: Name+ Value: !Sub ${Host}-data+ - Key: Type+ Value: !Sub ${Host}-or-data-not-in-use+ DeletionPolicy: Snapshot++ # DLM Policy for automatic snapshot creation++ DLMPolicy:+ Type: AWS::DLM::LifecyclePolicy+ DependsOn: EBSDataVolume+ Properties:+ ExecutionRoleArn: !Ref DLMExecutionRoleArn+ Description: 'or-data'+ State: ENABLED+ PolicyDetails:+ PolicyLanguage: STANDARD+ PolicyType: EBS_SNAPSHOT_MANAGEMENT+ TargetTags:+ - Key: Type+ Value: !Sub ${Host}-or-data-in-use+ ResourceTypes:+ - VOLUME+ Schedules:+ - Name: Daily Backup+ CreateRule:+ Interval: 24+ IntervalUnit: HOURS+ Times:+ - '05:00'+ RetainRule:+ Count: 5+ Tags:+ - Key: Name+ Value: !Ref Host++ # Telemetry and Alarms.EmailSubscription:Condition: CreateHealthChecks@@ -941,7 +998,7 @@ Resources:- Name: InstanceIdValue: !Ref EC2Instance- DiskUtilizationAlarm:+ RootDiskUtilizationAlarm:Type: AWS::CloudWatch::AlarmCondition: MetricsEnabledProperties:@@ -964,6 +1021,29 @@ Resources:- Name: fstypeValue: xfs+ DataDiskUtilizationAlarm:+ Type: AWS::CloudWatch::Alarm+ Condition: MetricsEnabled+ Properties:+ Namespace: CWAgent+ MetricName: disk_used_percent+ Statistic: Average+ Period: 3600+ EvaluationPeriods: 1+ ComparisonOperator: GreaterThanThreshold+ Threshold: 90+ AlarmActions:+ - !Ref SnsTopic+ OKActions:+ - !Ref SnsTopic+ Dimensions:+ - Name: InstanceId+ Value: !Ref EC2Instance+ - Name: path+ Value: /var/lib/docker/volumes+ - Name: fstype+ Value: xfs+MemoryUtilizationAlarm:Type: AWS::CloudWatch::AlarmCondition: MetricsEnableddiff --git a/.ci_cd/aws/cloudformation-create-efs.yml b/.ci_cd/aws/cloudformation-create-efs.ymlindex 9044154be..af77f40d3 100644--- a/.ci_cd/aws/cloudformation-create-efs.yml+++ b/.ci_cd/aws/cloudformation-create-efs.yml@@ -2,9 +2,8 @@# Provisions an EFS using the specified VPC and subnets; creates a security group called# `or-map`; but doesn't pre-configure ingress rules on this security group.# ----------------------------------------------------------------------------------------------AWSTemplateFormatVersion: 2010-09-09-Description: Hosts EFS-+AWSTemplateFormatVersion: '2010-09-09'+Description: 'Hosts EFS'Parameters:VpcId:Type: String@@ -12,15 +11,15 @@ Parameters:SubnetID1:Type: AWS::EC2::Subnet::Id- Description: Mount target 1 subnet ID+ Description: Mount target 1 Subnet IDSubnetID2:Type: AWS::EC2::Subnet::Id- Description: Mount target 2 subnet ID+ Description: Mount target 2 Subnet IDSubnetID3:Type: AWS::EC2::Subnet::Id- Description: Mount target 3 subnet ID+ Description: Mount target 3 Subnet IDConditions:CreateTarget1: !Not [!Equals [!Ref SubnetID1, ""]]@@ -61,8 +60,6 @@ Resources:# StringEquals:# 'aws:PrincipalOrgID': o-dhb7rpaban--MountTargetResource1:Type: AWS::EFS::MountTargetCondition: CreateTarget1diff --git a/.ci_cd/aws/cloudformation-create-smtp-user.yml b/.ci_cd/aws/cloudformation-create-smtp-user.ymlindex 4bd650840..601aa5733 100644--- a/.ci_cd/aws/cloudformation-create-smtp-user.yml+++ b/.ci_cd/aws/cloudformation-create-smtp-user.yml@@ -12,7 +12,6 @@ Parameters:#AllowedPattern: "^\S{4,}$"Resources:-SMTPUser:Type: AWS::IAM::UserProperties:diff --git a/.ci_cd/aws/cloudformation-create-ssm-documents.yml b/.ci_cd/aws/cloudformation-create-ssm-documents.ymlnew file mode 100644index 000000000..b88d49923--- /dev/null+++ b/.ci_cd/aws/cloudformation-create-ssm-documents.yml@@ -0,0 +1,498 @@+# ---------------------------------------------------------------------------------------------------------------------+#+# CloudFormation Template for creating SSM Documents for attaching, detaching and replacing an EBS data volume+#+# ---------------------------------------------------------------------------------------------------------------------++AWSTemplateFormatVersion: '2010-09-09'+Description: 'SSM documents for attaching, detaching and replacing an EBS data volume'+Resources:+ SSMMountVolumeDocument:+ Type: AWS::SSM::Document+ Properties:+ DocumentType: Command+ DocumentFormat: YAML+ TargetType: /AWS::EC2::Instance+ Name: mount_volume+ Content:+ schemaVersion: '2.2'+ description: 'Script for mounting an EBS data volume'+ parameters:+ DeviceName:+ type: String+ description: '(Required) Specify the Device name where the volume should be mounted on'+ allowedPattern: '^/dev/sd[b-z]$'+ mainSteps:+ # Mount the specified EBS data volume to the instance+ - name: MountVolume+ action: aws:runShellScript+ inputs:+ runCommand:+ - |+ FILESYSTEM=$(blkid -o value -s TYPE {{ DeviceName }})+ if [ -z "$FILESYSTEM" ]; then+ mkfs -t xfs {{ DeviceName }}+ mount {{ DeviceName }} /var/lib/docker/volumes+ else+ mount {{ DeviceName }} /var/lib/docker/volumes+ fi+ # Add the specified EBS data volume to the file system table+ - name: AddFileSystemEntry+ action: aws:runShellScript+ inputs:+ runCommand:+ - |+ UUID=$(blkid -o value -s UUID {{ DeviceName }})+ if [ -n "$UUID" ]; then+ cp /etc/fstab /etc/fstab.orig+ echo "UUID=$UUID /var/lib/docker/volumes xfs defaults,nofail 0 2" >> /etc/fstab+ else+ echo "Failed to add /etc/fstab entry .. UUID is not found"+ exit 1+ fi+ # Start the Docker service and socket+ - name: StartDocker+ action: aws:runShellScript+ inputs:+ runCommand:+ - systemctl start docker.socket docker.service++ SSMUmountVolumeDocument:+ Type: AWS::SSM::Document+ Properties:+ DocumentType: Command+ DocumentFormat: YAML+ TargetType: /AWS::EC2::Instance+ Name: umount_volume+ Content:+ schemaVersion: '2.2'+ description: 'Script for umounting an EBS data volume'+ parameters:+ DeviceName:+ type: String+ description: '(Required) Specify the Device name where the volume is mounted on'+ allowedPattern: '^/dev/sd[b-z]$'+ mainSteps:+ # Stop the Docker service and socket+ - name: StopDocker+ action: aws:runShellScript+ inputs:+ runCommand:+ - systemctl stop docker.socket docker.service+ # Remove the specified EBS data volume from the file systems table+ - name: RemoveFileSystemEntry+ action: aws:runShellScript+ inputs:+ runCommand:+ - |+ UUID=$(blkid -o value -s UUID {{ DeviceName }})+ if [ -n "$UUID" ]; then+ cp /etc/fstab /etc/fstab.orig+ sed -i '/UUID='$UUID'/d' /etc/fstab+ else+ echo "Failed to remove /etc/fstab entry .. UUID is not found"+ exit 1+ fi+ # Umount the specified EBS data volume+ - name: UmountVolume+ action: aws:runShellScript+ inputs:+ runCommand:+ - |+ MOUNT=$(findmnt -S {{ DeviceName }})+ if [ -n "$MOUNT" ]; then+ umount {{ DeviceName }}+ else+ echo "Device not mounted .. Skipping step"+ fi++ SSMAttachVolumeDocument:+ Type: AWS::SSM::Document+ Properties:+ DocumentType: Automation+ DocumentFormat: YAML+ TargetType: /AWS::EC2::Instance+ Name: attach_volume+ Content:+ schemaVersion: '0.3'+ description: 'Script for attaching an EBS data volume'+ parameters:+ VolumeId:+ type: String+ description: '(Required) Specify the VolumeId of the volume that should be attached'+ allowedPattern: '^vol-[a-z0-9]{8,17}$'+ InstanceId:+ type: String+ description: '(Required) Specify the InstanceId of the instance where the volume should be attached'+ allowedPattern: '^i-[a-z0-9]{8,17}$'+ DeviceName:+ type: String+ description: '(Required) Specify the Device name where the volume should be mounted on'+ allowedPattern: '^/dev/sd[b-z]$'+ mainSteps:+ # Retrieve instance details to get the Host name.+ - name: GetInstanceDetails+ action: aws:executeAwsApi+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: DescribeInstances+ InstanceIds:+ - '{{ InstanceId }}'+ outputs:+ - Name: Host+ Selector: $.Reservations[0].Instances[0].Tags[?(@.Key == 'Name')].Value+ Type: String+ nextStep: AttachVolume+ # Attach EBS data volume to specified instance+ - name: AttachVolume+ action: aws:executeAwsApi+ timeoutSeconds: 120+ onFailure: Abort+ isCritical: true+ inputs:+ Service: ec2+ Api: AttachVolume+ VolumeId: '{{ VolumeId }}'+ InstanceId: '{{ InstanceId }}'+ Device: '{{ DeviceName }}'+ nextStep: WaitForVolumeAttachment+ # Wait until the EBS data volume is successfully attached+ - name: WaitForVolumeAttachment+ action: aws:waitForAwsResourceProperty+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: DescribeVolumes+ VolumeIds:+ - '{{ VolumeId }}'+ PropertySelector: '$.Volumes[0].Attachments[0].State'+ DesiredValues:+ - attached+ nextStep: MountVolume+ # Mount EBS data volume to specified instance+ - name: MountVolume+ action: aws:runCommand+ timeoutSeconds: 120+ onFailure: Abort+ isCritical: true+ inputs:+ DocumentName: mount_volume+ Parameters:+ DeviceName:+ - '{{ DeviceName }}'+ InstanceIds:+ - '{{ InstanceId }}'+ nextStep: WaitForVolumeMounting+ # Wait until the EBS data volume is successfully mounted+ - name: WaitForVolumeMounting+ action: aws:waitForAwsResourceProperty+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ssm+ Api: GetCommandInvocation+ CommandId: '{{ MountVolume.CommandId }}'+ InstanceId: '{{ InstanceId }}'+ PropertySelector: '$.StatusDetails'+ DesiredValues:+ - Success+ nextStep: UpdateTag+ # Change tag to ensure the EBS data volume is targeted by the DLM policy+ - name: UpdateTag+ action: aws:createTags+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ ResourceType: EC2+ ResourceIds:+ - '{{ VolumeId }}'+ Tags:+ - Key: Type+ Value: '{{ GetInstanceDetails.Host }}-or-data-in-use'++ SSMDetachVolumeDocument:+ Type: AWS::SSM::Document+ Properties:+ DocumentType: Automation+ DocumentFormat: YAML+ TargetType: /AWS::EC2::Instance+ Name: detach_volume+ Content:+ schemaVersion: '0.3'+ description: 'Script for detaching an EBS data volume'+ parameters:+ VolumeId:+ type: String+ description: '(Required) Specify the VolumeId of the volume that should be detached'+ allowedPattern: '^vol-[a-z0-9]{8,17}$'+ mainSteps:+ # Retrieve EBS data volume details to get the InstanceId and Devicename+ - name: GetVolumeDetails+ action: aws:executeAwsApi+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: DescribeVolumes+ VolumeIds:+ - '{{ VolumeId }}'+ outputs:+ - Name: InstanceId+ Selector: '$.Volumes[0].Attachments[0].InstanceId'+ Type: String+ - Name: DeviceName+ Selector: '$.Volumes[0].Attachments[0].Device'+ Type: String+ nextStep: GetInstanceDetails+ # Retrieve instance details to get the Hostname+ - name: GetInstanceDetails+ action: aws:executeAwsApi+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: DescribeInstances+ InstanceIds:+ - '{{ GetVolumeDetails.InstanceId }}'+ outputs:+ - Name: Host+ Selector: $.Reservations[0].Instances[0].Tags[?(@.Key == 'Name')].Value+ Type: String+ nextStep: UmountVolume+ # Execute the SSM command that umounts the specified EBS data volume+ - name: UmountVolume+ action: aws:runCommand+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ DocumentName: umount_volume+ Parameters:+ DeviceName:+ - '{{ GetVolumeDetails.DeviceName }}'+ InstanceIds:+ - '{{ GetVolumeDetails.InstanceId }}'+ nextStep: WaitForUmount+ # Wait until the SSM document for umounting the EBS data volume is successfully executed+ - name: WaitForUmount+ action: aws:waitForAwsResourceProperty+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ssm+ Api: GetCommandInvocation+ CommandId: '{{ UmountVolume.CommandId }}'+ InstanceId: '{{ GetVolumeDetails.InstanceId }}'+ PropertySelector: '$.StatusDetails'+ DesiredValues:+ - Success+ nextStep: DetachVolume+ # Detach the EBS data volume+ - name: DetachVolume+ action: aws:executeAwsApi+ timeoutSeconds: 120+ onFailure: Abort+ isCritical: true+ inputs:+ Service: ec2+ Api: DetachVolume+ VolumeId: '{{ VolumeId }}'+ nextStep: WaitForVolumeDetachment+ # Wait until the EBS data volume is succesfully detached+ - name: WaitForVolumeDetachment+ action: aws:waitForAwsResourceProperty+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: DescribeVolumes+ VolumeIds:+ - '{{ VolumeId }}'+ PropertySelector: '$.Volumes[0].State'+ DesiredValues:+ - available+ nextStep: UpdateTag+ # Change tag to ensure the volume is no longer targeted by the DLM policy+ - name: UpdateTag+ action: aws:createTags+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ ResourceType: EC2+ ResourceIds:+ - '{{ VolumeId }}'+ Tags:+ - Key: Type+ Value: '{{ GetInstanceDetails.Host }}-or-data-not-in-use'++ SSMReplaceVolumeDocument:+ Type: AWS::SSM::Document+ Properties:+ DocumentType: Automation+ DocumentFormat: YAML+ TargetType: /AWS::EC2::Instance+ Name: replace_volume+ Content:+ schemaVersion: '0.3'+ description: 'Script for replacing an EBS data volume with a specified snapshot'+ parameters:+ VolumeId:+ type: String+ description: '(Required) Specify the VolumeId of the volume that needs to be replaced'+ allowedPattern: '^vol-[a-z0-9]{8,17}$'+ SnapshotId:+ type: String+ description: '(Required) Specify the SnapshotId of the snapshot to be used for the new volume'+ allowedPattern: '^snap-[a-z0-9]{8,17}$'+ DeleteVolume:+ type: Boolean+ description: '(Optional) Choose whether you want to delete the current volume'+ default: false+ mainSteps:+ # Retrieve EBS data volume details to get the DeviceName and InstanceId+ - name: GetVolumeDetails+ action: aws:executeAwsApi+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: DescribeVolumes+ VolumeIds:+ - '{{ VolumeId }}'+ outputs:+ - Name: InstanceId+ Selector: '$.Volumes[0].Attachments[0].InstanceId'+ Type: String+ - Name: DeviceName+ Selector: '$.Volumes[0].Attachments[0].Device'+ Type: String+ nextStep: GetInstanceDetails+ # Retrieve instance details to get the AvailabilityZone and Hostname+ - name: GetInstanceDetails+ action: aws:executeAwsApi+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: DescribeInstances+ InstanceIds:+ - '{{ GetVolumeDetails.InstanceId }}'+ outputs:+ - Name: AvailabilityZone+ Selector: '$.Reservations[0].Instances[0].Placement.AvailabilityZone'+ Type: String+ - Name: Host+ Selector: $.Reservations[0].Instances[0].Tags[?(@.Key == 'Name')].Value+ Type: String+ nextStep: CreateVolume+ # Create new EBS data volume using the retrieved details and specified snapshot+ - name: CreateVolume+ action: aws:executeAwsApi+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: CreateVolume+ AvailabilityZone: '{{ GetInstanceDetails.AvailabilityZone }}'+ SnapshotId: '{{ SnapshotId }}'+ VolumeType: gp3+ TagSpecifications:+ - ResourceType: volume+ Tags:+ - Key: Name+ Value: '{{ GetInstanceDetails.Host }}-data'+ - Key: Type+ Value: '{{ GetInstanceDetails.Host }}-or-data-not-in-use'+ outputs:+ - Name: VolumeId+ Selector: '$.VolumeId'+ Type: String+ nextStep: WaitForVolumeCreation+ # Wait until the EBS data volume is succesfully created+ - name: WaitForVolumeCreation+ action: aws:waitForAwsResourceProperty+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: DescribeVolumes+ VolumeIds:+ - '{{ CreateVolume.VolumeId }}'+ PropertySelector: '$.Volumes[0].State'+ DesiredValues:+ - available+ nextStep: DetachVolume+ # Detach the current EBS data volume+ - name: DetachVolume+ action: aws:executeAutomation+ timeoutSeconds: 120+ onFailure: Abort+ isCritical: true+ inputs:+ DocumentName: detach_volume+ RuntimeParameters:+ VolumeId:+ - '{{ VolumeId }}'+ nextStep: WaitForVolumeDetachment+ # Wait until the current EBS data volume is succesfully detached+ - name: WaitForVolumeDetachment+ action: aws:waitForAwsResourceProperty+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ssm+ Api: GetAutomationExecution+ AutomationExecutionId: '{{ DetachVolume.ExecutionId }}'+ PropertySelector: '$.AutomationExecution.AutomationExecutionStatus'+ DesiredValues:+ - Success+ nextStep: AttachVolume+ # Attach the newly created EBS data volume+ - name: AttachVolume+ action: aws:executeAutomation+ timeoutSeconds: 120+ onFailure: Abort+ isCritical: true+ inputs:+ DocumentName: attach_volume+ RuntimeParameters:+ DeviceName:+ - '{{ GetVolumeDetails.DeviceName }}'+ VolumeId:+ - '{{ CreateVolume.VolumeId }}'+ InstanceId:+ - '{{ GetVolumeDetails.InstanceId }}'+ nextStep: WaitForVolumeAttachment+ # Wait until the newly created EBS data volume is succesfully attached+ - name: WaitForVolumeAttachment+ action: aws:waitForAwsResourceProperty+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ssm+ Api: GetAutomationExecution+ AutomationExecutionId: '{{ AttachVolume.ExecutionId }}'+ PropertySelector: '$.AutomationExecution.AutomationExecutionStatus'+ DesiredValues:+ - Success+ nextStep: ChooseVolumeDeletion+ # Checks whether the old EBS data volume should be kept+ - name: ChooseVolumeDeletion+ action: aws:branch+ inputs:+ Choices:+ - NextStep: DeleteVolume+ Variable: '{{ DeleteVolume }}'+ BooleanEquals: true+ isEnd: true+ # Delete old EBS data volume if DeleteVolume variable is equal to true+ - name: DeleteVolume+ action: aws:executeAwsApi+ timeoutSeconds: 120+ onFailure: Abort+ inputs:+ Service: ec2+ Api: DeleteVolume+ VolumeId: '{{ VolumeId }}'\ No newline at end of filediff --git a/.ci_cd/aws/cloudformation-create-vpc.yml b/.ci_cd/aws/cloudformation-create-vpc.ymlindex 745412265..e6615cce3 100644--- a/.ci_cd/aws/cloudformation-create-vpc.yml+++ b/.ci_cd/aws/cloudformation-create-vpc.yml@@ -3,9 +3,8 @@# optional SSH access security group and optional VPC peer accepter role or peering request.# ----------------------------------------------------------------------------------------------------------------------AWSTemplateFormatVersion: 2010-09-09+AWSTemplateFormatVersion: '2010-09-09'Description: 'Creates standard OR VPC, subnets, route tables and IGW; optionally create VPC Peering request or acceptor role'-Parameters:IPV4CIDR:Description: IPv4 CIDR for VPCdiff --git a/.ci_cd/aws/cloudformation-healthcheck-alarm.yml b/.ci_cd/aws/cloudformation-healthcheck-alarm.ymlindex f9a46b776..c2e91ede4 100644--- a/.ci_cd/aws/cloudformation-healthcheck-alarm.yml+++ b/.ci_cd/aws/cloudformation-healthcheck-alarm.yml@@ -3,14 +3,12 @@# ---------------------------------------------------------------------------------------------------------------------AWSTemplateFormatVersion: '2010-09-09'Description: 'Route 53 Healthcheck alarm'-Parameters:Host:Description: FQDN for hostType: StringResources:-HealthCheck:Type: AWS::Route53::HealthCheckProperties:diff --git a/.ci_cd/aws/provision_account.sh b/.ci_cd/aws/provision_account.shindex 0742325c2..51a83f096 100644--- a/.ci_cd/aws/provision_account.sh+++ b/.ci_cd/aws/provision_account.sh@@ -177,7 +177,7 @@ if [ "$VPCID" == 'None' ]; then# Create standard stack resources in specified accountSTACK_ID=$(aws cloudformation create-stack --capabilities CAPABILITY_NAMED_IAM --stack-name $STACK_NAME --template-body file://$TEMPLATE_PATH --parameters $PARAMS --output text $ACCOUNT_PROFILE)- # Wait for cloud formation stack status to be CREATE_*+ # Wait for CloudFormation stack status to be CREATE_*echo "Waiting for stack to be created"STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[?StackId=='$STACK_ID'].StackStatus" --output text $ACCOUNT_PROFILE 2>/dev/null)@@ -277,38 +277,100 @@ elseecho "PARENT_DNS_ZONE not set so no DNS configuration will be attempted"fi-# Provision default dashboard-if [ -f "${awsDir}cloudformation-create-dashboard.yml" ]; then- TEMPLATE_PATH="${awsDir}cloudformation-create-dashboard.yml"-elif [ -f ".ci_cd/aws/cloudformation-create-dashboard.yml" ]; then- TEMPLATE_PATH=".ci_cd/aws/cloudformation-create-dashboard.yml"-elif [ -f "openremote/.ci_cd/aws/cloudformation-create-dashboard.yml" ]; then- TEMPLATE_PATH="openremote/.ci_cd/aws/cloudformation-create-dashboard.yml"+STACK_NAME=or-ssm++# Provision SSM documents using CloudFormation (if stack doesn't already exist)+echo "Provisioning SSM Documents"+STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackStatus" --output text 2>/dev/null)++if [ -n "$STATUS" ] && [ "$STATUS" != 'DELETE_COMPLETE' ]; then+ echo "Stack already exists for this host .. Current status is '$STATUS'"+ STACK_ID=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackId" --output text 2>/dev/null)else- echo "Cannot determine location of cloudformation-create-dashboard.yml"- exit 1++ if [ -f "${awsDir}cloudformation-create-ssm-documents.yml" ]; then+ TEMPLATE_PATH="${awsDir}cloudformation-create-ssm-documents.yml"+ elif [ -f ".ci_cd/aws/cloudformation-create-ssm-documents.yml" ]; then+ TEMPLATE_PATH=".ci_cd/aws/cloudformation-create-ssm-documents.yml"+ elif [ -f "openremote/.ci_cd/aws/cloudformation-create-ssm-documents.yml" ]; then+ TEMPLATE_PATH="openremote/.ci_cd/aws/cloudformation-create-ssm-documents.yml"+ else+ echo "Cannot determine location of cloudformation-create-ssm-documents.yml"+ exit 1+ fi++ # Create SSM Documents for attaching, detaching and replacing an EBS data volume in specified account+ STACK_ID=$(aws cloudformation create-stack --capabilities CAPABILITY_NAMED_IAM --stack-name $STACK_NAME --template-body file://$TEMPLATE_PATH --output text $ACCOUNT_PROFILE)++ if [ $? -ne 0 ]; then+ echo "Create stack failed"+ exit 1+ fi++ # Wait for CloudFormation stack status to be CREATE_*+ echo "Waiting for stack to be created"+ STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackStatus" --output text $ACCOUNT_PROFILE 2>/dev/null)++ while [[ "$STATUS" == 'CREATE_IN_PROGRESS' ]]; do+ echo "Stack creation is still in progress .. Sleeping 30 seconds"+ sleep 30+ STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackStatus" --output text $ACCOUNT_PROFILE 2>/dev/null)+ done++ if [ "$STATUS" != 'CREATE_COMPLETE' ] && [ "$STATUS" != 'UPDATE_COMPLETE' ]; then+ echo "Stack creation has failed status is '$STATUS'"+ exit 1+ else+ echo "Stack creation is complete"+ fifiSTACK_NAME=or-dashboard-default-# Create dashboard in specified account-STACK_ID=$(aws cloudformation create-stack --capabilities CAPABILITY_NAMED_IAM --stack-name $STACK_NAME --template-body file://$TEMPLATE_PATH --output text $ACCOUNT_PROFILE)+# Provision CloudWatch Dashboard using CloudFormation (if stack doesn't already exist)+echo "Provisioning CloudWatch Dashboard"+STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackStatus" --output text 2>/dev/null)-# Wait for cloud formation stack status to be CREATE_*-echo "Waiting for stack to be created"-STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackStatus" --output text $ACCOUNT_PROFILE 2>/dev/null)+if [ -n "$STATUS" ] && [ "$STATUS" != 'DELETE_COMPLETE' ]; then+ echo "Stack already exists for this host .. Current status is '$STATUS'"+ STACK_ID=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackId" --output text 2>/dev/null)+else-while [[ "$STATUS" == 'CREATE_IN_PROGRESS' ]]; do- echo "Stack creation is still in progress .. Sleeping 30 seconds"- sleep 30+ if [ -f "${awsDir}cloudformation-create-dashboard.yml" ]; then+ TEMPLATE_PATH="${awsDir}cloudformation-create-dashboard.yml"+ elif [ -f ".ci_cd/aws/cloudformation-create-dashboard.yml" ]; then+ TEMPLATE_PATH=".ci_cd/aws/cloudformation-create-dashboard.yml"+ elif [ -f "openremote/.ci_cd/aws/cloudformation-create-dashboard.yml" ]; then+ TEMPLATE_PATH="openremote/.ci_cd/aws/cloudformation-create-dashboard.yml"+ else+ echo "Cannot determine location of cloudformation-create-dashboard.yml"+ exit 1+ fi++ # Create dashboard in specified account+ STACK_ID=$(aws cloudformation create-stack --capabilities CAPABILITY_NAMED_IAM --stack-name $STACK_NAME --template-body file://$TEMPLATE_PATH --output text $ACCOUNT_PROFILE)++ if [ $? -ne 0 ]; then+ echo "Create stack failed"+ exit 1+ fi++ # Wait for CloudFormation stack status to be CREATE_*+ echo "Waiting for stack to be created"STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackStatus" --output text $ACCOUNT_PROFILE 2>/dev/null)-done-if [ "$STATUS" != 'CREATE_COMPLETE' ] && [ "$STATUS" != 'UPDATE_COMPLETE' ]; then- echo "Stack creation has failed status is '$STATUS'" >&2- exit 1-else- echo "Stack creation is complete"+ while [[ "$STATUS" == 'CREATE_IN_PROGRESS' ]]; do+ echo "Stack creation is still in progress .. Sleeping 30 seconds"+ sleep 30+ STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackStatus" --output text $ACCOUNT_PROFILE 2>/dev/null)+ done++ if [ "$STATUS" != 'CREATE_COMPLETE' ] && [ "$STATUS" != 'UPDATE_COMPLETE' ]; then+ echo "Stack creation has failed status is '$STATUS'"+ exit 1+ else+ echo "Stack creation is complete"+ fifi# Update SSH Whitelist for this accountdiff --git a/.ci_cd/aws/provision_host.sh b/.ci_cd/aws/provision_host.shindex e07f1cf13..89176cde9 100644--- a/.ci_cd/aws/provision_host.sh+++ b/.ci_cd/aws/provision_host.sh@@ -18,11 +18,13 @@# 1 - AWS_ACCOUNT_NAME where resources should be created (defaults to callers account)# 2 - HOST FQDN for e.g. staging.demo.openremote.app# 3 - INSTANCE_TYPE EC2 instance type see cloud formation template parameter-# 4 - DISK_SIZE to use for created EBS root volume (GB)-# 5 - ELASTIC_IP if 'true' then create an elastic public IP for this host-# 6 - PROVISION_S3_BUCKET set to 'false' to not provision an S3 bucket for this host-# 7 - ENABLE_METRICS set to 'false' to not enable cloudwatch metrics for this instance-# 8 - WAIT_FOR_STACK if 'false' script will not wait until the cloud formation stack is running+# 4 - ROOT_DISK_SIZE to use for created EBS root volume (GB)+# 5 - DATA_DISK_SIZE to use for created EBS data volume (GB)+# 6 - SNAPSHOT_ID to use for creating the EBS data volume based off an existing snapshot+# 7 - ELASTIC_IP if 'true' then create an elastic public IP for this host+# 8 - PROVISION_S3_BUCKET set to 'false' to not provision an S3 bucket for this host+# 9 - ENABLE_METRICS set to 'false' to not enable cloudwatch metrics for this instance+# 10 - WAIT_FOR_STACK if 'false' script will not wait until the cloud formation stack is runningif [[ $BASH_SOURCE = */* ]]; thenawsDir=${BASH_SOURCE%/*}/@@ -33,11 +35,13 @@ fiAWS_ACCOUNT_NAME=${1,,}HOST=${2,,}INSTANCE_TYPE=${3,,}-DISK_SIZE=${4,,}-ELASTIC_IP=${5,,}-PROVISION_S3_BUCKET=${6,,}-ENABLE_METRICS=${7,,}-WAIT_FOR_STACK=${8,,}+ROOT_DISK_SIZE=${4,,}+DATA_DISK_SIZE=${5,,}+SNAPSHOT_ID=${6,,}+ELASTIC_IP=${7,,}+PROVISION_S3_BUCKET=${8,,}+ENABLE_METRICS=${9,,}+WAIT_FOR_STACK=${10,,}if [ -z "$HOST" ]; thenecho "Host must be set"@@ -68,8 +72,9 @@ fiSTACK_NAME=$(tr '.' '-' <<< "$HOST")SMTP_STACK_NAME="$STACK_NAME-smtp"HEALTH_STACK_NAME="$STACK_NAME-healthcheck"+SSM_STACK_NAME="$STACK_NAME-ssm-attach-detach-documents"-# Provision SMTP user using cloud formation (if stack doesn't already exist)+# Provision SMTP user using CloudFormation (if stack doesn't already exist)echo "Provisioning SMTP user"STATUS=$(aws cloudformation describe-stacks --stack-name $SMTP_STACK_NAME --query "Stacks[0].StackStatus" --output text 2>/dev/null)@@ -89,7 +94,7 @@ elseexit 1fi- #Configure parameters+ # Configure parametersPARAMS="ParameterKey=UserName,ParameterValue='$SMTP_STACK_NAME'"# Create standard stack resources in specified account@@ -101,7 +106,7 @@ elsefiif [ "$WAIT_FOR_STACK" != 'false' ]; then- # Wait for cloud formation stack status to be CREATE_*+ # Wait for CloudFormation stack status to be CREATE_*echo "Waiting for stack to be created"STATUS=$(aws cloudformation describe-stacks --stack-name $SMTP_STACK_NAME --query "Stacks[?StackId=='$STACK_ID'].StackStatus" --output text 2>/dev/null)@@ -127,6 +132,7 @@ if [ -n "$STATUS" ] && [ "$STATUS" != 'DELETE_COMPLETE' ]; thenecho "Stack already exists for this host '$HOST' current status is '$STATUS'"STACK_ID=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[0].StackId" --output text $ACCOUNT_PROFILE 2>/dev/null)else+# Configure parametersPARAMS="ParameterKey=Host,ParameterValue=$HOST"@@ -230,11 +236,16 @@ EOF# Look for EFS mount target in caller account for the same availability zone ID (no costs if within same AZ) - Don't use name as name to IDs vary between accountsEFS_ID=$(aws efs describe-file-systems --query "FileSystems[?Name=='or-map-efs'].FileSystemId" --output text)EFS_DNS=$(aws efs describe-mount-targets --file-system-id $EFS_ID --query "MountTargets[?AvailabilityZoneId=='$SUBNET_AZ'].IpAddress" --output text)++ # Get role to be assumed by DLM+ ROLE_ARN="arn:aws:iam::$AWS_ACCOUNT_ID:role/$AWS_ROLE_NAME-$AWS_REGION"PARAMS="$PARAMS ParameterKey=VpcId,ParameterValue=$VPCID"PARAMS="$PARAMS ParameterKey=SSHSecurityGroupId,ParameterValue=$SGID"PARAMS="$PARAMS ParameterKey=SubnetId,ParameterValue=$SUBNETID"PARAMS="$PARAMS ParameterKey=EFSDNS,ParameterValue=$EFS_DNS"+ PARAMS="$PARAMS ParameterKey=SnapshotId,ParameterValue=$SNAPSHOT_ID"+ PARAMS="$PARAMS ParameterKey=DLMExecutionRoleArn,ParameterValue=$ROLE_ARN"# Create standard stack resources in specified accountSTACK_ID=$(aws cloudformation create-stack --capabilities CAPABILITY_NAMED_IAM --stack-name $STACK_NAME --template-body file://$TEMPLATE_PATH --parameters $PARAMS --output text $ACCOUNT_PROFILE)@@ -247,7 +258,7 @@ EOFfiif [ "$WAIT_FOR_STACK" != 'false' ]; then- # Wait for cloud formation stack status to be CREATE_*+ # Wait for CloudFormation stack status to be CREATE_*echo "Waiting for stack to be created"STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query "Stacks[?StackId=='$STACK_ID'].StackStatus" --output text $ACCOUNT_PROFILE 2>/dev/null)@@ -265,6 +276,43 @@ if [ "$WAIT_FOR_STACK" != 'false' ]; thenfifi+# Attaching/Mounting EBS data volume (if volume is not already attached)+echo "Attaching/Mounting EBS data volume"+STATUS=$(aws ec2 describe-volumes --filters "Name=tag:Name,Values='$HOST-data'" --query "Volumes[?Tags[?Value=='$STACK_ID']].State" --output text $ACCOUNT_PROFILE 2>/dev/null)++if [ -n "$STATUS" ] && [ "$STATUS" != 'available' ]; then+ echo "EBS data volume is already attached or not available for this host '$HOST' current status is '$STATUS'"+else++ EBS_DEVICE_NAME="/dev/sdf" # Only change if you know what you are doing+ INSTANCE_ID=$(aws ec2 describe-instances --filters "Name=tag:Name,Values='$HOST'" --query "Reservations[].Instances[?Tags[?Value=='$STACK_ID']].InstanceId" --output text $ACCOUNT_PROFILE 2>/dev/null)+ VOLUME_ID=$(aws ec2 describe-volumes --filters "Name=tag:Name,Values='$HOST-data'" --query "Volumes[?Tags[?Value=='$STACK_ID']].VolumeId" --output text $ACCOUNT_PROFILE 2>/dev/null)++ PARAMS="InstanceId=$INSTANCE_ID,VolumeId=$VOLUME_ID,DeviceName=$EBS_DEVICE_NAME"++ EXECUTION_ID=$(aws ssm start-automation-execution --document-name attach_volume --parameters $PARAMS --output text $ACCOUNT_PROFILE)++ if [ $? -ne 0 ]; then+ echo "Attaching/Mounting EBS data volume failed"+ exit 1+ fi++ STATUS=$(aws ssm get-automation-execution --automation-execution-id $EXECUTION_ID --query "AutomationExecution.AutomationExecutionStatus" --output text $ACCOUNT_PROFILE 2>/dev/null)++ while [[ "$STATUS" == 'InProgress' ]]; do+ echo "Attaching/Mounting EBS data volume is still in progress .. Sleeping 30 seconds"+ sleep 30+ STATUS=$(aws ssm get-automation-execution --automation-execution-id $EXECUTION_ID --query "AutomationExecution.AutomationExecutionStatus" --output text $ACCOUNT_PROFILE 2>/dev/null)+ done++ if [ "$STATUS" != 'Success' ]; then+ echo "Attaching/Mounting EBS data volume has failed status is '$STATUS'"+ exit 1+ else+ echo "Attaching/Mounting EBS data volume is complete"+ fi+fi+# Provision S3 bucketif [ "$PROVISION_S3_BUCKET" != 'false' ]; thenecho "Provisioning S3 bucket for host '$HOST'"@@ -285,7 +333,7 @@ if [ "$PROVISION_S3_BUCKET" != 'false' ]; thenfifi-# Provision Route 53 healthcheck alarm cloud formation (if stack doesn't already exist) - must be in the us-east-1 region+# Provision Route 53 healthcheck alarm CloudFormation (if stack doesn't already exist) - must be in the us-east-1 regionecho "Provisioning Healthcheck Alarm"STATUS=$(aws cloudformation describe-stacks --stack-name $HEALTH_STACK_NAME --query "Stacks[0].StackStatus" --output text $ACCOUNT_PROFILE --region us-east-1 2>/dev/null)@@ -305,7 +353,7 @@ elseexit 1fi- #Configure parameters+ # Configure parametersPARAMS="ParameterKey=Host,ParameterValue='$HOST'"# Create standard stack resources in specified account in us-east-1 region@@ -317,7 +365,7 @@ elsefiif [ "$WAIT_FOR_STACK" != 'false' ]; then- # Wait for cloud formation stack status to be CREATE_*+ # Wait for CloudFormation stack status to be CREATE_*echo "Waiting for stack to be created"STATUS=$(aws cloudformation describe-stacks --stack-name $HEALTH_STACK_NAME --query "Stacks[?StackId=='$STACK_ID'].StackStatus" --output text $ACCOUNT_PROFILE --region us-east-1 2>/dev/null)diff --git a/.github/workflows/provision-host.yml b/.github/workflows/provision-host.ymlindex 0d8833c9a..1c0c2dfe3 100644--- a/.github/workflows/provision-host.yml+++ b/.github/workflows/provision-host.yml@@ -27,11 +27,20 @@ on:- m6g.large- m6g.xlargerequired: true- DISK_SIZE:+ ROOT_DISK_SIZE:description: 'Override EC2 root EBS volume size (GB)'type: stringdefault: '16'required: false+ DATA_DISK_SIZE:+ description: 'Override EC2 data EBS volume size (GB)'+ type: string+ default: '16'+ required: false+ SNAPSHOT_ID:+ description: 'Snapshot ID to create volume based off an existing snapshot'+ type: string+ required: falseENABLE_METRICS:description: 'Enable metrics'type: boolean@@ -72,12 +81,14 @@ jobs:if: ${{ steps.checkUserMember.outputs.isTeamMember == 'true' }}run: |chmod +x .ci_cd/aws/*- .ci_cd/aws/provision_host.sh "$ACCOUNT_NAME" "$HOST" "$INSTANCE_TYPE" "$DISK_SIZE" "$ELASTIC_IP" "$PROVISION_S3_BUCKET" "$ENABLE_METRICS"+ .ci_cd/aws/provision_host.sh "$ACCOUNT_NAME" "$HOST" "$INSTANCE_TYPE" "$ROOT_DISK_SIZE" "$DATA_DISK_SIZE" "$SNAPSHOT_ID" "$ELASTIC_IP" "$PROVISION_S3_BUCKET" "$ENABLE_METRICS"env:ACCOUNT_NAME: ${{ github.event.inputs.ACCOUNT_NAME }}HOST: ${{ github.event.inputs.HOST }}INSTANCE_TYPE: ${{ github.event.inputs.INSTANCE_TYPE }}- DISK_SIZE: ${{ github.event.inputs.DISK_SIZE }}+ ROOT_DISK_SIZE: ${{ github.event.inputs.ROOT_DISK_SIZE }}+ DATA_DISK_SIZE: ${{ github.event.inputs.DATA_DISK_SIZE }}+ SNAPSHOT_ID: ${{ github.event.inputs.SNAPSHOT_ID }}ELASTIC_IP: ${{ github.event.inputs.ELASTIC_IP }}PROVISION_S3_BUCKET: ${{ github.event.inputs.PROVISION_S3_BUCKET }}AWS_ACCESS_KEY_ID: ${{ secrets._TEMP_AWS_ACCESS_KEY_ID }}