Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

14.1 KB
Raw Blame
#!/bin/bash
# ---------------------------------------------------------------------------------------------------------------------
#                    !!!!!!!!! MUST BE RUN FROM THE REPO ROOT DIR !!!!!!!!!
#
# Script that handles packaging deployment files and executing stack down/up via SCP/SSH; between stack down/up the
# host init script is executed (see .ci_cd/host_init/init.sh) this initialises the host and configures any standard
# deployment maintenance tasks (daily restart, daily backups, etc.).
#
# If AWS CLI authentication variables are configured then this script can use the AWS CLI to perform AWS specific
# configuration; if CIDR env variable is set then this will be temporarily added to the ssh-access security group
# ingress for SSH access on TCP port 22.
#
# Optionally supports rollback using ROLLBACK_ON_ERROR='true' but CLEAN_INSTALL must also be 'true' for rollback to
# work.
# ---------------------------------------------------------------------------------------------------------------------

# Function to be called before exiting to remove runner from AWS ssh-access security group
revoke_ssh () {
  if [ "$SSH_GRANTED" == 'true' ]; then
      if [ -n "$CIDR" ]; then
        "temp/aws/ssh_revoke.sh" "$CIDR" "github-da"
      fi
  fi
}

# Load the environment variables into this session
if [ -f "temp/env" ]; then
  echo "Loading environment variables: 'temp/env'"
  set -a
  . ./temp/env
  set +a
fi

# Load temp environment variables into this session
if [ -f "temp.env" ]; then
  echo "Loading temp environment variables: 'temp.env'"
  set -a
  . ./temp.env
  set +a
fi

# Check host is defined
if [ -z "$OR_HOSTNAME" ]; then
 echo "Host is not set"
 exit 1
fi
HOST="$OR_HOSTNAME"

# Copy CI/CD files into temp dir
echo "Copying CI/CD files into temp dir"
if [ "$IS_CORE_REPO" == 'false' ]; then
  cp -r core/.ci_cd/host_init temp/
  cp -r core/.ci_cd/aws temp/
fi
if [ -d ".ci_cd/host_init" ]; then
  cp -r .ci_cd/host_init temp/
fi
if [ -d ".ci_cd/aws" ]; then
  cp -r .ci_cd/aws temp/
fi

chmod -R +rx temp/

# Determine compose file to use and copy to temp dir (do this here as all env variables are loaded)
if [ -z "$ENV_COMPOSE_FILE" ]; then
  if [ -f "profile/$ENVIRONMENT.yml" ]; then
    cp "profile/$ENVIRONMENT.yml" temp/docker-compose.yml
  elif [ -f "docker-compose.yml" ]; then
    cp docker-compose.yml temp/docker-compose.yml
  fi
elif [ -f "$ENV_COMPOSE_FILE" ]; then
  cp "$ENV_COMPOSE_FILE" temp/docker-compose.yml
else
  cp docker-compose.yml temp/docker-compose.yml
fi
# Check docker compose file is present
if [ ! -f "temp/docker-compose.yml" ]; then
  echo "Couldn't determine docker compose file"
  exit 1
fi

# Set SSH/SCP command variables
sshCommandPrefix="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
scpCommandPrefix="scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
if [ -n "$SSH_PORT" ]; then
  sshCommandPrefix="$sshCommandPrefix -p $SSH_PORT"
  scpCommandPrefix="$scpCommandPrefix -P $SSH_PORT"
fi
if [ -f "ssh.key" ]; then
  chmod 400 ssh.key
  sshCommandPrefix="$sshCommandPrefix -i ssh.key"
  scpCommandPrefix="$scpCommandPrefix -i ssh.key"
fi
hostStr="$OR_HOSTNAME"
if [ -n "$SSH_USER" ]; then
  hostStr="${SSH_USER}@$hostStr"
fi

# Grant SSH access to this runner's public IP on AWS
if [ "$SKIP_SSH_WHITELIST" != 'true' ] && [ -n "$AWS_ACCESS_KEY_ID" ]; then

  echo "Granting AWS SSH access to github runner"

  source temp/aws/login.sh

  if [ -n "$CIDR" ]; then
    if [ -z "$AWS_ACCOUNT_NAME" ] && [ -z "$AWS_ACCOUNT_ID" ]; then

      echo "Account ID or name is not set so searching for it"
      source temp/aws/get_account_id_from_host.sh

      if [ -z "$AWS_ACCOUNT_ID" ]; then
        echo "Unable to determine account for host '$HOST'"
        exit 1
      fi
    fi

    source temp/aws/set_github-da_account_arn.sh

    echo "Attempting to add runner to AWS SSH whitelist"
    "temp/aws/ssh_whitelist.sh" "$CIDR" "github-runner" "github-da"
    if [ $? -eq 0 ]; then
      SSH_GRANTED=true
    fi
  fi
fi

# Determine host platform via ssh for deployment image building (can't export/import manifests)
PLATFORM=$($sshCommandPrefix $hostStr -- uname -m)
if [ $? -ne 0 ] || [ -z "$PLATFORM" ]; then
  echo "Failed to determine host platform, most likely SSH credentials and/or settings are invalid"
  revoke_ssh
  exit 1
fi
if [ "$PLATFORM" == "x86_64" ]; then
  PLATFORM="amd64"
fi
PLATFORM="linux/$PLATFORM"


# Verify manager tag and create docker image tarballs as required
if [ "$MANAGER_TAG" != '#ref' ]; then
  echo "Checking existence of manager docker image with tag: $MANAGER_TAG"
  docker buildx imagetools inspect --raw openremote/manager:$MANAGER_TAG
  if [ $? -ne 0 ]; then
    echo "Specified manager tag does not exist in docker hub"
    revoke_ssh
    exit 1
  fi
else
  echo "Using commit SHA for manager docker tag: $MANAGER_REF"
  MANAGER_TAG="$MANAGER_REF"
  # Export manager docker image for host platform
  docker build -o type=docker,dest=- --build-arg GIT_REPO=$REPO_NAME --build-arg GIT_COMMIT=$MANAGER_REF --platform $PLATFORM -t openremote/manager:$MANAGER_REF $MANAGER_DOCKER_BUILD_PATH | gzip > temp/manager.tar.gz
  if [ $? -ne 0 ] || [ ! -f temp/manager.tar.gz ]; then
    echo "Failed to export manager image with tag: $MANAGER_REF"
    revoke_ssh
    exit 1
  fi
fi
if [ -n "$DEPLOYMENT_REF" ]; then
  # Export deployment docker image for host platform
  docker build -o type=docker,dest=- --build-arg GIT_REPO=$REPO_NAME --build-arg GIT_COMMIT=$DEPLOYMENT_REF --platform $PLATFORM -t openremote/deployment:$DEPLOYMENT_REF $DEPLOYMENT_DOCKER_BUILD_PATH | gzip > temp/deployment.tar.gz
  if [ $? -ne 0 ] || [ ! -f temp/deployment.tar.gz ]; then
    echo "Failed to export deployment image"
    revoke_ssh
    exit 1
  fi
fi

# Set version variables
MANAGER_VERSION="$MANAGER_TAG"
DEPLOYMENT_VERSION="$DEPLOYMENT_REF"
echo "MANAGER_VERSION=\"$MANAGER_VERSION\"" >> temp/env
echo "DEPLOYMENT_VERSION=\"$DEPLOYMENT_VERSION\"" >> temp/env

echo "GZipping temp dir"
tar -zcvf temp.tar.gz temp

echo "Copying temp dir to host"
$scpCommandPrefix temp.tar.gz ${hostStr}:~

if [ "$ROLLBACK_ON_ERROR" == 'true' ]; then
  if [ "$CLEAN_INSTALL" != 'true' ]; then
    echo "ROLLBACK_ON_ERROR can only be used if CLEAN_INSTALL is set"
    ROLLBACK_ON_ERROR=false
  fi
fi

echo "Running deployment on host"
$sshCommandPrefix ${hostStr} << EOF

if [ "$ROLLBACK_ON_ERROR" == 'true' ]; then
  echo "Moving old temp dir to temp_old"
  rm -fr temp_old
  mv temp temp_old
  # Tag existing manager image with previous tag (current tag might not be available in docker hub anymore or it could have been overwritten)
  docker tag '`docker images openremote/manager -q | head -1`' openremote/manager:previous
else
  echo "Removing old temp deployment dir"
  rm -fr temp
fi

echo "Extracting temp dir"
tar -xvzf temp.tar.gz
chmod +x -R temp/

set -a
. ./temp/env
set +a

if [ -f "temp/manager.tar.gz" ]; then
  echo "Loading manager docker image"
  docker load < temp/manager.tar.gz
fi

if [ -f "temp/deployment.tar.gz" ]; then
  echo "Loading deployment docker image"
  docker load < temp/deployment.tar.gz
fi

# Login to Private ECR Registry to download restricted images
if [ -n "$OR_AWS_ECR_ACCOUNT_ID" ] && [ -n "$OR_AWS_ECR_REGION" ]; then
  aws ecr get-login-password --region $OR_AWS_ECR_REGION | docker login --username AWS --password-stdin $OR_AWS_ECR_ACCOUNT_ID.dkr.ecr.$OR_AWS_ECR_REGION.amazonaws.com &>/dev/null
fi

# Make sure we have correct keycloak, proxy and postgres images
echo "Pulling requested service versions from docker hub"
docker-compose -p or -f temp/docker-compose.yml pull --ignore-pull-failures

if [ \$? -ne 0 ]; then
  echo "Deployment failed to pull docker images"
  exit 1
fi

# Attempt docker compose down
CONTAINER_IDS=\$(docker ps -q)
if [ -n "\$CONTAINER_IDS" ]; then
  echo "Stopping existing stack"
  docker-compose -f temp/docker-compose.yml -p or down --remove-orphans 2> /dev/null

  if [ \$? -ne 0 ]; then
    echo "Deployment failed to stop the existing stack"
    exit 1
  fi
fi

# Run host init
hostInitCmd=
if [ -n "$HOST_INIT_SCRIPT" ]; then
  if [ ! -f "temp/host_init/${HOST_INIT_SCRIPT}.sh" ]; then
    echo "HOST_INIT_SCRIPT (temp/host_init/${HOST_INIT_SCRIPT}.sh) does not exist"
    exit 1
  fi
  hostInitCmd="temp/host_init/${HOST_INIT_SCRIPT}.sh"
elif [ -f "temp/host_init/init_${ENVIRONMENT}.sh" ]; then
  hostInitCmd="temp/host_init/init_${ENVIRONMENT}.sh"
elif [ -f "temp/host_init/init.sh" ]; then
  hostInitCmd="temp/host_init/init.sh"
fi
if [ -n "\$hostInitCmd" ]; then
  echo "Running host init script: '\$hostInitCmd'"
  sudo -E \$hostInitCmd
else
  echo "No host init script"
fi

# Delete any deployment volume so we get the latest
echo "Deleting existing deployment data volume"
docker volume rm or_deployment-data 1>/dev/null

# Get IP of interface on private subnet to expose metrics
TOKEN=\$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60" 2>/dev/null)

if [ -n "\$TOKEN" ]; then
  export PRIVATE_IP=\$(curl -H "X-aws-ec2-metadata-token: \$TOKEN" -s http://169.254.169.254/latest/meta-data/local-ipv4 2>/dev/null)
else
  echo "Could not get token to query for private IP"
fi
echo "Private IP \$PRIVATE_IP"

# Start the stack, including any one-shot deployment service, and wait for readiness
if ! bash temp/host_init/start.sh temp/docker-compose.yml or; then
  exit 1
fi

# Run host post init
hostPostInitCmd=
if [ -f "temp/host_init/post_init_${ENVIRONMENT}.sh" ]; then
  hostPostInitCmd="temp/host_init/post_init_${ENVIRONMENT}.sh"
elif [ -f "temp/host_init/post_init.sh" ]; then
  hostPostInitCmd="temp/host_init/post_init.sh"
fi
if [ -n "\$hostPostInitCmd" ]; then
  echo "Running host post init script: '\$hostPostInitCmd'"
  sudo -E \$hostPostInitCmd
else
  echo "No host post init script"
fi

# Store deployment snapshot data if the host can access S3 bucket with the same name as the host
docker image inspect \$(docker image ls -aq) > temp/image-info.txt
docker inspect \$(docker ps -aq) > temp/container-info.txt

aws s3 cp temp/image-info.txt s3://${OR_HOSTNAME}/image-info.txt &>/dev/null
aws s3 cp temp/container-info.txt s3://${OR_HOSTNAME}/container-info.txt &>/dev/null
exit 0
EOF

if [ $? -ne 0 ]; then
  echo "Deployment failed or is unhealthy"
  revoke_ssh
  exit 1
fi

echo "Testing manager web server https://$OR_HOSTNAME..."
response=$(curl --output /dev/null --silent --head --write-out "%{http_code}" https://$OR_HOSTNAME/manager/)
count=0
while [[ $response -ne 200 ]] && [ $count -lt 12 ]; do
  echo "https://$OR_HOSTNAME/manager/ RESPONSE CODE: $response...Sleeping 5 seconds"
  sleep 5
  response=$(curl --output /dev/null --silent --head --write-out "%{http_code}" https://$OR_HOSTNAME/manager/)
  count=$((count+1))
done

if [ $response -ne 200 ]; then
  echo "Manager web server is unreachable https://$OR_HOSTNAME..."
  revoke_ssh
  exit 1
else
  echo "Manager web server now reachable https://$OR_HOSTNAME..."
fi

revoke_ssh
exit 0

# ROLLBACK FUNCTIONALITY DISABLED FOR NOW AS WASN'T WORKING AND MANUAL INTERVENTION IS FINE AT THIS POINT

#if [ "$ROLLBACK_ON_ERROR" != 'true' ]; then
#else
#  DO_ROLLBACK=true
#fi

if [ "$DO_ROLLBACK" == 'true' ]; then
  echo "Attempting rollback"
  $sshCommandPrefix ${hostStr} << EOF

if [ ! -d "temp_old" ]; then
  echo "Previous deployment files not found so cannot rollback"
  exit 1
fi

rm -fr temp
mv temp_old temp

# Set MANAGER_VERSION to previous
echo 'MANAGER_VERSION="previous"' >> temp/env

set -a
. ./temp/env
set +a

if [ -f "temp/deployment.tar.gz" ]; then
  echo "Loading deployment docker image"
  docker load < temp/deployment.tar.gz
fi

# Login to Private ECR Registry to download restricted images
if [ -n "$OR_AWS_ECR_ACCOUNT_ID" ] && [ -n "$OR_AWS_ECR_REGION" ]; then
  aws ecr get-login-password --region $OR_AWS_ECR_REGION | docker login --username AWS --password-stdin $OR_AWS_ECR_ACCOUNT_ID.dkr.ecr.$OR_AWS_ECR_REGION.amazonaws.com &>/dev/null
fi

# Make sure we have correct keycloak, proxy and postgres images
echo "Pulling requested service versions from docker hub"
docker-compose -p or -f temp/docker-compose.yml pull --ignore-pull-failures

if [ \$? -ne 0 ]; then
  echo "Deployment failed to pull docker images"
  exit 1
fi

# Attempt docker compose down
echo "Stopping existing stack"
docker-compose -f temp/docker-compose.yml -p or down 2> /dev/null

# Run host init
hostInitCmd=
if [ -n "$HOST_INIT_SCRIPT" ]; then
  if [ ! -f "temp/host_init/${HOST_INIT_SCRIPT}.sh" ]; then
    echo "HOST_INIT_SCRIPT (temp/host_init/${HOST_INIT_SCRIPT}.sh) does not exist"
    exit 1
  fi
  hostInitCmd="temp/host_init/${HOST_INIT_SCRIPT}.sh"
elif [ -f "temp/host_init/init_${ENVIRONMENT}.sh" ]; then
  hostInitCmd="temp/host_init/init_${ENVIRONMENT}.sh"
elif [ -f "temp/host_init/init.sh" ]; then
  hostInitCmd="temp/host_init/init.sh"
fi
if [ -n "\$hostInitCmd" ]; then
  echo "Running host init script: '\$hostInitCmd'"
  sudo -E \$hostInitCmd
else
  echo "No host init script"
fi

# Delete any deployment volume so we get the latest
echo "Deleting existing deployment data volume"
docker volume rm or_deployment-data 1>/dev/null

# Start the stack, including any one-shot deployment service, and wait for readiness
if ! bash temp/host_init/start.sh temp/docker-compose.yml or; then
  exit 1
fi

# Run host post init
hostPostInitCmd=
if [ -f "temp/host_init/post_init_${ENVIRONMENT}.sh" ]; then
  hostPostInitCmd="temp/host_init/post_init_${ENVIRONMENT}.sh"
elif [ -f "temp/host_init/post_init.sh" ]; then
  hostPostInitCmd="temp/host_init/post_init.sh"
fi
if [ -n "\$hostPostInitCmd" ]; then
  echo "Running host post init script: '\$hostPostInitCmd'"
  sudo -E \$hostPostInitCmd
else
  echo "No host post init script"
fi

EOF
fi

echo "Testing manager web server https://$OR_HOSTNAME..."
response=$(curl --output /dev/null --silent --head --write-out "%{http_code}" https://$OR_HOSTNAME/manager/)
count=0
while [[ $response -ne 200 ]] && [ $count -lt 12 ]; do
  echo "https://$OR_HOSTNAME/manager/ RESPONSE CODE: $response...Sleeping 5 seconds"
  sleep 5
  response=$(curl --output /dev/null --silent --head --write-out "%{http_code}" https://$OR_HOSTNAME/manager/)
  count=$((count+1))
done

if [ $response -ne 200 ]; then
  echo "Manager web server is unreachable https://$OR_HOSTNAME..."
  revoke_ssh
  exit 1
else
  echo "Manager web server now reachable https://$OR_HOSTNAME..."
fi

revoke_ssh