100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
#!/usr/bin/env bash
# AWS-aware orchestration for an OpenRemote stack on an existing EKS cluster.
# Namespace-scoped Kubernetes resources remain owned by the portable or-stack
# command; this facade validates shared EKS services and coordinates endpoint
# readiness around it.
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/lib/hostname-reservations"
DEFAULT_AWS_REGION="eu-west-1"
DEFAULT_TIMEOUT="20m"
EXTERNAL_DNS_RELEASE_NAME="openremote-external-dns"
EXTERNAL_DNS_SERVICE_ACCOUNT_NAME="external-dns"
CERTIFICATE_MANAGED_BY_TAG_KEY="openremote.io/managed-by"
CERTIFICATE_MANAGED_BY_TAG_VALUE="or-eks-stack"
CERTIFICATE_CLUSTER_TAG_KEY="openremote.io/cluster"
CERTIFICATE_STACK_TAG_KEY="openremote.io/stack"
CERTIFICATE_HOSTNAME_TAG_KEY="openremote.io/hostname"
CERTIFICATE_ENDPOINT_TAG_KEY="openremote.io/endpoint"
CERTIFICATE_MODE_ANNOTATION="openremote.io/certificate-mode"
CERTIFICATE_ARN_ANNOTATION="openremote.io/certificate-arn"
HOSTNAME_ANNOTATION="openremote.io/hostname"
MQTTS_CERTIFICATE_MODE_ANNOTATION="openremote.io/mqtts-certificate-mode"
MQTTS_CERTIFICATE_ARN_ANNOTATION="openremote.io/mqtts-certificate-arn"
MQTTS_HOSTNAME_ANNOTATION="openremote.io/mqtts-hostname"
SHARED_CERTIFICATE_ARN_ANNOTATION="openremote.io/shared-certificate-arn"
ACM_VALIDATION_DNS_ENDPOINT_NAME="acm-validation"
ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME="acm-validation-mqtts"
COMMAND=""
STACK_NAME="${OR_EKS_STACK_NAME:-}"
CLUSTER_NAME="${OR_EKS_STACK_CLUSTER:-}"
AWS_REGION_NAME="${OR_EKS_STACK_AWS_REGION:-${AWS_REGION:-$DEFAULT_AWS_REGION}}"
# Only --profile selects an explicit profile; otherwise leave credential
# resolution to AWS, including its standard environment settings.
AWS_PROFILE_NAME=""
KUBE_CONTEXT="${OR_EKS_STACK_KUBE_CONTEXT:-}"
EXPOSURE="${OR_EKS_STACK_EXPOSURE:-haproxy}"
HOSTNAME_VALUE="${OR_EKS_STACK_HOSTNAME:-}"
CERTIFICATE_MODE="${OR_EKS_STACK_CERTIFICATE_MODE:-}"
CERTIFICATE_ARN="${OR_EKS_STACK_CERTIFICATE_ARN:-}"
VALUES_DIR="${OR_EKS_STACK_VALUES_DIR:-}"
TIMEOUT="${OR_EKS_STACK_TIMEOUT:-$DEFAULT_TIMEOUT}"
HTTP_PORT="${OR_EKS_STACK_HTTP_PORT:-80}"
HTTPS_PORT="${OR_EKS_STACK_HTTPS_PORT:-443}"
MQTTS_ENABLED="${OR_EKS_STACK_MQTTS:-false}"
MQTTS_PORT="${OR_EKS_STACK_MQTTS_PORT:-8883}"
MQTTS_PORT_CONFIGURED=false
MQTTS_HOSTNAME_VALUE="${OR_EKS_STACK_MQTTS_HOSTNAME:-}"
MQTTS_CERTIFICATE_MODE="${OR_EKS_STACK_MQTTS_CERTIFICATE_MODE:-}"
MQTTS_CERTIFICATE_ARN="${OR_EKS_STACK_MQTTS_CERTIFICATE_ARN:-}"
CONFIRM_STACK_NAME=""
OR_STACK_COMMAND="${OR_EKS_STACK_OR_STACK:-$SCRIPT_DIR/or-stack}"
LOAD_BALANCER_ADDRESS=""
MQTTS_LOAD_BALANCER_ADDRESS=""
EXTERNAL_DNS_ARGS=""
CERTIFICATE_JSON=""
CERTIFICATE_EXISTS=false
MQTTS_CERTIFICATE_JSON=""
MQTTS_CERTIFICATE_EXISTS=false
STACK_NAMESPACE_EXISTS=false
VALIDATION_DNS_RECORDS=""
MQTTS_VALIDATION_DNS_RECORDS=""
MANAGED_DNS_DOMAIN=""
AUTHORITATIVE_NAME_SERVERS=""
STACK_INGRESS_JSON=""
STACK_MQTTS_SERVICE_JSON=""
VALIDATION_DNS_ENDPOINT_EXISTS=false
MQTTS_VALIDATION_DNS_ENDPOINT_EXISTS=false
MANAGED_CERTIFICATE_DESTROY=false
MANAGED_MQTTS_CERTIFICATE_DESTROY=false
if [ "${OR_EKS_STACK_MQTTS_PORT+x}" = "x" ]; then
MQTTS_PORT_CONFIGURED=true
fi
usage() {
cat <<'EOF'
Manage an OpenRemote stack on an existing Amazon EKS cluster.
Usage:
or-eks-stack apply --name <name> --cluster <cluster> --hostname <host> [options]
or-eks-stack status --name <name> --cluster <cluster> [options]
or-eks-stack credentials --name <name> --cluster <cluster> [options]
or-eks-stack uninstall --name <name> --cluster <cluster> [options]
or-eks-stack destroy --name <name> --cluster <cluster> --confirm <name> [options]
Commands:
apply Install or upgrade the stack, wait for its load balancer
and DNS, resolve TLS, and verify trusted HTTPS and any
enabled MQTTS endpoint.
status Show the stack's Kubernetes status.
credentials Show the Manager administrator login credentials.
uninstall Remove workloads while retaining namespace, credentials,
PVCs, and certificate state.
destroy Delete the stack namespace and its data after exact-name
confirmation.
Options:
--name <name> Stack and namespace name. Env: OR_EKS_STACK_NAME.
--cluster <name> Existing EKS cluster name. Env: OR_EKS_STACK_CLUSTER.
--region <region> AWS region. Env: OR_EKS_STACK_AWS_REGION.
Default: eu-west-1.
--profile <name> Explicit AWS profile. Default: no profile argument;
use the normal AWS credential chain. An empty value
also leaves profile selection to AWS.
--kube-context <c> Kubernetes context. Defaults to <cluster>@<region>.
Env: OR_EKS_STACK_KUBE_CONTEXT.
--exposure <mode> Public exposure: haproxy or ingress. Env:
OR_EKS_STACK_EXPOSURE. Default: haproxy.
--hostname <host> Public, ExternalDNS-managed hostname. Required by apply.
Must be a strict subdomain of a managed DNS domain.
Env: OR_EKS_STACK_HOSTNAME.
--certificate-mode <mode>
Certificate ownership mode for Ingress: existing,
shared, or managed. Reapply preserves the recorded mode.
On first selection, omission chooses a configured shared
certificate, otherwise managed. In-place mode migration
is not supported. Env:
OR_EKS_STACK_CERTIFICATE_MODE.
--certificate-arn <arn>
Existing ACM ARN for mode existing; reapply can omit it
to reuse the recorded ARN. Env:
OR_EKS_STACK_CERTIFICATE_ARN. Mode shared reads the ARN
from the cluster's openremote-alb IngressClass.
--http-port <port> Public HAProxy Service HTTP port. Env:
OR_EKS_STACK_HTTP_PORT. HAProxy only; must remain 80 for
automated HTTP-01 certificate issuance.
--https-port <port> Public HAProxy Service HTTPS port. Env:
OR_EKS_STACK_HTTPS_PORT. HAProxy only; default: 443.
--mqtts Expose MQTTS. HAProxy uses the web hostname and its
own certificate; Ingress uses a dedicated NLB,
hostname, and ACM certificate. Env:
OR_EKS_STACK_MQTTS. Default: false. Plaintext MQTT is
not exposed.
--mqtts-port <port> Public MQTTS port. Env: OR_EKS_STACK_MQTTS_PORT.
Default: 8883. Requires --mqtts.
--mqtts-hostname <host>
Dedicated ExternalDNS-managed hostname for Ingress
MQTTS. Required on first apply; reapply may reuse the
recorded hostname. Env: OR_EKS_STACK_MQTTS_HOSTNAME.
--mqtts-certificate-mode <mode>
Certificate ownership mode for Ingress MQTTS: existing,
shared, or managed. An explicit ARN implies existing;
otherwise first apply selects the cluster's configured
shared certificate, falling back to managed. Reapply
preserves the recorded mode. Env:
OR_EKS_STACK_MQTTS_CERTIFICATE_MODE.
--mqtts-certificate-arn <arn>
Existing ACM certificate for the Ingress MQTTS NLB in
mode existing. Reapply may reuse the recorded ARN. Env:
OR_EKS_STACK_MQTTS_CERTIFICATE_ARN.
--values-dir <path> Optional per-component values directory passed to
or-stack. Env: OR_EKS_STACK_VALUES_DIR.
--timeout <duration> Timeout used by Helm and each endpoint readiness phase.
Accepts seconds or an s, m, or h suffix. Env:
OR_EKS_STACK_TIMEOUT. Default: 20m.
--confirm <name> Required exact stack name for destroy.
-h, --help Show this help.
This is an AWS-aware facade over or-stack. It requires the OpenRemote-managed
ExternalDNS release installed by or-eks-cluster, with active Route 53 writes
for the requested hostname. The portable or-stack command remains available
for local clusters, unmanaged DNS, and internal-only stacks.
Before workloads or ACM requests, the facade prepares the namespace and reserves
its hostname through or-stack. A hostname cannot belong to two stacks or change
on an existing stack. Reservations survive uninstall and failed operations;
destroy releases them only after external cleanup. See kubernetes/README.md for
registry permissions, required stack metadata, and manual recovery. Destroy
requires a hostname reservation in every mode; missing state is an error and
never bypasses DNS cleanup.
Ingress accepts an explicit existing ACM ARN, the externally managed shared
certificate configured by or-eks-cluster, or a stack-owned managed certificate.
Managed mode requests or reuses an exact-hostname ACM certificate and retains
its namespaced DNS-validation record through uninstall. Destroy removes the
stack route and namespace, waits for its public and validation DNS records to
disappear, then deletes only the exactly owned ACM certificate.
Managed mode requires exclusive use of the certificate and its validation
records by this stack. Other certificates must not depend on those records;
this deployment requirement is not established by the script's ownership checks.
The first certificate mode is retained across failed applies and uninstall.
Changing mode requires destroying and recreating the stack; back up any data
to retain first. Certificate rotation within existing or shared mode is supported.
Ingress MQTTS accepts an existing user-owned ACM certificate, the cluster's
externally managed shared certificate, or a dedicated stack-managed
certificate. Managed mode owns a separate ACM certificate and DNS-validation
resource for the MQTTS endpoint; destroy deletes only that exactly tagged
certificate after the NLB and validation records are removed.
EOF
}
log() {
printf '[or-eks-stack] %s\n' "$*"
}
warn() {
printf '[or-eks-stack] WARNING: %s\n' "$*" >&2
}
die() {
printf '[or-eks-stack] ERROR: %s\n' "$*" >&2
exit 1
}
require_command() {
command -v "$1" >/dev/null 2>&1 || die \
"Required command '$1' was not found in PATH."
}
aws_cli() {
if [ -n "$AWS_PROFILE_NAME" ]; then
aws "$@" --region "$AWS_REGION_NAME" --profile "$AWS_PROFILE_NAME"
else
aws "$@" --region "$AWS_REGION_NAME"
fi
}
aws_eks() {
aws_cli eks "$@"
}
aws_acm() {
aws_cli acm "$@"
}
aws_elbv2() {
aws_cli elbv2 "$@"
}
aws_sts() {
aws_cli sts "$@"
}
run_or_stack() {
"$OR_STACK_COMMAND" "$@"
}
parse_arguments() {
if [ "$#" -eq 0 ]; then
usage >&2
exit 1
fi
case "$1" in
-h|--help)
usage
exit 0
;;
apply|status|credentials|uninstall|destroy)
COMMAND="$1"
shift
;;
*)
die "Unknown command '$1'. Run with --help for usage."
;;
esac
while [ "$#" -gt 0 ]; do
case "$1" in
--name)
[ "$#" -ge 2 ] || die "Option --name requires a value."
STACK_NAME="$2"
shift 2
;;
--cluster)
[ "$#" -ge 2 ] || die "Option --cluster requires a value."
CLUSTER_NAME="$2"
shift 2
;;
--region)
[ "$#" -ge 2 ] || die "Option --region requires a value."
AWS_REGION_NAME="$2"
shift 2
;;
--profile)
[ "$#" -ge 2 ] || die "Option --profile requires a value."
AWS_PROFILE_NAME="$2"
shift 2
;;
--kube-context)
[ "$#" -ge 2 ] || die "Option --kube-context requires a value."
KUBE_CONTEXT="$2"
shift 2
;;
--exposure)
[ "$#" -ge 2 ] || die "Option --exposure requires a value."
EXPOSURE="$2"
shift 2
;;
--hostname)
[ "$#" -ge 2 ] || die "Option --hostname requires a value."
HOSTNAME_VALUE="$2"
shift 2
;;
--certificate-mode)
[ "$#" -ge 2 ] || die "Option --certificate-mode requires a value."
CERTIFICATE_MODE="$2"
shift 2
;;
--certificate-arn)
[ "$#" -ge 2 ] || die "Option --certificate-arn requires a value."
CERTIFICATE_ARN="$2"
shift 2
;;
--http-port)
[ "$#" -ge 2 ] || die "Option --http-port requires a value."
HTTP_PORT="$2"
shift 2
;;
--https-port)
[ "$#" -ge 2 ] || die "Option --https-port requires a value."
HTTPS_PORT="$2"
shift 2
;;
--mqtts)
MQTTS_ENABLED=true
shift
;;
--mqtts-port)
[ "$#" -ge 2 ] || die "Option --mqtts-port requires a value."
MQTTS_PORT="$2"
MQTTS_PORT_CONFIGURED=true
shift 2
;;
--mqtts-hostname)
[ "$#" -ge 2 ] || die "Option --mqtts-hostname requires a value."
MQTTS_HOSTNAME_VALUE="$2"
shift 2
;;
--mqtts-certificate-mode)
[ "$#" -ge 2 ] || die "Option --mqtts-certificate-mode requires a value."
MQTTS_CERTIFICATE_MODE="$2"
shift 2
;;
--mqtts-certificate-arn)
[ "$#" -ge 2 ] || die "Option --mqtts-certificate-arn requires a value."
MQTTS_CERTIFICATE_ARN="$2"
shift 2
;;
--values-dir)
[ "$#" -ge 2 ] || die "Option --values-dir requires a value."
VALUES_DIR="$2"
shift 2
;;
--timeout)
[ "$#" -ge 2 ] || die "Option --timeout requires a value."
TIMEOUT="$2"
shift 2
;;
--confirm)
[ "$#" -ge 2 ] || die "Option --confirm requires a value."
CONFIRM_STACK_NAME="$2"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
die "Unknown option '$1'. Run with --help for usage."
;;
esac
done
}
validate_stack_name() {
[ -n "$STACK_NAME" ] || die "A stack name is required; use --name."
[ "${#STACK_NAME}" -le 63 ] || die \
"Stack name must contain at most 63 characters."
if [[ ! "$STACK_NAME" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then
die "Stack name must be a lowercase Kubernetes namespace name."
fi
case "$STACK_NAME" in
default|kube-system|kube-public|kube-node-lease)
die "Namespace '$STACK_NAME' is reserved and cannot be used as a stack name."
;;
esac
}
validate_cluster_name() {
[ -n "$CLUSTER_NAME" ] || die "An EKS cluster name is required; use --cluster."
[ "${#CLUSTER_NAME}" -le 100 ] || die \
"EKS cluster name must contain at most 100 characters."
if [[ ! "$CLUSTER_NAME" =~ ^[A-Za-z0-9][A-Za-z0-9_-]*$ ]]; then
die "EKS cluster name may contain only letters, numbers, underscores, and hyphens."
fi
}
validate_port() {
local port="$1"
local name="$2"
[[ "$port" =~ ^[0-9]+$ ]] || die \
"$name port must be a number from 1 to 65535."
[ "$port" -ge 1 ] && [ "$port" -le 65535 ] || die \
"$name port must be a number from 1 to 65535."
}
duration_seconds() {
local duration="$1"
local multiplier
if [[ ! "$duration" =~ ^([0-9]+)([smh]?)$ ]]; then
return 1
fi
case "${BASH_REMATCH[2]}" in
""|s) multiplier=1 ;;
m) multiplier=60 ;;
h) multiplier=3600 ;;
*) return 1 ;;
esac
printf '%s\n' "$((10#${BASH_REMATCH[1]} * multiplier))"
}
validate_common_options() {
local timeout_seconds
validate_stack_name
validate_cluster_name
[ -n "$AWS_REGION_NAME" ] || die "AWS region cannot be empty."
if [ -z "$KUBE_CONTEXT" ]; then
KUBE_CONTEXT="${CLUSTER_NAME}@${AWS_REGION_NAME}"
fi
[ -n "$KUBE_CONTEXT" ] || die "Kubernetes context cannot be empty."
if ! timeout_seconds="$(duration_seconds "$TIMEOUT")"; then
die "Timeout must be seconds or a duration with an s, m, or h suffix."
fi
[ "$timeout_seconds" -gt 0 ] || die "Timeout must be greater than zero."
}
validate_hostname_value() {
local hostname="$1"
local description="$2"
[ -n "$hostname" ] || die "$description is required."
[ "${#hostname}" -le 253 ] || die \
"$description must contain at most 253 characters."
if [[ ! "$hostname" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)+$ ]]; then
die "$description must be a lowercase, fully qualified DNS name."
fi
}
validate_hostname() {
HOSTNAME_VALUE="$(printf '%s' "$HOSTNAME_VALUE" | tr '[:upper:]' '[:lower:]')"
HOSTNAME_VALUE="${HOSTNAME_VALUE%.}"
[ -n "$HOSTNAME_VALUE" ] || die \
"A public hostname is required for apply; use --hostname."
validate_hostname_value "$HOSTNAME_VALUE" "Hostname"
}
normalize_and_validate_mqtts_hostname() {
MQTTS_HOSTNAME_VALUE="$(printf '%s' "$MQTTS_HOSTNAME_VALUE" | tr '[:upper:]' '[:lower:]')"
MQTTS_HOSTNAME_VALUE="${MQTTS_HOSTNAME_VALUE%.}"
[ -n "$MQTTS_HOSTNAME_VALUE" ] || die \
"An MQTTS hostname is required for Ingress exposure; use --mqtts-hostname."
validate_hostname_value "$MQTTS_HOSTNAME_VALUE" "MQTTS hostname"
}
validate_apply_options() {
validate_hostname
case "$MQTTS_ENABLED" in
true|false)
;;
*)
die "OR_EKS_STACK_MQTTS must be true or false."
;;
esac
if [ "$MQTTS_ENABLED" != true ] && [ "$MQTTS_PORT_CONFIGURED" = true ]; then
die "Option --mqtts-port requires --mqtts."
fi
if [ "$MQTTS_ENABLED" != true ] && \
{ [ -n "$MQTTS_HOSTNAME_VALUE" ] || [ -n "$MQTTS_CERTIFICATE_MODE" ] || \
[ -n "$MQTTS_CERTIFICATE_ARN" ]; }; then
die "Options --mqtts-hostname, --mqtts-certificate-mode, and --mqtts-certificate-arn require --mqtts."
fi
case "$EXPOSURE" in
haproxy)
[ -z "$CERTIFICATE_MODE" ] || die \
"Option --certificate-mode is only valid with Ingress exposure."
[ -z "$CERTIFICATE_ARN" ] || die \
"Option --certificate-arn is only valid with Ingress exposure."
[ -z "$MQTTS_HOSTNAME_VALUE" ] || die \
"Option --mqtts-hostname is only valid with Ingress exposure."
[ -z "$MQTTS_CERTIFICATE_MODE" ] || die \
"Option --mqtts-certificate-mode is only valid with Ingress exposure."
[ -z "$MQTTS_CERTIFICATE_ARN" ] || die \
"Option --mqtts-certificate-arn is only valid with Ingress exposure."
validate_port "$HTTP_PORT" "HTTP"
validate_port "$HTTPS_PORT" "HTTPS"
[ "$HTTP_PORT" != "$HTTPS_PORT" ] || die \
"HAProxy HTTP and HTTPS ports must be different."
if [ "$MQTTS_ENABLED" = true ]; then
validate_port "$MQTTS_PORT" "MQTTS"
[ "$MQTTS_PORT" != "$HTTP_PORT" ] && [ "$MQTTS_PORT" != "$HTTPS_PORT" ] || die \
"HAProxy HTTP, HTTPS, and MQTTS ports must be different."
fi
[ "$HTTP_PORT" = "80" ] || die \
"The automated HAProxy ACME workflow requires public HTTP port 80. Use or-stack directly for a custom HTTP port."
;;
ingress)
if [ "$MQTTS_ENABLED" = true ]; then
validate_port "$MQTTS_PORT" "MQTTS"
if [ -n "$MQTTS_HOSTNAME_VALUE" ]; then
normalize_and_validate_mqtts_hostname
[ "$MQTTS_HOSTNAME_VALUE" != "$HOSTNAME_VALUE" ] || die \
"Ingress web and MQTTS hostnames must be different."
fi
if [ -n "$MQTTS_CERTIFICATE_ARN" ] && \
[[ ! "$MQTTS_CERTIFICATE_ARN" =~ ^arn:[^:]+:acm:[a-z0-9-]+:[0-9]{12}:certificate/[A-Za-z0-9-]+$ ]]; then
die "MQTTS certificate ARN must be a valid ACM certificate ARN."
fi
case "$MQTTS_CERTIFICATE_MODE" in
existing|shared|managed|"") ;;
*)
die "Unsupported Ingress MQTTS certificate mode '$MQTTS_CERTIFICATE_MODE'; expected existing, shared, or managed."
;;
esac
if { [ "$MQTTS_CERTIFICATE_MODE" = shared ] || \
[ "$MQTTS_CERTIFICATE_MODE" = managed ]; } && \
[ -n "$MQTTS_CERTIFICATE_ARN" ]; then
die "Option --mqtts-certificate-arn requires --mqtts-certificate-mode existing; shared and managed modes resolve it automatically."
fi
fi
case "$CERTIFICATE_MODE" in
existing)
# An existing stack may reuse its recorded ARN; resolve it after
# reading namespace metadata, before validating the certificate.
if [ -n "$CERTIFICATE_ARN" ] && [[ ! "$CERTIFICATE_ARN" =~ ^arn:[^:]+:acm:[a-z0-9-]+:[0-9]{12}:certificate/[A-Za-z0-9-]+$ ]]; then
die "Certificate ARN must be a valid ACM certificate ARN."
fi
;;
shared|managed|"")
[ -z "$CERTIFICATE_ARN" ] || die \
"Option --certificate-arn requires --certificate-mode existing; shared and managed modes resolve it automatically."
;;
*)
die "Unsupported Ingress certificate mode '$CERTIFICATE_MODE'; expected existing, shared, or managed."
;;
esac
[ "$HTTP_PORT" = "80" ] && [ "$HTTPS_PORT" = "443" ] || die \
"Options --http-port and --https-port are only configurable with HAProxy exposure."
;;
*)
die "Unsupported exposure '$EXPOSURE'; expected haproxy or ingress."
;;
esac
if [ -n "$VALUES_DIR" ] && [ ! -d "$VALUES_DIR" ]; then
die "Values directory '$VALUES_DIR' does not exist."
fi
}
validate_command_options() {
if [ "$COMMAND" != "destroy" ] && [ -n "$CONFIRM_STACK_NAME" ]; then
die "Option --confirm is only valid with destroy."
fi
if [ "$COMMAND" = "destroy" ] && [ "$CONFIRM_STACK_NAME" != "$STACK_NAME" ]; then
die "Destroy requires --confirm '$STACK_NAME'."
fi
if [ "$COMMAND" = "apply" ]; then
validate_apply_options
fi
}
require_active_cluster() {
local cluster_status
local output
local status
set +e
output="$(aws_eks describe-cluster \
--name "$CLUSTER_NAME" \
--query 'cluster.status' \
--output text 2>&1)"
status=$?
set -e
if [ "$status" -ne 0 ]; then
case "$output" in
*ResourceNotFoundException*)
die "EKS cluster '$CLUSTER_NAME' does not exist in region '$AWS_REGION_NAME'."
;;
*)
die "Unable to query EKS cluster '$CLUSTER_NAME': $output"
;;
esac
fi
cluster_status="$output"
[ "$cluster_status" = "ACTIVE" ] || die \
"EKS cluster '$CLUSTER_NAME' is '$cluster_status', not ACTIVE."
}
require_matching_kube_context() {
local configured_context
local context_endpoint
local expected_endpoint
configured_context="$(kubectl config get-contexts "$KUBE_CONTEXT" -o name \
2>/dev/null || true)"
[ "$configured_context" = "$KUBE_CONTEXT" ] || die \
"Kubernetes context '$KUBE_CONTEXT' does not exist; run or-eks-cluster kubeconfig first."
expected_endpoint="$(aws_eks describe-cluster \
--name "$CLUSTER_NAME" \
--query 'cluster.endpoint' \
--output text)" || die \
"Unable to read the endpoint for EKS cluster '$CLUSTER_NAME'."
context_endpoint="$(kubectl --context "$KUBE_CONTEXT" config view \
--minify \
-o 'jsonpath={.clusters[0].cluster.server}')" || die \
"Unable to inspect Kubernetes context '$KUBE_CONTEXT'."
[ "$context_endpoint" = "$expected_endpoint" ] || die \
"Kubernetes context '$KUBE_CONTEXT' does not point to EKS cluster '$CLUSTER_NAME'."
kubectl --context "$KUBE_CONTEXT" get namespace kube-system >/dev/null 2>&1 || die \
"Kubernetes context '$KUBE_CONTEXT' cannot reach the EKS cluster."
}
require_eks_target() {
require_command aws
require_command kubectl
require_active_cluster
require_matching_kube_context
}
require_external_dns_argument() {
local external_dns_args="$1"
local required_argument="$2"
grep -Fx -- "$required_argument" <<< "$external_dns_args" >/dev/null || die \
"ExternalDNS is missing required argument '$required_argument'; run or-eks-cluster apply with the managed DNS configuration."
}
dns_name_is_covered() {
local dns_name="$1"
local domain_filter="$2"
dns_name="${dns_name%.}"
domain_filter="${domain_filter%.}"
[ "$dns_name" = "$domain_filter" ] || \
[[ "$dns_name" == *."$domain_filter" ]]
}
hostname_is_covered() {
local domain_filter="$1"
local hostname="${2:-$HOSTNAME_VALUE}"
dns_name_is_covered "$hostname" "$domain_filter"
}
require_managed_external_dns() {
local domain_filter
local external_dns_args
local hostname_covered=false
local mqtts_hostname_covered=false
local service_account_role_arn
helm status "$EXTERNAL_DNS_RELEASE_NAME" \
--kube-context "$KUBE_CONTEXT" \
--namespace kube-system >/dev/null 2>&1 || die \
"OpenRemote-managed ExternalDNS is not installed; configure it with or-eks-cluster apply first."
kubectl --context "$KUBE_CONTEXT" --namespace kube-system rollout status \
"deployment/$EXTERNAL_DNS_RELEASE_NAME" --timeout "$TIMEOUT" >/dev/null || die \
"ExternalDNS is not ready; inspect deployment/$EXTERNAL_DNS_RELEASE_NAME in kube-system."
external_dns_args="$(kubectl --context "$KUBE_CONTEXT" \
--namespace kube-system \
get deployment "$EXTERNAL_DNS_RELEASE_NAME" \
-o 'go-template={{range (index .spec.template.spec.containers 0).args}}{{println .}}{{end}}')" || die \
"Unable to inspect the ExternalDNS deployment."
if [ "$EXPOSURE" = "ingress" ]; then
require_external_dns_argument "$external_dns_args" "--source=ingress"
if [ "$MQTTS_ENABLED" = true ]; then
require_external_dns_argument "$external_dns_args" "--source=service"
fi
else
require_external_dns_argument "$external_dns_args" "--source=service"
fi
require_external_dns_argument "$external_dns_args" "--policy=sync"
require_external_dns_argument "$external_dns_args" "--registry=txt"
require_external_dns_argument "$external_dns_args" \
"--label-filter=openremote.io/managed-dns=true"
if [ "$CERTIFICATE_MODE" = "managed" ] || \
[ "$MQTTS_CERTIFICATE_MODE" = "managed" ]; then
require_external_dns_argument "$external_dns_args" "--source=crd"
kubectl --context "$KUBE_CONTEXT" get \
customresourcedefinition/dnsendpoints.externaldns.k8s.io >/dev/null 2>&1 || die \
"ExternalDNS DNSEndpoint support is missing; run or-eks-cluster apply before using managed certificates."
fi
grep -Eq '^--txt-owner-id=.+$' <<< "$external_dns_args" || die \
"ExternalDNS has no TXT owner ID; reconcile its managed configuration before applying a stack."
if grep -Eq '^--dry-run(=true)?$' <<< "$external_dns_args"; then
die "ExternalDNS dry-run is enabled; disable it with or-eks-cluster apply before applying a stack."
fi
while IFS= read -r domain_filter; do
[ -n "$domain_filter" ] || continue
if hostname_is_covered "$domain_filter"; then
hostname_covered=true
fi
if [ "$MQTTS_ENABLED" = true ] && [ "$EXPOSURE" = ingress ] && \
hostname_is_covered "$domain_filter" "$MQTTS_HOSTNAME_VALUE"; then
mqtts_hostname_covered=true
fi
done < <(sed -n 's/^--domain-filter=//p' <<< "$external_dns_args")
[ "$hostname_covered" = true ] || die \
"ExternalDNS domain filters do not cover '$HOSTNAME_VALUE'; reconcile the cluster DNS configuration first."
if [ "$MQTTS_ENABLED" = true ] && [ "$EXPOSURE" = ingress ]; then
[ "$mqtts_hostname_covered" = true ] || die \
"ExternalDNS domain filters do not cover MQTTS hostname '$MQTTS_HOSTNAME_VALUE'; reconcile the cluster DNS configuration first."
fi
service_account_role_arn="$(kubectl --context "$KUBE_CONTEXT" \
--namespace kube-system \
get serviceaccount "$EXTERNAL_DNS_SERVICE_ACCOUNT_NAME" \
-o 'jsonpath={.metadata.annotations.eks\.amazonaws\.com/role-arn}')" || die \
"Unable to inspect the ExternalDNS ServiceAccount."
[ -n "$service_account_role_arn" ] || die \
"ExternalDNS ServiceAccount kube-system/$EXTERNAL_DNS_SERVICE_ACCOUNT_NAME has no IRSA role annotation."
EXTERNAL_DNS_ARGS="$external_dns_args"
}
require_stack_hostname_subdomain() {
local domain_filter
local hostname="${1:-$HOSTNAME_VALUE}"
# ExternalDNS prefixes ownership TXT names with the record type. At the
# managed domain itself that name leaves the IAM domain boundary (for example,
# example.com -> a-example.com). Nested subdomains keep it inside the boundary.
# This is an apply-only check so it never prevents cleanup of existing state.
while IFS= read -r domain_filter; do
domain_filter="${domain_filter%.}"
[ -n "$domain_filter" ] || continue
if [[ "$hostname" == *."$domain_filter" ]]; then
return
fi
done < <(sed -n 's/^--domain-filter=//p' <<< "$EXTERNAL_DNS_ARGS")
die "Hostname '$hostname' must be a strict subdomain of a managed ExternalDNS domain; using the domain itself would place ownership TXT records outside the permitted DNS boundary."
}
read_certificate_namespace() {
local namespace_json
namespace_json="$(kubectl --context "$KUBE_CONTEXT" get namespace "$STACK_NAME" \
--ignore-not-found -o json)" || die "Unable to read the stack's certificate configuration."
if [ -n "$namespace_json" ]; then
jq -e --arg stack "$STACK_NAME" '.metadata.labels["openremote.io/stack"] == $stack' \
<<< "$namespace_json" >/dev/null || die \
"Namespace '$STACK_NAME' is not owned by this stack; refusing to use its certificate configuration."
fi
printf '%s\n' "$namespace_json"
}
resolve_ingress_certificate() {
local shared_certificate_arn namespace_json recorded_mode recorded_arn
namespace_json="$(read_certificate_namespace)" || die "Unable to resolve the stack's certificate mode."
recorded_mode="$(jq -r '.metadata.annotations["openremote.io/certificate-mode"] // empty' <<< "$namespace_json")"
recorded_arn="$(jq -r '.metadata.annotations["openremote.io/certificate-arn"] // empty' <<< "$namespace_json")"
if [ -n "$recorded_mode" ]; then
case "$recorded_mode" in
existing|shared|managed) ;;
*) die "Stack '$STACK_NAME' records incompatible certificate mode '$recorded_mode'." ;;
esac
[ -z "$CERTIFICATE_MODE" ] || [ "$CERTIFICATE_MODE" = "$recorded_mode" ] || die \
"Stack '$STACK_NAME' uses certificate mode '$recorded_mode'; in-place certificate mode migration to '$CERTIFICATE_MODE' is not supported."
# Cluster defaults apply only to first selection. Re-evaluating them on
# reapply can abandon a managed certificate and make destroy inconsistent.
CERTIFICATE_MODE="$recorded_mode"
fi
case "$CERTIFICATE_MODE" in
existing)
CERTIFICATE_ARN="${CERTIFICATE_ARN:-$recorded_arn}"
[ -n "$CERTIFICATE_ARN" ] || die "Ingress with certificate mode existing requires --certificate-arn on first apply."
return
;;
managed)
return
;;
esac
if ! shared_certificate_arn="$(kubectl --context "$KUBE_CONTEXT" \
get ingressclass openremote-alb \
-o "go-template={{with .metadata.annotations}}{{with index . \"$SHARED_CERTIFICATE_ARN_ANNOTATION\"}}{{.}}{{end}}{{end}}" \
2>/dev/null)"; then
die "Unable to inspect shared certificate configuration on IngressClass 'openremote-alb'."
fi
if [ -z "$shared_certificate_arn" ]; then
if [ "$CERTIFICATE_MODE" = "shared" ]; then
die "Ingress certificate mode shared requires a certificate configured with or-eks-cluster --shared-certificate-arn."
fi
CERTIFICATE_MODE=managed
log "No shared ACM certificate is configured; selected stack-managed certificate mode."
return
fi
CERTIFICATE_MODE=shared
CERTIFICATE_ARN="$shared_certificate_arn"
log "Selected the cluster's shared ACM certificate '$CERTIFICATE_ARN'."
}
resolve_ingress_mqtts_configuration() {
local namespace_json
local recorded_arn
local recorded_hostname
local recorded_mode
local requested_arn="$MQTTS_CERTIFICATE_ARN"
local requested_mode="$MQTTS_CERTIFICATE_MODE"
local shared_certificate_arn=""
namespace_json="$(read_certificate_namespace)" || die \
"Unable to resolve the stack's MQTTS certificate configuration."
recorded_hostname="$(jq -r '.metadata.annotations["openremote.io/mqtts-hostname"] // empty' \
<<< "$namespace_json")"
recorded_mode="$(jq -r '.metadata.annotations["openremote.io/mqtts-certificate-mode"] // empty' \
<<< "$namespace_json")"
recorded_arn="$(jq -r '.metadata.annotations["openremote.io/mqtts-certificate-arn"] // empty' \
<<< "$namespace_json")"
if [ "$MQTTS_ENABLED" != true ]; then
if [ -n "$recorded_hostname" ] || [ -n "$recorded_mode" ] || [ -n "$recorded_arn" ]; then
die "Stack '$STACK_NAME' records an Ingress MQTTS endpoint; repeat --mqtts to preserve it. Disabling it in place is not supported."
fi
return
fi
case "$recorded_mode" in
existing|shared|managed|"") ;;
*) die "Stack '$STACK_NAME' records incompatible MQTTS certificate mode '$recorded_mode'." ;;
esac
if [ -z "$recorded_mode" ] && [ -n "$recorded_arn" ]; then
die "Stack '$STACK_NAME' has incomplete MQTTS certificate state; inspect its namespace annotations before retrying."
fi
if [ -n "$recorded_mode" ]; then
[ -z "$requested_mode" ] || [ "$requested_mode" = "$recorded_mode" ] || die \
"Stack '$STACK_NAME' uses MQTTS certificate mode '$recorded_mode'; in-place certificate mode migration to '$requested_mode' is not supported."
MQTTS_CERTIFICATE_MODE="$recorded_mode"
elif [ -z "$MQTTS_CERTIFICATE_MODE" ]; then
if [ -n "$requested_arn" ]; then
MQTTS_CERTIFICATE_MODE=existing
else
if ! shared_certificate_arn="$(kubectl --context "$KUBE_CONTEXT" \
get ingressclass openremote-alb \
-o "go-template={{with .metadata.annotations}}{{with index . \"$SHARED_CERTIFICATE_ARN_ANNOTATION\"}}{{.}}{{end}}{{end}}" \
2>/dev/null)"; then
die "Unable to inspect shared MQTTS certificate configuration on IngressClass 'openremote-alb'."
fi
if [ -n "$shared_certificate_arn" ]; then
MQTTS_CERTIFICATE_MODE=shared
else
MQTTS_CERTIFICATE_MODE=managed
log "No shared MQTTS certificate is configured; selected stack-managed MQTTS certificate mode."
fi
fi
fi
if [ -n "$recorded_hostname" ]; then
if [ -n "$MQTTS_HOSTNAME_VALUE" ] && \
[ "${MQTTS_HOSTNAME_VALUE%.}" != "${recorded_hostname%.}" ]; then
die "Stack '$STACK_NAME' uses MQTTS hostname '$recorded_hostname'; in-place hostname migration to '$MQTTS_HOSTNAME_VALUE' is not supported."
fi
MQTTS_HOSTNAME_VALUE="$recorded_hostname"
fi
[ -n "$MQTTS_HOSTNAME_VALUE" ] || die \
"Ingress MQTTS requires --mqtts-hostname on first apply."
normalize_and_validate_mqtts_hostname
[ "$MQTTS_HOSTNAME_VALUE" != "$HOSTNAME_VALUE" ] || die \
"Ingress web and MQTTS hostnames must be different."
case "$MQTTS_CERTIFICATE_MODE" in
existing)
MQTTS_CERTIFICATE_ARN="${requested_arn:-$recorded_arn}"
[ -n "$MQTTS_CERTIFICATE_ARN" ] || die \
"Ingress MQTTS certificate mode existing requires --mqtts-certificate-arn on first apply."
;;
shared)
[ -z "$requested_arn" ] || die \
"Option --mqtts-certificate-arn cannot be used with shared MQTTS certificate mode."
if [ -z "$shared_certificate_arn" ]; then
if ! shared_certificate_arn="$(kubectl --context "$KUBE_CONTEXT" \
get ingressclass openremote-alb \
-o "go-template={{with .metadata.annotations}}{{with index . \"$SHARED_CERTIFICATE_ARN_ANNOTATION\"}}{{.}}{{end}}{{end}}" \
2>/dev/null)"; then
die "Unable to inspect shared MQTTS certificate configuration on IngressClass 'openremote-alb'."
fi
fi
[ -n "$shared_certificate_arn" ] || die \
"Ingress MQTTS certificate mode shared requires a certificate configured with or-eks-cluster --shared-certificate-arn."
MQTTS_CERTIFICATE_ARN="$shared_certificate_arn"
log "Selected the cluster's shared ACM certificate '$MQTTS_CERTIFICATE_ARN' for MQTTS."
;;
managed)
[ -z "$requested_arn" ] || die \
"Option --mqtts-certificate-arn cannot be used with managed MQTTS certificate mode."
# A recorded managed ARN is loaded and ownership-checked after the
# namespace and hostname reservations have been prepared.
MQTTS_CERTIFICATE_ARN=""
;;
esac
}
record_initial_certificate_mode() {
local namespace_json recorded_mode resource_version
local annotations=("$CERTIFICATE_MODE_ANNOTATION=$CERTIFICATE_MODE")
namespace_json="$(read_certificate_namespace)" || die "Unable to record the stack's certificate mode."
[ -n "$namespace_json" ] || die "Prepared namespace '$STACK_NAME' is missing; unable to record certificate mode."
recorded_mode="$(jq -r '.metadata.annotations["openremote.io/certificate-mode"] // empty' <<< "$namespace_json")"
if [ -n "$recorded_mode" ]; then
[ "$recorded_mode" = "$CERTIFICATE_MODE" ] || die \
"Stack '$STACK_NAME' now uses certificate mode '$recorded_mode'; in-place certificate mode migration is not supported. Retry apply with that mode."
return
fi
resource_version="$(jq -er '.metadata.resourceVersion | select(type == "string" and length > 0)' \
<<< "$namespace_json")" || die "Unable to verify namespace version before recording certificate mode."
if [ "$CERTIFICATE_MODE" = existing ] || [ "$CERTIFICATE_MODE" = shared ]; then
annotations+=("$CERTIFICATE_ARN_ANNOTATION=$CERTIFICATE_ARN")
fi
# Pin the mode before ACM requests or Helm. The conditional write prevents
# two initial applies from choosing different modes. Retain it after failure;
# managed mode without an ARN resumes through existing tagged-ACM discovery.
kubectl --context "$KUBE_CONTEXT" annotate namespace "$STACK_NAME" \
"${annotations[@]}" --overwrite --resource-version="$resource_version" >/dev/null || die \
"Unable to record certificate mode; the namespace may have changed concurrently. Retry apply."
}
record_initial_mqtts_configuration() {
local namespace_json
local recorded_hostname
local recorded_mode
local resource_version
[ "$EXPOSURE" = ingress ] && [ "$MQTTS_ENABLED" = true ] || return 0
namespace_json="$(read_certificate_namespace)" || die \
"Unable to record the stack's MQTTS lifecycle configuration."
[ -n "$namespace_json" ] || die \
"Prepared namespace '$STACK_NAME' is missing; unable to record MQTTS lifecycle configuration."
recorded_mode="$(jq -r '.metadata.annotations["openremote.io/mqtts-certificate-mode"] // empty' \
<<< "$namespace_json")"
recorded_hostname="$(jq -r '.metadata.annotations["openremote.io/mqtts-hostname"] // empty' \
<<< "$namespace_json")"
if [ -n "$recorded_mode" ]; then
[ "$recorded_mode" = "$MQTTS_CERTIFICATE_MODE" ] || die \
"Stack '$STACK_NAME' now uses MQTTS certificate mode '$recorded_mode', not '$MQTTS_CERTIFICATE_MODE'."
[ -n "$recorded_hostname" ] || die \
"Stack '$STACK_NAME' has incomplete MQTTS lifecycle configuration."
[ "$recorded_hostname" = "$MQTTS_HOSTNAME_VALUE" ] || die \
"Stack '$STACK_NAME' now records MQTTS hostname '$recorded_hostname', not '$MQTTS_HOSTNAME_VALUE'."
return
fi
resource_version="$(jq -er '.metadata.resourceVersion | select(type == "string" and length > 0)' \
<<< "$namespace_json")" || die \
"Unable to verify namespace version before recording MQTTS lifecycle configuration."
# Record the facade's ownership mode before ACM requests or Helm. If apply is
# interrupted, destroy can still distinguish this workflow from a low-level
# or-stack endpoint. Existing-mode ARNs are recorded only after Helm succeeds
# so rotation never advances namespace state ahead of the Service. Managed
# mode records its ARN immediately after an owned certificate is resolved.
kubectl --context "$KUBE_CONTEXT" annotate namespace "$STACK_NAME" \
"$MQTTS_CERTIFICATE_MODE_ANNOTATION=$MQTTS_CERTIFICATE_MODE" \
"$MQTTS_HOSTNAME_ANNOTATION=$MQTTS_HOSTNAME_VALUE" \
--overwrite --resource-version="$resource_version" >/dev/null || die \
"Unable to record MQTTS lifecycle configuration; the namespace may have changed concurrently. Retry apply."
}
certificate_name_covers_hostname() {
local certificate_name
local hostname="${2:-$HOSTNAME_VALUE}"
local wildcard_suffix
certificate_name="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')"
certificate_name="${certificate_name%.}"
if [ "$certificate_name" = "$hostname" ]; then
return 0
fi
case "$certificate_name" in
\*.*)
wildcard_suffix="${certificate_name#*.}"
[ "${hostname#*.}" = "$wildcard_suffix" ]
;;
*)
return 1
;;
esac
}
require_selected_mqtts_acm_certificate() {
local certificate_account
local certificate_json
local certificate_name
local certificate_region
local certificate_status
local certificate_tags
local current_account
local describe_status
local hostname_covered=false
local tags_status
if [[ ! "$MQTTS_CERTIFICATE_ARN" =~ ^arn:[^:]+:acm:([^:]+):([0-9]{12}):certificate/[A-Za-z0-9-]+$ ]]; then
die "MQTTS certificate ARN must be a valid ACM certificate ARN."
fi
certificate_region="${BASH_REMATCH[1]}"
certificate_account="${BASH_REMATCH[2]}"
[ "$certificate_region" = "$AWS_REGION_NAME" ] || die \
"MQTTS certificate '$MQTTS_CERTIFICATE_ARN' is in region '$certificate_region', but cluster '$CLUSTER_NAME' is in '$AWS_REGION_NAME'."
current_account="$(aws_sts get-caller-identity --query Account --output text)" || die \
"Unable to determine the EKS AWS account."
[ "$certificate_account" = "$current_account" ] || die \
"MQTTS certificate '$MQTTS_CERTIFICATE_ARN' belongs to account '$certificate_account', but the EKS account is '$current_account'."
set +e
certificate_json="$(aws_acm describe-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
[ "$describe_status" -eq 0 ] || die \
"Unable to describe MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN': $certificate_json"
certificate_status="$(jq -r '.Certificate.Status // empty' <<< "$certificate_json")" || die \
"Unable to read MQTTS ACM certificate metadata."
[ "$certificate_status" = ISSUED ] || die \
"The MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' is '$certificate_status', not ISSUED."
while IFS= read -r certificate_name; do
[ -n "$certificate_name" ] || continue
if certificate_name_covers_hostname "$certificate_name" "$MQTTS_HOSTNAME_VALUE"; then
hostname_covered=true
break
fi
done < <(jq -r \
'[.Certificate.DomainName, .Certificate.SubjectAlternativeNames[]?] | unique[] | select(. != null)' \
<<< "$certificate_json")
[ "$hostname_covered" = true ] || die \
"The MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' does not cover '$MQTTS_HOSTNAME_VALUE'."
set +e
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" \
--output json 2>&1)"
tags_status=$?
set -e
[ "$tags_status" -eq 0 ] || die \
"Unable to inspect ownership tags on MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN': $certificate_tags"
if jq -e \
--arg key "$CERTIFICATE_MANAGED_BY_TAG_KEY" \
--arg value "$CERTIFICATE_MANAGED_BY_TAG_VALUE" \
'.Tags[]? | select(.Key == $key and .Value == $value)' \
<<< "$certificate_tags" >/dev/null; then
if [ "$MQTTS_CERTIFICATE_MODE" = shared ]; then
die "MQTTS certificate '$MQTTS_CERTIFICATE_ARN' is marked as managed by or-eks-stack and cannot be used in shared mode."
fi
die "MQTTS certificate '$MQTTS_CERTIFICATE_ARN' is marked as managed by or-eks-stack and cannot be used as an existing certificate."
fi
if [ "$MQTTS_CERTIFICATE_MODE" = shared ]; then
log "Validated externally managed shared ACM certificate '$MQTTS_CERTIFICATE_ARN' for MQTTS hostname '$MQTTS_HOSTNAME_VALUE'."
else
log "Validated user-owned ACM certificate '$MQTTS_CERTIFICATE_ARN' for MQTTS hostname '$MQTTS_HOSTNAME_VALUE'."
fi
}
require_selected_acm_certificate() {
local certificate_account
local certificate_json
local certificate_name
local certificate_region
local certificate_status
local certificate_tags
local current_account
local describe_status
local hostname_covered=false
local tags_status
if [[ ! "$CERTIFICATE_ARN" =~ ^arn:[^:]+:acm:([^:]+):([0-9]{12}):certificate/[A-Za-z0-9-]+$ ]]; then
die "Certificate ARN must be a valid ACM certificate ARN."
fi
certificate_region="${BASH_REMATCH[1]}"
certificate_account="${BASH_REMATCH[2]}"
[ "$certificate_region" = "$AWS_REGION_NAME" ] || die \
"Certificate '$CERTIFICATE_ARN' is in region '$certificate_region', but cluster '$CLUSTER_NAME' is in '$AWS_REGION_NAME'."
current_account="$(aws_sts get-caller-identity --query Account --output text)" || die \
"Unable to determine the EKS AWS account."
[ "$certificate_account" = "$current_account" ] || die \
"Certificate '$CERTIFICATE_ARN' belongs to account '$certificate_account', but the EKS account is '$current_account'."
set +e
certificate_json="$(aws_acm describe-certificate \
--certificate-arn "$CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
[ "$describe_status" -eq 0 ] || die \
"Unable to describe $CERTIFICATE_MODE ACM certificate '$CERTIFICATE_ARN': $certificate_json"
certificate_status="$(jq -r '.Certificate.Status // empty' \
<<< "$certificate_json")" || die \
"Unable to read $CERTIFICATE_MODE ACM certificate metadata."
[ "$certificate_status" = "ISSUED" ] || die \
"The $CERTIFICATE_MODE ACM certificate '$CERTIFICATE_ARN' is '$certificate_status', not ISSUED."
while IFS= read -r certificate_name; do
[ -n "$certificate_name" ] || continue
if certificate_name_covers_hostname "$certificate_name"; then
hostname_covered=true
break
fi
done < <(jq -r \
'[.Certificate.DomainName, .Certificate.SubjectAlternativeNames[]?] | unique[] | select(. != null)' \
<<< "$certificate_json")
[ "$hostname_covered" = true ] || die \
"The $CERTIFICATE_MODE ACM certificate '$CERTIFICATE_ARN' does not cover '$HOSTNAME_VALUE'."
set +e
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$CERTIFICATE_ARN" \
--output json 2>&1)"
tags_status=$?
set -e
[ "$tags_status" -eq 0 ] || die \
"Unable to inspect ownership tags on $CERTIFICATE_MODE ACM certificate '$CERTIFICATE_ARN': $certificate_tags"
if jq -e \
--arg key "$CERTIFICATE_MANAGED_BY_TAG_KEY" \
--arg value "$CERTIFICATE_MANAGED_BY_TAG_VALUE" \
'.Tags[]? | select(.Key == $key and .Value == $value)' \
<<< "$certificate_tags" >/dev/null; then
die "Certificate '$CERTIFICATE_ARN' is marked as managed by or-eks-stack and cannot be used in $CERTIFICATE_MODE mode."
fi
if [ "$CERTIFICATE_MODE" = "shared" ]; then
log "Validated externally managed shared ACM certificate '$CERTIFICATE_ARN' for '$HOSTNAME_VALUE'."
else
log "Validated user-owned ACM certificate '$CERTIFICATE_ARN' for '$HOSTNAME_VALUE'."
fi
}
managed_certificate_tags_match_for_endpoint() {
local tags_json="$1"
local hostname="$2"
local endpoint="$3"
jq -e \
--arg managed_by_key "$CERTIFICATE_MANAGED_BY_TAG_KEY" \
--arg managed_by_value "$CERTIFICATE_MANAGED_BY_TAG_VALUE" \
--arg cluster_key "$CERTIFICATE_CLUSTER_TAG_KEY" \
--arg cluster_value "$CLUSTER_NAME" \
--arg stack_key "$CERTIFICATE_STACK_TAG_KEY" \
--arg stack_value "$STACK_NAME" \
--arg hostname_key "$CERTIFICATE_HOSTNAME_TAG_KEY" \
--arg hostname_value "$hostname" \
--arg endpoint_key "$CERTIFICATE_ENDPOINT_TAG_KEY" \
--arg endpoint_value "$endpoint" \
'def values_for($key): [.Tags[]? | select(.Key == $key) | .Value];
values_for($managed_by_key) == [$managed_by_value] and
values_for($cluster_key) == [$cluster_value] and
values_for($stack_key) == [$stack_value] and
values_for($hostname_key) == [$hostname_value] and
(values_for($endpoint_key) == [$endpoint_value] or
($endpoint_value == "web" and values_for($endpoint_key) == []))' \
<<< "$tags_json" >/dev/null
}
managed_certificate_tags_match_stack_for_endpoint() {
local tags_json="$1"
local endpoint="$2"
jq -e \
--arg managed_by_key "$CERTIFICATE_MANAGED_BY_TAG_KEY" \
--arg managed_by_value "$CERTIFICATE_MANAGED_BY_TAG_VALUE" \
--arg cluster_key "$CERTIFICATE_CLUSTER_TAG_KEY" \
--arg cluster_value "$CLUSTER_NAME" \
--arg stack_key "$CERTIFICATE_STACK_TAG_KEY" \
--arg stack_value "$STACK_NAME" \
--arg endpoint_key "$CERTIFICATE_ENDPOINT_TAG_KEY" \
--arg endpoint_value "$endpoint" \
'def values_for($key): [.Tags[]? | select(.Key == $key) | .Value];
values_for($managed_by_key) == [$managed_by_value] and
values_for($cluster_key) == [$cluster_value] and
values_for($stack_key) == [$stack_value] and
(values_for($endpoint_key) == [$endpoint_value] or
($endpoint_value == "web" and values_for($endpoint_key) == []))' \
<<< "$tags_json" >/dev/null
}
managed_certificate_tags_match() {
managed_certificate_tags_match_for_endpoint "$1" "$HOSTNAME_VALUE" web
}
managed_certificate_tags_match_stack() {
managed_certificate_tags_match_stack_for_endpoint "$1" web
}
managed_mqtts_certificate_tags_match() {
managed_certificate_tags_match_for_endpoint "$1" "$MQTTS_HOSTNAME_VALUE" mqtts
}
managed_mqtts_certificate_tags_match_stack() {
managed_certificate_tags_match_stack_for_endpoint "$1" mqtts
}
require_managed_certificate_ownership() {
local certificate_account
local certificate_region
local certificate_tags
local current_account
if [[ ! "$CERTIFICATE_ARN" =~ ^arn:[^:]+:acm:([^:]+):([0-9]{12}):certificate/[A-Za-z0-9-]+$ ]]; then
die "Managed certificate ARN '$CERTIFICATE_ARN' is invalid."
fi
certificate_region="${BASH_REMATCH[1]}"
certificate_account="${BASH_REMATCH[2]}"
[ "$certificate_region" = "$AWS_REGION_NAME" ] || die \
"Managed certificate '$CERTIFICATE_ARN' is in region '$certificate_region', not '$AWS_REGION_NAME'."
current_account="$(aws_sts get-caller-identity --query Account --output text)" || die \
"Unable to determine the EKS AWS account."
[ "$certificate_account" = "$current_account" ] || die \
"Managed certificate '$CERTIFICATE_ARN' belongs to account '$certificate_account', not '$current_account'."
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$CERTIFICATE_ARN" \
--output json)" || die \
"Unable to inspect ownership tags on managed certificate '$CERTIFICATE_ARN'."
managed_certificate_tags_match "$certificate_tags" || die \
"Certificate '$CERTIFICATE_ARN' does not carry the exact managed ownership tags for stack '$STACK_NAME' and hostname '$HOSTNAME_VALUE'."
}
discover_managed_certificate() {
local candidate_arn
local candidate_arns=""
local candidate_count=0
local certificate_domain
local certificate_list
local certificate_tagged_hostname
local certificate_tags
certificate_list="$(aws_acm list-certificates \
--certificate-statuses \
PENDING_VALIDATION ISSUED INACTIVE EXPIRED VALIDATION_TIMED_OUT REVOKED FAILED \
--output json)" || die "Unable to list ACM certificates for managed-certificate discovery."
while IFS=$'\t' read -r candidate_arn certificate_domain; do
[ -n "$candidate_arn" ] || continue
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$candidate_arn" \
--output json)" || die \
"Unable to inspect ownership tags on ACM certificate '$candidate_arn'."
managed_certificate_tags_match_stack "$certificate_tags" || continue
certificate_tagged_hostname="$(jq -r \
--arg key "$CERTIFICATE_HOSTNAME_TAG_KEY" \
'.Tags[]? | select(.Key == $key) | .Value' \
<<< "$certificate_tags")"
if ! managed_certificate_tags_match "$certificate_tags" || \
[ "${certificate_domain%.}" != "$HOSTNAME_VALUE" ]; then
die "Managed ACM certificate '$candidate_arn' is already owned by stack '$STACK_NAME' for hostname '${certificate_tagged_hostname:-${certificate_domain%.}}'; refusing to create another certificate for '$HOSTNAME_VALUE'."
fi
candidate_count="$((candidate_count + 1))"
candidate_arns+="${candidate_arns:+$'\n'}$candidate_arn"
done < <(jq -r \
'.CertificateSummaryList[]? | [.CertificateArn, (.DomainName | ascii_downcase)] | @tsv' \
<<< "$certificate_list")
[ "$candidate_count" -le 1 ] || die \
"Found $candidate_count managed ACM certificates for stack '$STACK_NAME' and hostname '$HOSTNAME_VALUE'; refusing to choose one."
if [ "$candidate_count" -eq 1 ]; then
CERTIFICATE_ARN="$candidate_arns"
log "Reusing tagged managed ACM certificate '$CERTIFICATE_ARN'."
fi
}
request_managed_certificate() {
local idempotency_token
idempotency_token="$(printf '%s' \
"$CLUSTER_NAME|$AWS_REGION_NAME|$STACK_NAME|$HOSTNAME_VALUE" | \
cksum | awk '{print "or" $1}')"
log "Requesting a managed ACM certificate for '$HOSTNAME_VALUE'."
CERTIFICATE_ARN="$(aws_acm request-certificate \
--domain-name "$HOSTNAME_VALUE" \
--validation-method DNS \
--idempotency-token "$idempotency_token" \
--tags \
"Key=$CERTIFICATE_MANAGED_BY_TAG_KEY,Value=$CERTIFICATE_MANAGED_BY_TAG_VALUE" \
"Key=$CERTIFICATE_CLUSTER_TAG_KEY,Value=$CLUSTER_NAME" \
"Key=$CERTIFICATE_STACK_TAG_KEY,Value=$STACK_NAME" \
"Key=$CERTIFICATE_HOSTNAME_TAG_KEY,Value=$HOSTNAME_VALUE" \
"Key=$CERTIFICATE_ENDPOINT_TAG_KEY,Value=web" \
--query CertificateArn \
--output text)" || die \
"Unable to request a managed ACM certificate for '$HOSTNAME_VALUE'."
[ -n "$CERTIFICATE_ARN" ] && [ "$CERTIFICATE_ARN" != "None" ] || die \
"ACM did not return a certificate ARN for '$HOSTNAME_VALUE'."
log "Requested managed ACM certificate '$CERTIFICATE_ARN'."
}
stack_certificate_mode() {
kubectl --context "$KUBE_CONTEXT" get namespace "$STACK_NAME" \
-o 'go-template={{with index .metadata.annotations "openremote.io/certificate-mode"}}{{.}}{{end}}'
}
stack_certificate_arn() {
kubectl --context "$KUBE_CONTEXT" get namespace "$STACK_NAME" \
-o 'go-template={{with index .metadata.annotations "openremote.io/certificate-arn"}}{{.}}{{end}}'
}
stack_hostname() {
kubectl --context "$KUBE_CONTEXT" get namespace "$STACK_NAME" \
-o 'go-template={{with index .metadata.annotations "openremote.io/hostname"}}{{.}}{{end}}'
}
stack_mqtts_certificate_mode() {
kubectl --context "$KUBE_CONTEXT" get namespace "$STACK_NAME" \
-o 'go-template={{with index .metadata.annotations "openremote.io/mqtts-certificate-mode"}}{{.}}{{end}}'
}
stack_mqtts_certificate_arn() {
kubectl --context "$KUBE_CONTEXT" get namespace "$STACK_NAME" \
-o 'go-template={{with index .metadata.annotations "openremote.io/mqtts-certificate-arn"}}{{.}}{{end}}'
}
stack_mqtts_hostname() {
kubectl --context "$KUBE_CONTEXT" get namespace "$STACK_NAME" \
-o 'go-template={{with index .metadata.annotations "openremote.io/mqtts-hostname"}}{{.}}{{end}}'
}
inspect_stack_namespace_for_destroy() {
local namespace_identity
local namespace_name
local stack_label
namespace_identity="$(kubectl --context "$KUBE_CONTEXT" \
get namespace "$STACK_NAME" \
--ignore-not-found \
-o 'go-template={{.metadata.name}}|{{with index .metadata.labels "openremote.io/stack"}}{{.}}{{end}}')" || die \
"Unable to inspect namespace '$STACK_NAME' before destroy."
if [ -z "$namespace_identity" ]; then
STACK_NAMESPACE_EXISTS=false
return
fi
IFS='|' read -r namespace_name stack_label <<< "$namespace_identity"
[ "$namespace_name" = "$STACK_NAME" ] || die \
"Unable to identify namespace '$STACK_NAME' before destroy."
[ "$stack_label" = "$STACK_NAME" ] || die \
"Namespace '$STACK_NAME' does not have the matching openremote.io/stack label; refusing to modify it."
STACK_NAMESPACE_EXISTS=true
}
prepare_portable_stack() {
local stack_args=(
prepare
--name "$STACK_NAME" \
--kube-context "$KUBE_CONTEXT" \
--target eks \
--exposure "$EXPOSURE" \
--dns external-dns \
--hostname "$HOSTNAME_VALUE"
)
if [ "$EXPOSURE" = ingress ] && [ "$MQTTS_ENABLED" = true ]; then
stack_args+=(--mqtts --mqtts-hostname "$MQTTS_HOSTNAME_VALUE")
fi
run_or_stack "${stack_args[@]}"
}
resolve_managed_certificate() {
local recorded_certificate_arn
local recorded_certificate_mode
recorded_certificate_mode="$(stack_certificate_mode)" || die "Unable to read the recorded certificate mode."
recorded_certificate_arn="$(stack_certificate_arn)" || die "Unable to read the recorded certificate ARN."
if [ "$recorded_certificate_mode" = "managed" ] && [ -n "$recorded_certificate_arn" ]; then
CERTIFICATE_ARN="$recorded_certificate_arn"
log "Reusing the managed ACM certificate recorded by stack '$STACK_NAME'."
else
discover_managed_certificate
if [ -z "$CERTIFICATE_ARN" ]; then
request_managed_certificate
fi
fi
require_managed_certificate_ownership
record_certificate_configuration
}
require_managed_mqtts_certificate_ownership() {
local certificate_account
local certificate_region
local certificate_tags
local current_account
if [[ ! "$MQTTS_CERTIFICATE_ARN" =~ ^arn:[^:]+:acm:([^:]+):([0-9]{12}):certificate/[A-Za-z0-9-]+$ ]]; then
die "Managed MQTTS certificate ARN '$MQTTS_CERTIFICATE_ARN' is invalid."
fi
certificate_region="${BASH_REMATCH[1]}"
certificate_account="${BASH_REMATCH[2]}"
[ "$certificate_region" = "$AWS_REGION_NAME" ] || die \
"Managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN' is in region '$certificate_region', not '$AWS_REGION_NAME'."
current_account="$(aws_sts get-caller-identity --query Account --output text)" || die \
"Unable to determine the EKS AWS account."
[ "$certificate_account" = "$current_account" ] || die \
"Managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN' belongs to account '$certificate_account', not '$current_account'."
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" \
--output json)" || die \
"Unable to inspect ownership tags on managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN'."
managed_mqtts_certificate_tags_match "$certificate_tags" || die \
"Certificate '$MQTTS_CERTIFICATE_ARN' does not carry the exact managed MQTTS ownership tags for stack '$STACK_NAME' and hostname '$MQTTS_HOSTNAME_VALUE'."
}
discover_managed_mqtts_certificate() {
local candidate_arn
local candidate_arns=""
local candidate_count=0
local certificate_domain
local certificate_list
local certificate_tagged_hostname
local certificate_tags
certificate_list="$(aws_acm list-certificates \
--certificate-statuses \
PENDING_VALIDATION ISSUED INACTIVE EXPIRED VALIDATION_TIMED_OUT REVOKED FAILED \
--output json)" || die \
"Unable to list ACM certificates for managed MQTTS certificate discovery."
while IFS=$'\t' read -r candidate_arn certificate_domain; do
[ -n "$candidate_arn" ] || continue
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$candidate_arn" \
--output json)" || die \
"Unable to inspect ownership tags on ACM certificate '$candidate_arn'."
managed_mqtts_certificate_tags_match_stack "$certificate_tags" || continue
certificate_tagged_hostname="$(jq -r \
--arg key "$CERTIFICATE_HOSTNAME_TAG_KEY" \
'.Tags[]? | select(.Key == $key) | .Value' \
<<< "$certificate_tags")"
if ! managed_mqtts_certificate_tags_match "$certificate_tags" || \
[ "${certificate_domain%.}" != "$MQTTS_HOSTNAME_VALUE" ]; then
die "Managed MQTTS ACM certificate '$candidate_arn' is already owned by stack '$STACK_NAME' for hostname '${certificate_tagged_hostname:-${certificate_domain%.}}'; refusing to create another certificate for '$MQTTS_HOSTNAME_VALUE'."
fi
candidate_count="$((candidate_count + 1))"
candidate_arns+="${candidate_arns:+$'\n'}$candidate_arn"
done < <(jq -r \
'.CertificateSummaryList[]? | [.CertificateArn, (.DomainName | ascii_downcase)] | @tsv' \
<<< "$certificate_list")
[ "$candidate_count" -le 1 ] || die \
"Found $candidate_count managed MQTTS ACM certificates for stack '$STACK_NAME' and hostname '$MQTTS_HOSTNAME_VALUE'; refusing to choose one."
if [ "$candidate_count" -eq 1 ]; then
MQTTS_CERTIFICATE_ARN="$candidate_arns"
log "Reusing tagged managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN'."
fi
}
request_managed_mqtts_certificate() {
local idempotency_token
idempotency_token="$(printf '%s' \
"$CLUSTER_NAME|$AWS_REGION_NAME|$STACK_NAME|mqtts|$MQTTS_HOSTNAME_VALUE" | \
cksum | awk '{print "or" $1}')"
log "Requesting a managed MQTTS ACM certificate for '$MQTTS_HOSTNAME_VALUE'."
MQTTS_CERTIFICATE_ARN="$(aws_acm request-certificate \
--domain-name "$MQTTS_HOSTNAME_VALUE" \
--validation-method DNS \
--idempotency-token "$idempotency_token" \
--tags \
"Key=$CERTIFICATE_MANAGED_BY_TAG_KEY,Value=$CERTIFICATE_MANAGED_BY_TAG_VALUE" \
"Key=$CERTIFICATE_CLUSTER_TAG_KEY,Value=$CLUSTER_NAME" \
"Key=$CERTIFICATE_STACK_TAG_KEY,Value=$STACK_NAME" \
"Key=$CERTIFICATE_HOSTNAME_TAG_KEY,Value=$MQTTS_HOSTNAME_VALUE" \
"Key=$CERTIFICATE_ENDPOINT_TAG_KEY,Value=mqtts" \
--query CertificateArn \
--output text)" || die \
"Unable to request a managed MQTTS ACM certificate for '$MQTTS_HOSTNAME_VALUE'."
[ -n "$MQTTS_CERTIFICATE_ARN" ] && [ "$MQTTS_CERTIFICATE_ARN" != "None" ] || die \
"ACM did not return a certificate ARN for '$MQTTS_HOSTNAME_VALUE'."
log "Requested managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN'."
}
resolve_managed_mqtts_certificate() {
local recorded_certificate_arn
local recorded_certificate_mode
recorded_certificate_mode="$(stack_mqtts_certificate_mode)" || die \
"Unable to read the recorded MQTTS certificate mode."
recorded_certificate_arn="$(stack_mqtts_certificate_arn)" || die \
"Unable to read the recorded MQTTS certificate ARN."
[ "$recorded_certificate_mode" = managed ] || die \
"Stack '$STACK_NAME' no longer records managed MQTTS certificate mode."
if [ -n "$recorded_certificate_arn" ]; then
MQTTS_CERTIFICATE_ARN="$recorded_certificate_arn"
log "Reusing the managed MQTTS ACM certificate recorded by stack '$STACK_NAME'."
else
discover_managed_mqtts_certificate
if [ -z "$MQTTS_CERTIFICATE_ARN" ]; then
request_managed_mqtts_certificate
fi
fi
require_managed_mqtts_certificate_ownership
record_mqtts_certificate_configuration
}
require_actionable_managed_mqtts_certificate_status() {
local certificate_status="$1"
case "$certificate_status" in
PENDING_VALIDATION|ISSUED) ;;
*)
die "Managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' is '$certificate_status' and cannot be reconciled automatically."
;;
esac
}
wait_for_managed_mqtts_certificate_validation_records() {
local certificate_json
local certificate_status="unknown"
local deadline
local describe_status
local last_error=""
local next_notice
local now
local timeout_seconds
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for ACM to publish MQTTS DNS validation records."
while [ "$now" -lt "$deadline" ]; do
set +e
certificate_json="$(aws_acm describe-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
if [ "$describe_status" -eq 0 ]; then
certificate_status="$(jq -r '.Certificate.Status // empty' \
<<< "$certificate_json")" || die \
"Unable to read managed MQTTS ACM certificate metadata."
require_actionable_managed_mqtts_certificate_status "$certificate_status"
jq -e \
--arg hostname "$MQTTS_HOSTNAME_VALUE" \
'(.Certificate.DomainName | ascii_downcase) == $hostname and
([.Certificate.DomainValidationOptions[]?] | length) > 0 and
([.Certificate.DomainValidationOptions[]?] |
all((.ResourceRecord.Name // "") != "" and
(.ResourceRecord.Type // "") != "" and
(.ResourceRecord.Value // "") != ""))' \
<<< "$certificate_json" >/dev/null && {
MQTTS_CERTIFICATE_JSON="$certificate_json"
return
}
last_error="ACM has not published every MQTTS validation record yet"
else
last_error="$certificate_json"
fi
if [ "$now" -ge "$next_notice" ]; then
log "ACM MQTTS validation records are still pending."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "ACM did not publish DNS validation records for MQTTS certificate '$MQTTS_CERTIFICATE_ARN' within '$TIMEOUT': $last_error"
}
validate_managed_mqtts_certificate_records() {
local record_name
local record_target
local record_type
while IFS=$'\t' read -r record_name record_type record_target; do
[ -n "$record_name" ] && [ -n "$record_target" ] || die \
"ACM returned an incomplete DNS validation record for MQTTS certificate '$MQTTS_CERTIFICATE_ARN'."
[ "$record_type" = CNAME ] || die \
"ACM returned unsupported MQTTS validation record type '$record_type'."
validation_record_is_in_managed_domain "$record_name" || die \
"ACM MQTTS validation record '$record_name' is outside the configured ExternalDNS domains."
done < <(jq -r \
'.Certificate.DomainValidationOptions[]?.ResourceRecord |
[.Name, .Type, .Value] | @tsv' <<< "$MQTTS_CERTIFICATE_JSON")
}
require_owned_mqtts_validation_dns_endpoint() {
local existing_certificate_arn
local existing_endpoint_kind
local existing_endpoint
local existing_managed_by
# Only NotFound permits creation; a failed read cannot establish ownership.
existing_endpoint="$(kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
get dnsendpoint "$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME" \
--ignore-not-found -o json)" || die \
"Unable to inspect DNSEndpoint '$STACK_NAME/$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME'; refusing to write ACM MQTTS validation records without ownership checks."
[ -n "$existing_endpoint" ] || return 0
existing_managed_by="$(jq -r \
'.metadata.labels["app.kubernetes.io/managed-by"] // empty' \
<<< "$existing_endpoint")"
existing_certificate_arn="$(jq -r \
'.metadata.annotations["openremote.io/certificate-arn"] // empty' \
<<< "$existing_endpoint")"
existing_endpoint_kind="$(jq -r \
'.metadata.annotations["openremote.io/endpoint"] // empty' \
<<< "$existing_endpoint")"
[ "$existing_managed_by" = "$CERTIFICATE_MANAGED_BY_TAG_VALUE" ] && \
[ "$existing_certificate_arn" = "$MQTTS_CERTIFICATE_ARN" ] && \
[ "$existing_endpoint_kind" = mqtts ] || die \
"DNSEndpoint '$STACK_NAME/$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME' is not owned by managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN'; refusing to replace it."
}
apply_managed_mqtts_certificate_validation_records() {
local endpoint_manifest
validate_managed_mqtts_certificate_records
require_owned_mqtts_validation_dns_endpoint
endpoint_manifest="$(jq -c \
--arg name "$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME" \
--arg namespace "$STACK_NAME" \
--arg certificate_arn "$MQTTS_CERTIFICATE_ARN" \
--arg managed_by "$CERTIFICATE_MANAGED_BY_TAG_VALUE" \
'{
apiVersion: "externaldns.k8s.io/v1alpha1",
kind: "DNSEndpoint",
metadata: {
name: $name,
namespace: $namespace,
labels: {
"app.kubernetes.io/managed-by": $managed_by,
"openremote.io/managed-dns": "true"
},
annotations: {
"openremote.io/certificate-arn": $certificate_arn,
"openremote.io/endpoint": "mqtts"
}
},
spec: {
endpoints: ([.Certificate.DomainValidationOptions[]?.ResourceRecord | {
dnsName: .Name,
recordType: .Type,
recordTTL: 300,
targets: [.Value]
}] | unique_by(.dnsName, .recordType, .targets[0]))
}
}' <<< "$MQTTS_CERTIFICATE_JSON")" || die \
"Unable to render ACM MQTTS validation records as a DNSEndpoint."
log "Reconciling ACM MQTTS DNS validation records in '$STACK_NAME/$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME'."
kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
apply -f - <<< "$endpoint_manifest" >/dev/null || die \
"Unable to apply ACM MQTTS validation DNSEndpoint '$STACK_NAME/$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME'."
}
wait_for_managed_mqtts_certificate() {
local certificate_status="unknown"
local deadline
local next_notice
local now
local timeout_seconds
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' to be issued."
while [ "$now" -lt "$deadline" ]; do
certificate_status="$(aws_acm describe-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" \
--query 'Certificate.Status' \
--output text 2>/dev/null || true)"
if [ "$certificate_status" = ISSUED ]; then
log "Managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' is issued."
return
fi
if [ -n "$certificate_status" ] && [ "$certificate_status" != None ]; then
require_actionable_managed_mqtts_certificate_status "$certificate_status"
fi
if [ "$now" -ge "$next_notice" ]; then
log "Managed MQTTS ACM certificate validation is still pending."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "Managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' remained '$certificate_status' beyond '$TIMEOUT'. The validation DNSEndpoint is retained for a later retry."
}
discover_managed_certificate_for_destroy() {
local certificate_arn
local certificate_count=0
local certificate_list
local certificate_tags
local discovered_certificate_arn=""
certificate_list="$(aws_acm list-certificates \
--certificate-statuses \
PENDING_VALIDATION ISSUED INACTIVE EXPIRED VALIDATION_TIMED_OUT REVOKED FAILED \
--output json)" || die \
"Unable to check for stack-managed ACM certificates before destroy."
while IFS= read -r certificate_arn; do
[ -n "$certificate_arn" ] || continue
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$certificate_arn" \
--output json)" || die \
"Unable to inspect ownership tags on ACM certificate '$certificate_arn'."
if managed_certificate_tags_match_stack "$certificate_tags"; then
certificate_count="$((certificate_count + 1))"
discovered_certificate_arn="$certificate_arn"
fi
done < <(jq -r '.CertificateSummaryList[]?.CertificateArn' \
<<< "$certificate_list")
[ "$certificate_count" -le 1 ] || die \
"Found $certificate_count managed ACM certificates for stack '$STACK_NAME'; refusing destructive cleanup."
if [ "$certificate_count" -eq 1 ]; then
printf '%s\n' "$discovered_certificate_arn"
fi
}
load_managed_certificate_for_destroy() {
local certificate_domain
local certificate_json
local certificate_tags
local describe_status
local tagged_hostname
set +e
certificate_json="$(aws_acm describe-certificate \
--certificate-arn "$CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
if [ "$describe_status" -ne 0 ]; then
case "$certificate_json" in
*ResourceNotFoundException*)
CERTIFICATE_EXISTS=false
CERTIFICATE_JSON=""
return
;;
*)
die "Unable to describe managed ACM certificate '$CERTIFICATE_ARN' before destroy: $certificate_json"
;;
esac
fi
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$CERTIFICATE_ARN" \
--output json)" || die \
"Unable to inspect ownership tags on managed certificate '$CERTIFICATE_ARN'."
managed_certificate_tags_match_stack "$certificate_tags" || die \
"Certificate '$CERTIFICATE_ARN' does not carry the exact managed ownership tags for stack '$STACK_NAME'."
tagged_hostname="$(jq -r \
--arg key "$CERTIFICATE_HOSTNAME_TAG_KEY" \
'[.Tags[]? | select(.Key == $key) | .Value] |
if length == 1 then .[0] else empty end' \
<<< "$certificate_tags")" || die \
"Unable to read the managed hostname tag from certificate '$CERTIFICATE_ARN'."
[ -n "$tagged_hostname" ] || die \
"Certificate '$CERTIFICATE_ARN' does not have exactly one managed hostname tag."
if [ -n "$HOSTNAME_VALUE" ] && [ "$HOSTNAME_VALUE" != "$tagged_hostname" ]; then
die "Namespace '$STACK_NAME' records hostname '$HOSTNAME_VALUE', but its managed certificate records '$tagged_hostname'."
fi
HOSTNAME_VALUE="$tagged_hostname"
validate_hostname
certificate_domain="$(jq -r '.Certificate.DomainName // empty | ascii_downcase' \
<<< "$certificate_json")" || die \
"Unable to read the domain of managed certificate '$CERTIFICATE_ARN'."
[ "${certificate_domain%.}" = "$HOSTNAME_VALUE" ] || die \
"Managed certificate '$CERTIFICATE_ARN' is for '$certificate_domain', not '$HOSTNAME_VALUE'."
require_managed_certificate_ownership
CERTIFICATE_EXISTS=true
CERTIFICATE_JSON="$certificate_json"
}
capture_stack_ingress_for_destroy() {
STACK_INGRESS_JSON=""
[ "$STACK_NAMESPACE_EXISTS" = true ] || return 0
STACK_INGRESS_JSON="$(kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
get ingress proxy \
--ignore-not-found \
-o json)" || die \
"Unable to inspect Ingress '$STACK_NAME/proxy' before destroy."
}
resolve_managed_certificate_for_destroy() {
local discovered_certificate_arn=""
local validation_endpoint
local recorded_certificate_arn=""
local recorded_certificate_mode=""
local recorded_hostname=""
MANAGED_CERTIFICATE_DESTROY=false
if [ "$STACK_NAMESPACE_EXISTS" = true ]; then
recorded_certificate_mode="$(stack_certificate_mode)" || die \
"Unable to read certificate mode from namespace '$STACK_NAME'."
recorded_certificate_arn="$(stack_certificate_arn)" || die \
"Unable to read certificate ARN from namespace '$STACK_NAME'."
recorded_hostname="$(stack_hostname)" || die \
"Unable to read hostname from namespace '$STACK_NAME'."
[ -z "$recorded_hostname" ] || [ "$recorded_hostname" = "$HOSTNAME_VALUE" ] || die \
"Namespace '$STACK_NAME' records hostname '$recorded_hostname', but reserves '$HOSTNAME_VALUE'; refusing destructive cleanup."
fi
if ! discovered_certificate_arn="$(discover_managed_certificate_for_destroy)"; then
die "Unable to safely discover a unique managed ACM certificate for stack '$STACK_NAME'."
fi
if [ "$recorded_certificate_mode" = "managed" ]; then
if [ -n "$recorded_certificate_arn" ] && \
[ -n "$discovered_certificate_arn" ] && \
[ "$recorded_certificate_arn" != "$discovered_certificate_arn" ]; then
die "Namespace '$STACK_NAME' records managed certificate '$recorded_certificate_arn', but ownership tags identify '$discovered_certificate_arn'."
fi
CERTIFICATE_ARN="${recorded_certificate_arn:-$discovered_certificate_arn}"
if [ -z "$CERTIFICATE_ARN" ]; then
# First apply may have stopped after recording mode but before requesting
# ACM. With no tagged certificate or validation resource, ordinary stack
# cleanup is sufficient; do not strand the namespace and hostname claim.
validation_endpoint="$(kubectl --context "$KUBE_CONTEXT" --namespace "$STACK_NAME" \
get dnsendpoint "$ACM_VALIDATION_DNS_ENDPOINT_NAME" --ignore-not-found -o json)" || die \
"Unable to inspect validation resources for the incomplete managed certificate configuration."
[ -z "$validation_endpoint" ] || die \
"Stack '$STACK_NAME' has validation resources without an identifiable managed certificate; refusing destructive cleanup."
return 0
fi
elif [ -n "$discovered_certificate_arn" ]; then
CERTIFICATE_ARN="$discovered_certificate_arn"
log "Recovered tagged managed ACM certificate '$CERTIFICATE_ARN' for stack '$STACK_NAME'."
else
return 0
fi
load_managed_certificate_for_destroy
validate_hostname
MANAGED_CERTIFICATE_DESTROY=true
}
discover_managed_mqtts_certificate_for_destroy() {
local certificate_arn
local certificate_count=0
local certificate_list
local certificate_tags
local discovered_certificate_arn=""
certificate_list="$(aws_acm list-certificates \
--certificate-statuses \
PENDING_VALIDATION ISSUED INACTIVE EXPIRED VALIDATION_TIMED_OUT REVOKED FAILED \
--output json)" || die \
"Unable to check for stack-managed MQTTS ACM certificates before destroy."
while IFS= read -r certificate_arn; do
[ -n "$certificate_arn" ] || continue
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$certificate_arn" \
--output json)" || die \
"Unable to inspect ownership tags on ACM certificate '$certificate_arn'."
if managed_mqtts_certificate_tags_match_stack "$certificate_tags"; then
certificate_count="$((certificate_count + 1))"
discovered_certificate_arn="$certificate_arn"
fi
done < <(jq -r '.CertificateSummaryList[]?.CertificateArn' \
<<< "$certificate_list")
[ "$certificate_count" -le 1 ] || die \
"Found $certificate_count managed MQTTS ACM certificates for stack '$STACK_NAME'; refusing destructive cleanup."
if [ "$certificate_count" -eq 1 ]; then
printf '%s\n' "$discovered_certificate_arn"
fi
}
load_managed_mqtts_certificate_for_destroy() {
local certificate_domain
local certificate_json
local certificate_tags
local describe_status
local tagged_hostname
set +e
certificate_json="$(aws_acm describe-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
if [ "$describe_status" -ne 0 ]; then
case "$certificate_json" in
*ResourceNotFoundException*)
MQTTS_CERTIFICATE_EXISTS=false
MQTTS_CERTIFICATE_JSON=""
return
;;
*)
die "Unable to describe managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' before destroy: $certificate_json"
;;
esac
fi
certificate_tags="$(aws_acm list-tags-for-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" \
--output json)" || die \
"Unable to inspect ownership tags on managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN'."
managed_mqtts_certificate_tags_match_stack "$certificate_tags" || die \
"Certificate '$MQTTS_CERTIFICATE_ARN' does not carry the exact managed MQTTS ownership tags for stack '$STACK_NAME'."
tagged_hostname="$(jq -r \
--arg key "$CERTIFICATE_HOSTNAME_TAG_KEY" \
'[.Tags[]? | select(.Key == $key) | .Value] |
if length == 1 then .[0] else empty end' \
<<< "$certificate_tags")" || die \
"Unable to read the managed hostname tag from MQTTS certificate '$MQTTS_CERTIFICATE_ARN'."
[ -n "$tagged_hostname" ] || die \
"MQTTS certificate '$MQTTS_CERTIFICATE_ARN' does not have exactly one managed hostname tag."
if [ -n "$MQTTS_HOSTNAME_VALUE" ] && \
[ "$MQTTS_HOSTNAME_VALUE" != "$tagged_hostname" ]; then
die "Namespace '$STACK_NAME' records MQTTS hostname '$MQTTS_HOSTNAME_VALUE', but its managed MQTTS certificate records '$tagged_hostname'."
fi
MQTTS_HOSTNAME_VALUE="$tagged_hostname"
normalize_and_validate_mqtts_hostname
certificate_domain="$(jq -r '.Certificate.DomainName // empty | ascii_downcase' \
<<< "$certificate_json")" || die \
"Unable to read the domain of managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN'."
[ "${certificate_domain%.}" = "$MQTTS_HOSTNAME_VALUE" ] || die \
"Managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN' is for '$certificate_domain', not '$MQTTS_HOSTNAME_VALUE'."
require_managed_mqtts_certificate_ownership
MQTTS_CERTIFICATE_EXISTS=true
MQTTS_CERTIFICATE_JSON="$certificate_json"
}
resolve_managed_mqtts_certificate_for_destroy() {
local discovered_certificate_arn=""
local recorded_certificate_arn="$MQTTS_CERTIFICATE_ARN"
local recorded_certificate_mode="$MQTTS_CERTIFICATE_MODE"
local validation_endpoint
MANAGED_MQTTS_CERTIFICATE_DESTROY=false
if [ "$STACK_NAMESPACE_EXISTS" = true ]; then
recorded_certificate_mode="$(stack_mqtts_certificate_mode)" || die \
"Unable to read MQTTS certificate mode from namespace '$STACK_NAME'."
recorded_certificate_arn="$(stack_mqtts_certificate_arn)" || die \
"Unable to read MQTTS certificate ARN from namespace '$STACK_NAME'."
fi
if ! discovered_certificate_arn="$(discover_managed_mqtts_certificate_for_destroy)"; then
die "Unable to safely discover a unique managed MQTTS ACM certificate for stack '$STACK_NAME'."
fi
if [ "$recorded_certificate_mode" = managed ]; then
if [ -n "$recorded_certificate_arn" ] && \
[ -n "$discovered_certificate_arn" ] && \
[ "$recorded_certificate_arn" != "$discovered_certificate_arn" ]; then
die "Namespace '$STACK_NAME' records managed MQTTS certificate '$recorded_certificate_arn', but ownership tags identify '$discovered_certificate_arn'."
fi
MQTTS_CERTIFICATE_ARN="${recorded_certificate_arn:-$discovered_certificate_arn}"
if [ -z "$MQTTS_CERTIFICATE_ARN" ]; then
validation_endpoint="$(kubectl --context "$KUBE_CONTEXT" --namespace "$STACK_NAME" \
get dnsendpoint "$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME" \
--ignore-not-found -o json)" || die \
"Unable to inspect validation resources for the incomplete managed MQTTS certificate configuration."
[ -z "$validation_endpoint" ] || die \
"Stack '$STACK_NAME' has MQTTS validation resources without an identifiable managed certificate; refusing destructive cleanup."
return 0
fi
elif [ -n "$discovered_certificate_arn" ]; then
[ -z "$recorded_certificate_mode" ] || die \
"Stack '$STACK_NAME' records MQTTS certificate mode '$recorded_certificate_mode', but also owns managed MQTTS certificate '$discovered_certificate_arn'."
MQTTS_CERTIFICATE_ARN="$discovered_certificate_arn"
log "Recovered tagged managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' for stack '$STACK_NAME'."
else
return 0
fi
load_managed_mqtts_certificate_for_destroy
MQTTS_CERTIFICATE_MODE=managed
MANAGED_MQTTS_CERTIFICATE_DESTROY=true
}
capture_mqtts_validation_dns_records_for_destroy() {
local certificate_records_json='[]'
local endpoint_certificate_arn
local endpoint_kind
local endpoint_managed_by
local endpoint_managed_dns
local endpoint_records_json='[]'
local validation_endpoint=""
MQTTS_VALIDATION_DNS_ENDPOINT_EXISTS=false
MQTTS_VALIDATION_DNS_RECORDS=""
if [ "$MQTTS_CERTIFICATE_EXISTS" = true ]; then
certificate_records_json="$(jq -c \
'[.Certificate.DomainValidationOptions[]?.ResourceRecord |
select((.Name // "") != "" and (.Type // "") != "" and
(.Value // "") != "") | {
dnsName: .Name,
recordType: .Type,
targets: [.Value]
}] | unique_by(.dnsName, .recordType, .targets[0]) |
sort_by(.dnsName, .recordType, .targets[0])' \
<<< "$MQTTS_CERTIFICATE_JSON")" || die \
"Unable to read validation records from managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN'."
fi
if [ "$STACK_NAMESPACE_EXISTS" = true ]; then
validation_endpoint="$(kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
get dnsendpoint "$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME" \
--ignore-not-found -o json)" || die \
"Unable to inspect DNSEndpoint '$STACK_NAME/$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME' before destroy."
fi
if [ -n "$validation_endpoint" ]; then
MQTTS_VALIDATION_DNS_ENDPOINT_EXISTS=true
endpoint_managed_by="$(jq -r \
'.metadata.labels["app.kubernetes.io/managed-by"] // empty' \
<<< "$validation_endpoint")"
endpoint_managed_dns="$(jq -r \
'.metadata.labels["openremote.io/managed-dns"] // empty' \
<<< "$validation_endpoint")"
endpoint_certificate_arn="$(jq -r \
'.metadata.annotations["openremote.io/certificate-arn"] // empty' \
<<< "$validation_endpoint")"
endpoint_kind="$(jq -r \
'.metadata.annotations["openremote.io/endpoint"] // empty' \
<<< "$validation_endpoint")"
[ "$endpoint_managed_by" = "$CERTIFICATE_MANAGED_BY_TAG_VALUE" ] && \
[ "$endpoint_managed_dns" = true ] && \
[ "$endpoint_certificate_arn" = "$MQTTS_CERTIFICATE_ARN" ] && \
[ "$endpoint_kind" = mqtts ] || die \
"DNSEndpoint '$STACK_NAME/$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME' does not exactly match managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN'; refusing to delete it."
endpoint_records_json="$(jq -c \
'[.spec.endpoints[]? | {
dnsName: (.dnsName // ""),
recordType: (.recordType // ""),
targets: ((.targets // []) | sort)
}] | unique_by(.dnsName, .recordType, .targets[0]) |
sort_by(.dnsName, .recordType, .targets[0])' \
<<< "$validation_endpoint")" || die \
"Unable to read records from DNSEndpoint '$STACK_NAME/$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME'."
[ "$(jq 'length' <<< "$endpoint_records_json")" -gt 0 ] || die \
"DNSEndpoint '$STACK_NAME/$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME' has no validation records."
if [ "$MQTTS_CERTIFICATE_EXISTS" = true ] && \
[ "$endpoint_records_json" != "$certificate_records_json" ]; then
die "DNSEndpoint '$STACK_NAME/$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME' does not contain exactly the records returned by ACM; refusing destructive cleanup."
fi
elif [ "$MQTTS_CERTIFICATE_EXISTS" = false ]; then
die "Managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN' and its validation DNSEndpoint are both absent; validation-record ownership cannot be verified."
fi
if [ "$MQTTS_VALIDATION_DNS_ENDPOINT_EXISTS" = true ]; then
MQTTS_VALIDATION_DNS_RECORDS="$(jq -r \
'.[] | [.dnsName, .recordType] | @tsv' \
<<< "$endpoint_records_json")"
else
MQTTS_VALIDATION_DNS_RECORDS="$(jq -r \
'.[] | [.dnsName, .recordType] | @tsv' \
<<< "$certificate_records_json")"
fi
while IFS=$'\t' read -r record_name record_type; do
[ -n "$record_name" ] || continue
[ "$record_type" = CNAME ] || die \
"Managed MQTTS validation record '$record_name' has unsupported type '$record_type'."
validation_record_is_in_managed_domain "$record_name" || die \
"Managed MQTTS validation record '$record_name' is outside the configured ExternalDNS domains."
done <<< "$MQTTS_VALIDATION_DNS_RECORDS"
}
validate_stack_ingress_for_managed_destroy() {
local ingress_certificate_arn
local ingress_hostname
local managed_dns_label
[ -n "$STACK_INGRESS_JSON" ] || return 0
managed_dns_label="$(jq -r \
'.metadata.labels["openremote.io/managed-dns"] // empty' \
<<< "$STACK_INGRESS_JSON")"
ingress_hostname="$(jq -r \
'.metadata.annotations["external-dns.kubernetes.io/hostname"] // empty' <<< "$STACK_INGRESS_JSON")"
ingress_certificate_arn="$(jq -r \
'.metadata.annotations["alb.ingress.kubernetes.io/certificate-arn"] // empty' \
<<< "$STACK_INGRESS_JSON")"
[ "$managed_dns_label" = "true" ] || die \
"Ingress '$STACK_NAME/proxy' is not marked for OpenRemote-managed DNS; refusing to delete it."
[ "$ingress_hostname" = "$HOSTNAME_VALUE" ] || die \
"Ingress '$STACK_NAME/proxy' records hostname '$ingress_hostname', not '$HOSTNAME_VALUE'."
[ "$ingress_certificate_arn" = "$CERTIFICATE_ARN" ] || die \
"Ingress '$STACK_NAME/proxy' uses certificate '$ingress_certificate_arn', not owned certificate '$CERTIFICATE_ARN'."
}
capture_validation_dns_records_for_destroy() {
local certificate_records_json='[]'
local endpoint_certificate_arn
local endpoint_managed_by
local endpoint_managed_dns
local endpoint_records_json='[]'
local validation_endpoint=""
VALIDATION_DNS_ENDPOINT_EXISTS=false
VALIDATION_DNS_RECORDS=""
if [ "$CERTIFICATE_EXISTS" = true ]; then
certificate_records_json="$(jq -c \
'[.Certificate.DomainValidationOptions[]?.ResourceRecord |
select((.Name // "") != "" and (.Type // "") != "" and
(.Value // "") != "") | {
dnsName: .Name,
recordType: .Type,
targets: [.Value]
}] | unique_by(.dnsName, .recordType, .targets[0]) |
sort_by(.dnsName, .recordType, .targets[0])' \
<<< "$CERTIFICATE_JSON")" || die \
"Unable to read validation records from managed certificate '$CERTIFICATE_ARN'."
fi
if [ "$STACK_NAMESPACE_EXISTS" = true ]; then
validation_endpoint="$(kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
get dnsendpoint "$ACM_VALIDATION_DNS_ENDPOINT_NAME" \
--ignore-not-found \
-o json)" || die \
"Unable to inspect DNSEndpoint '$STACK_NAME/$ACM_VALIDATION_DNS_ENDPOINT_NAME' before destroy."
fi
if [ -n "$validation_endpoint" ]; then
VALIDATION_DNS_ENDPOINT_EXISTS=true
endpoint_managed_by="$(jq -r \
'.metadata.labels["app.kubernetes.io/managed-by"] // empty' \
<<< "$validation_endpoint")"
endpoint_managed_dns="$(jq -r \
'.metadata.labels["openremote.io/managed-dns"] // empty' \
<<< "$validation_endpoint")"
endpoint_certificate_arn="$(jq -r \
'.metadata.annotations["openremote.io/certificate-arn"] // empty' \
<<< "$validation_endpoint")"
[ "$endpoint_managed_by" = "$CERTIFICATE_MANAGED_BY_TAG_VALUE" ] && \
[ "$endpoint_managed_dns" = "true" ] && \
[ "$endpoint_certificate_arn" = "$CERTIFICATE_ARN" ] || die \
"DNSEndpoint '$STACK_NAME/$ACM_VALIDATION_DNS_ENDPOINT_NAME' does not exactly match managed certificate '$CERTIFICATE_ARN'; refusing to delete it."
endpoint_records_json="$(jq -c \
'[.spec.endpoints[]? | {
dnsName: (.dnsName // ""),
recordType: (.recordType // ""),
targets: ((.targets // []) | sort)
}] | unique_by(.dnsName, .recordType, .targets[0]) |
sort_by(.dnsName, .recordType, .targets[0])' \
<<< "$validation_endpoint")" || die \
"Unable to read records from DNSEndpoint '$STACK_NAME/$ACM_VALIDATION_DNS_ENDPOINT_NAME'."
[ "$(jq 'length' <<< "$endpoint_records_json")" -gt 0 ] || die \
"DNSEndpoint '$STACK_NAME/$ACM_VALIDATION_DNS_ENDPOINT_NAME' has no validation records."
if [ "$CERTIFICATE_EXISTS" = true ] && \
[ "$endpoint_records_json" != "$certificate_records_json" ]; then
die "DNSEndpoint '$STACK_NAME/$ACM_VALIDATION_DNS_ENDPOINT_NAME' does not contain exactly the records returned by ACM; refusing destructive cleanup."
fi
elif [ "$CERTIFICATE_EXISTS" = false ]; then
die "Managed certificate '$CERTIFICATE_ARN' and its validation DNSEndpoint are both absent; validation-record ownership cannot be verified."
fi
if [ "$VALIDATION_DNS_ENDPOINT_EXISTS" = true ]; then
VALIDATION_DNS_RECORDS="$(jq -r \
'.[] | [.dnsName, .recordType] | @tsv' \
<<< "$endpoint_records_json")"
else
VALIDATION_DNS_RECORDS="$(jq -r \
'.[] | [.dnsName, .recordType] | @tsv' \
<<< "$certificate_records_json")"
fi
while IFS=$'\t' read -r record_name record_type; do
[ -n "$record_name" ] || continue
[ "$record_type" = "CNAME" ] || die \
"Managed validation record '$record_name' has unsupported type '$record_type'."
validation_record_is_in_managed_domain "$record_name" || die \
"Managed validation record '$record_name' is outside the configured ExternalDNS domains."
done <<< "$VALIDATION_DNS_RECORDS"
}
require_actionable_managed_certificate_status() {
local certificate_status="$1"
case "$certificate_status" in
PENDING_VALIDATION|ISSUED)
;;
*)
die "Managed ACM certificate '$CERTIFICATE_ARN' is '$certificate_status' and cannot be reconciled automatically."
;;
esac
}
wait_for_managed_certificate_validation_records() {
local certificate_json
local certificate_status="unknown"
local deadline
local describe_status
local last_error=""
local next_notice
local now
local timeout_seconds
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for ACM to publish DNS validation records."
while [ "$now" -lt "$deadline" ]; do
set +e
certificate_json="$(aws_acm describe-certificate \
--certificate-arn "$CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
if [ "$describe_status" -eq 0 ]; then
certificate_status="$(jq -r '.Certificate.Status // empty' \
<<< "$certificate_json")" || die \
"Unable to read managed ACM certificate metadata."
require_actionable_managed_certificate_status "$certificate_status"
jq -e \
--arg hostname "$HOSTNAME_VALUE" \
'(.Certificate.DomainName | ascii_downcase) == $hostname and
([.Certificate.DomainValidationOptions[]?] | length) > 0 and
([.Certificate.DomainValidationOptions[]?] |
all((.ResourceRecord.Name // "") != "" and
(.ResourceRecord.Type // "") != "" and
(.ResourceRecord.Value // "") != ""))' \
<<< "$certificate_json" >/dev/null && {
CERTIFICATE_JSON="$certificate_json"
return
}
last_error="ACM has not published every validation record yet"
else
last_error="$certificate_json"
fi
if [ "$now" -ge "$next_notice" ]; then
log "ACM validation records are still pending."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "ACM did not publish DNS validation records for '$CERTIFICATE_ARN' within '$TIMEOUT': $last_error"
}
validation_record_is_in_managed_domain() {
local domain_filter
local record_name="$1"
while IFS= read -r domain_filter; do
[ -n "$domain_filter" ] || continue
if dns_name_is_covered "$record_name" "$domain_filter"; then
return 0
fi
done < <(sed -n 's/^--domain-filter=//p' <<< "$EXTERNAL_DNS_ARGS")
return 1
}
validate_managed_certificate_records() {
local record_name
local record_target
local record_type
while IFS=$'\t' read -r record_name record_type record_target; do
[ -n "$record_name" ] && [ -n "$record_target" ] || die \
"ACM returned an incomplete DNS validation record for '$CERTIFICATE_ARN'."
[ "$record_type" = "CNAME" ] || die \
"ACM returned unsupported validation record type '$record_type'."
validation_record_is_in_managed_domain "$record_name" || die \
"ACM validation record '$record_name' is outside the configured ExternalDNS domains."
done < <(jq -r \
'.Certificate.DomainValidationOptions[]?.ResourceRecord |
[.Name, .Type, .Value] | @tsv' <<< "$CERTIFICATE_JSON")
}
require_owned_validation_dns_endpoint() {
local existing_endpoint
local existing_certificate_arn
local existing_managed_by
# Only NotFound permits creation; a failed read cannot establish ownership.
existing_endpoint="$(kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
get dnsendpoint "$ACM_VALIDATION_DNS_ENDPOINT_NAME" \
--ignore-not-found -o json)" || die \
"Unable to inspect DNSEndpoint '$STACK_NAME/$ACM_VALIDATION_DNS_ENDPOINT_NAME'; refusing to write ACM validation records without ownership checks."
[ -n "$existing_endpoint" ] || return 0
existing_managed_by="$(jq -r \
'.metadata.labels["app.kubernetes.io/managed-by"] // empty' \
<<< "$existing_endpoint")"
existing_certificate_arn="$(jq -r \
'.metadata.annotations["openremote.io/certificate-arn"] // empty' \
<<< "$existing_endpoint")"
[ "$existing_managed_by" = "$CERTIFICATE_MANAGED_BY_TAG_VALUE" ] && \
[ "$existing_certificate_arn" = "$CERTIFICATE_ARN" ] || die \
"DNSEndpoint '$STACK_NAME/$ACM_VALIDATION_DNS_ENDPOINT_NAME' is not owned by managed certificate '$CERTIFICATE_ARN'; refusing to replace it."
}
apply_managed_certificate_validation_records() {
local endpoint_manifest
validate_managed_certificate_records
require_owned_validation_dns_endpoint
endpoint_manifest="$(jq -c \
--arg name "$ACM_VALIDATION_DNS_ENDPOINT_NAME" \
--arg namespace "$STACK_NAME" \
--arg certificate_arn "$CERTIFICATE_ARN" \
--arg managed_by "$CERTIFICATE_MANAGED_BY_TAG_VALUE" \
'{
apiVersion: "externaldns.k8s.io/v1alpha1",
kind: "DNSEndpoint",
metadata: {
name: $name,
namespace: $namespace,
labels: {
"app.kubernetes.io/managed-by": $managed_by,
"openremote.io/managed-dns": "true"
},
annotations: {
"openremote.io/certificate-arn": $certificate_arn
}
},
spec: {
endpoints: ([.Certificate.DomainValidationOptions[]?.ResourceRecord | {
dnsName: .Name,
recordType: .Type,
recordTTL: 300,
targets: [.Value]
}] | unique_by(.dnsName, .recordType, .targets[0]))
}
}' <<< "$CERTIFICATE_JSON")" || die \
"Unable to render ACM validation records as a DNSEndpoint."
log "Reconciling ACM DNS validation records in '$STACK_NAME/$ACM_VALIDATION_DNS_ENDPOINT_NAME'."
kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
apply -f - <<< "$endpoint_manifest" >/dev/null || die \
"Unable to apply ACM validation DNSEndpoint '$STACK_NAME/$ACM_VALIDATION_DNS_ENDPOINT_NAME'."
}
wait_for_managed_certificate() {
local certificate_status="unknown"
local deadline
local next_notice
local now
local timeout_seconds
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for managed ACM certificate '$CERTIFICATE_ARN' to be issued."
while [ "$now" -lt "$deadline" ]; do
certificate_status="$(aws_acm describe-certificate \
--certificate-arn "$CERTIFICATE_ARN" \
--query 'Certificate.Status' \
--output text 2>/dev/null || true)"
if [ "$certificate_status" = "ISSUED" ]; then
log "Managed ACM certificate '$CERTIFICATE_ARN' is issued."
return
fi
if [ -n "$certificate_status" ] && [ "$certificate_status" != "None" ]; then
require_actionable_managed_certificate_status "$certificate_status"
fi
if [ "$now" -ge "$next_notice" ]; then
log "Managed ACM certificate validation is still pending."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "Managed ACM certificate '$CERTIFICATE_ARN' remained '$certificate_status' beyond '$TIMEOUT'. The validation DNSEndpoint is retained for a later retry."
}
apply_portable_stack() {
local stack_args=(
apply
--name "$STACK_NAME"
--kube-context "$KUBE_CONTEXT"
--target eks
--exposure "$EXPOSURE"
--dns external-dns
--hostname "$HOSTNAME_VALUE"
--timeout "$TIMEOUT"
)
if [ "$EXPOSURE" = "haproxy" ]; then
stack_args+=(--http-port "$HTTP_PORT" --https-port "$HTTPS_PORT")
if [ "$MQTTS_ENABLED" = true ]; then
stack_args+=(--mqtts --mqtts-port "$MQTTS_PORT")
fi
else
stack_args+=(--certificate-arn "$CERTIFICATE_ARN")
if [ "$MQTTS_ENABLED" = true ]; then
stack_args+=(
--mqtts
--mqtts-port "$MQTTS_PORT"
--mqtts-hostname "$MQTTS_HOSTNAME_VALUE"
--mqtts-certificate-arn "$MQTTS_CERTIFICATE_ARN"
)
fi
fi
if [ -n "$VALUES_DIR" ]; then
stack_args+=(--values-dir "$VALUES_DIR")
fi
run_or_stack "${stack_args[@]}"
}
record_certificate_configuration() {
local certificate_mode=acme
if [ "$EXPOSURE" = "ingress" ]; then
certificate_mode="$CERTIFICATE_MODE"
kubectl --context "$KUBE_CONTEXT" annotate namespace "$STACK_NAME" \
"$CERTIFICATE_MODE_ANNOTATION=$certificate_mode" \
"$CERTIFICATE_ARN_ANNOTATION=$CERTIFICATE_ARN" \
"$HOSTNAME_ANNOTATION=$HOSTNAME_VALUE" \
--overwrite >/dev/null
else
kubectl --context "$KUBE_CONTEXT" annotate namespace "$STACK_NAME" \
"$CERTIFICATE_MODE_ANNOTATION=$certificate_mode" \
"$HOSTNAME_ANNOTATION=$HOSTNAME_VALUE" \
--overwrite >/dev/null
fi
}
record_mqtts_certificate_configuration() {
[ "$EXPOSURE" = ingress ] && [ "$MQTTS_ENABLED" = true ] || return 0
kubectl --context "$KUBE_CONTEXT" annotate namespace "$STACK_NAME" \
"$MQTTS_CERTIFICATE_MODE_ANNOTATION=$MQTTS_CERTIFICATE_MODE" \
"$MQTTS_CERTIFICATE_ARN_ANNOTATION=$MQTTS_CERTIFICATE_ARN" \
"$MQTTS_HOSTNAME_ANNOTATION=$MQTTS_HOSTNAME_VALUE" \
--overwrite >/dev/null || die \
"Unable to record the Ingress MQTTS certificate configuration."
}
wait_for_load_balancer() {
local deadline
local load_balancer_kind
local next_notice
local now
local resource_kind
local timeout_seconds
if [ "$EXPOSURE" = "ingress" ]; then
load_balancer_kind="shared ALB"
resource_kind="ingress"
else
load_balancer_kind="stack NLB"
resource_kind="service"
fi
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
LOAD_BALANCER_ADDRESS=""
log "Waiting for the $load_balancer_kind address."
while [ "$now" -lt "$deadline" ]; do
LOAD_BALANCER_ADDRESS="$(kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
get "$resource_kind" proxy \
-o 'jsonpath={.status.loadBalancer.ingress[0].ip}{.status.loadBalancer.ingress[0].hostname}' \
2>/dev/null || true)"
if [ -n "$LOAD_BALANCER_ADDRESS" ]; then
log "The $load_balancer_kind is '$LOAD_BALANCER_ADDRESS'."
return
fi
if [ "$now" -ge "$next_notice" ]; then
log "The $load_balancer_kind address is still pending."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "The proxy $resource_kind did not receive a $load_balancer_kind address within '$TIMEOUT'; inspect '$STACK_NAME/proxy' and the AWS Load Balancer Controller."
}
wait_for_mqtts_load_balancer() {
local deadline
local next_notice
local now
local timeout_seconds
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
MQTTS_LOAD_BALANCER_ADDRESS=""
log "Waiting for the stack MQTTS NLB address."
while [ "$now" -lt "$deadline" ]; do
MQTTS_LOAD_BALANCER_ADDRESS="$(kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
get service proxy-mqtts \
-o 'jsonpath={.status.loadBalancer.ingress[0].ip}{.status.loadBalancer.ingress[0].hostname}' \
2>/dev/null || true)"
if [ -n "$MQTTS_LOAD_BALANCER_ADDRESS" ]; then
log "The stack MQTTS NLB is '$MQTTS_LOAD_BALANCER_ADDRESS'."
return
fi
if [ "$now" -ge "$next_notice" ]; then
log "The stack MQTTS NLB address is still pending."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "Service '$STACK_NAME/proxy-mqtts' did not receive an NLB address within '$TIMEOUT'; inspect the Service and the AWS Load Balancer Controller."
}
resolve_addresses() {
local dns_name="$1"
{
dig +short "$dns_name" A 2>/dev/null || true
dig +short "$dns_name" AAAA 2>/dev/null || true
} | awk '/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ || index($0, ":") > 0 { print }' | sort -u
}
addresses_intersect() {
local first_addresses="$1"
local second_addresses="$2"
local address
while IFS= read -r address; do
[ -n "$address" ] || continue
if grep -Fx -- "$address" <<< "$second_addresses" >/dev/null; then
return 0
fi
done <<< "$first_addresses"
return 1
}
wait_for_dns() {
local deadline
local hostname_addresses
local load_balancer_address="$1"
local hostname="${2:-$HOSTNAME_VALUE}"
local load_balancer_addresses
local next_notice
local now
local timeout_seconds
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for '$hostname' to resolve to the current load balancer '$load_balancer_address'."
while [ "$now" -lt "$deadline" ]; do
hostname_addresses="$(resolve_addresses "$hostname")"
if [[ "$load_balancer_address" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || \
[[ "$load_balancer_address" == *:* ]]; then
load_balancer_addresses="$load_balancer_address"
else
load_balancer_addresses="$(resolve_addresses "$load_balancer_address")"
fi
if [ -n "$hostname_addresses" ] && [ -n "$load_balancer_addresses" ] && \
addresses_intersect "$hostname_addresses" "$load_balancer_addresses"; then
log "DNS now resolves to the current load balancer."
return
fi
if [ "$now" -ge "$next_notice" ]; then
log "DNS is still pending or points to another load balancer."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "DNS '$hostname' did not resolve to the current load balancer within '$TIMEOUT'. Inspect deployment/$EXTERNAL_DNS_RELEASE_NAME logs and allow for resolver negative caching before retrying."
}
select_managed_dns_domain() {
local domain_filter
local hostname="${1:-$HOSTNAME_VALUE}"
MANAGED_DNS_DOMAIN=""
while IFS= read -r domain_filter; do
domain_filter="${domain_filter%.}"
[ -n "$domain_filter" ] || continue
if dns_name_is_covered "$hostname" "$domain_filter" && \
[ "${#domain_filter}" -gt "${#MANAGED_DNS_DOMAIN}" ]; then
MANAGED_DNS_DOMAIN="$domain_filter"
fi
done < <(sed -n 's/^--domain-filter=//p' <<< "$EXTERNAL_DNS_ARGS")
[ -n "$MANAGED_DNS_DOMAIN" ] || die \
"Unable to select an ExternalDNS domain for '$hostname'."
}
discover_authoritative_name_servers() {
local candidate_domain="$MANAGED_DNS_DOMAIN"
local name_servers
AUTHORITATIVE_NAME_SERVERS=""
while [[ "$candidate_domain" == *.* ]]; do
name_servers="$(dig +time=5 +tries=2 +short "$candidate_domain" NS)" || die \
"Unable to discover authoritative name servers for '$candidate_domain'."
name_servers="$(awk \
'/^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?\.?$/ { print }' \
<<< "$name_servers" | sort -u)"
if [ -n "$name_servers" ]; then
AUTHORITATIVE_NAME_SERVERS="$name_servers"
MANAGED_DNS_DOMAIN="$candidate_domain"
log "Using authoritative DNS for '$MANAGED_DNS_DOMAIN' while verifying record removal."
return
fi
candidate_domain="${candidate_domain#*.}"
done
die "Unable to find authoritative name servers for managed DNS domain '$MANAGED_DNS_DOMAIN'."
}
authoritative_dns_record_exists() {
local answer
local name_server
local query_status
local query_failed=false
local record_name="$1"
local record_type="$2"
while IFS= read -r name_server; do
[ -n "$name_server" ] || continue
set +e
answer="$(dig +time=3 +tries=1 +norecurse +noall +comments +answer +authority \
"@$name_server" "$record_name" "$record_type" 2>/dev/null)"
query_status=$?
set -e
# A successful dig process can carry SERVFAIL/REFUSED or a referral. Those
# are not evidence of deletion and must never allow a hostname to be reused.
if [ "$query_status" -ne 0 ] ||
! grep -Eq '^;; ->>HEADER<<-.*status: (NOERROR|NXDOMAIN),' <<< "$answer" ||
! grep -Eq '^;; flags:([^;]* )?aa[ ;]' <<< "$answer"; then
query_failed=true
continue
fi
if awk -v type="$record_type" '!/^;/ && NF >= 5 && ($4 == type || $4 == "CNAME") { found=1 } END { exit !found }' \
<<< "$answer"; then
return 0
fi
done <<< "$AUTHORITATIVE_NAME_SERVERS"
[ "$query_failed" = false ] || return 2
return 1
}
wait_for_dns_records_removed() {
local deadline
local description="$1"
local next_notice
local now
local query_status
local record_name
local record_type
local records="$2"
local records_remain
local timeout_seconds
[ -n "$records" ] || return 0
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for $description DNS records to be removed."
while [ "$now" -lt "$deadline" ]; do
records_remain=false
while IFS=$'\t' read -r record_name record_type; do
[ -n "$record_name" ] || continue
if authoritative_dns_record_exists "$record_name" "$record_type"; then
query_status=0
else
query_status=$?
fi
if [ "$query_status" -ne 1 ]; then
records_remain=true
break
fi
done <<< "$records"
if [ "$records_remain" = false ]; then
log "The $description DNS records are removed."
return
fi
if [ "$now" -ge "$next_notice" ]; then
log "The $description DNS records are still present or could not be verified."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "The $description DNS records were not confirmed removed within '$TIMEOUT'. External resources were retained so destroy can be retried safely."
}
remove_managed_stack_ingress() {
if [ -z "$STACK_INGRESS_JSON" ]; then
log "Ingress '$STACK_NAME/proxy' is already absent."
return
fi
log "Deleting Ingress '$STACK_NAME/proxy' before managed certificate cleanup."
kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
delete ingress proxy \
--wait=true \
--timeout "$TIMEOUT" || die \
"Unable to delete Ingress '$STACK_NAME/proxy'."
STACK_INGRESS_JSON=""
}
wait_for_managed_certificate_detachment() {
local certificate_json
local deadline
local describe_status
local in_use_count
local last_error=""
local next_notice
local now
local timeout_seconds
[ "$CERTIFICATE_EXISTS" = true ] || return 0
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for managed ACM certificate '$CERTIFICATE_ARN' to be detached."
while [ "$now" -lt "$deadline" ]; do
set +e
certificate_json="$(aws_acm describe-certificate \
--certificate-arn "$CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
if [ "$describe_status" -eq 0 ]; then
in_use_count="$(jq -r '.Certificate.InUseBy // [] | length' \
<<< "$certificate_json")" || die \
"Unable to read attachment state for managed certificate '$CERTIFICATE_ARN'."
if [ "$in_use_count" -eq 0 ]; then
CERTIFICATE_JSON="$certificate_json"
log "Managed ACM certificate '$CERTIFICATE_ARN' is detached."
return
fi
last_error="certificate is still attached to $in_use_count AWS resource(s)"
elif [[ "$certificate_json" == *ResourceNotFoundException* ]]; then
CERTIFICATE_EXISTS=false
CERTIFICATE_JSON=""
log "Managed ACM certificate '$CERTIFICATE_ARN' is already absent."
return
else
last_error="$certificate_json"
fi
if [ "$now" -ge "$next_notice" ]; then
log "Managed ACM certificate detachment is still pending."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "Managed ACM certificate '$CERTIFICATE_ARN' did not detach within '$TIMEOUT': $last_error"
}
delete_managed_certificate() {
local delete_output
local delete_status
local describe_output
local describe_status
local deadline
local next_notice
local now
local timeout_seconds
[ "$CERTIFICATE_EXISTS" = true ] || return 0
require_managed_certificate_ownership
log "Deleting managed ACM certificate '$CERTIFICATE_ARN'."
set +e
delete_output="$(aws_acm delete-certificate \
--certificate-arn "$CERTIFICATE_ARN" 2>&1)"
delete_status=$?
set -e
if [ "$delete_status" -ne 0 ]; then
case "$delete_output" in
*ResourceNotFoundException*)
CERTIFICATE_EXISTS=false
log "Managed ACM certificate '$CERTIFICATE_ARN' is already absent."
return
;;
*)
die "Unable to delete managed ACM certificate '$CERTIFICATE_ARN': $delete_output"
;;
esac
fi
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
while [ "$now" -lt "$deadline" ]; do
set +e
describe_output="$(aws_acm describe-certificate \
--certificate-arn "$CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
if [ "$describe_status" -ne 0 ] && \
[[ "$describe_output" == *ResourceNotFoundException* ]]; then
CERTIFICATE_EXISTS=false
log "Managed ACM certificate '$CERTIFICATE_ARN' was deleted."
return
fi
if [ "$now" -ge "$next_notice" ]; then
log "Managed ACM certificate deletion is still propagating."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "Deletion of managed ACM certificate '$CERTIFICATE_ARN' was not confirmed within '$TIMEOUT'."
}
wait_for_managed_mqtts_certificate_detachment() {
local certificate_json
local deadline
local describe_status
local in_use_count
local last_error=""
local next_notice
local now
local timeout_seconds
[ "$MQTTS_CERTIFICATE_EXISTS" = true ] || return 0
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' to be detached."
while [ "$now" -lt "$deadline" ]; do
set +e
certificate_json="$(aws_acm describe-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
if [ "$describe_status" -eq 0 ]; then
in_use_count="$(jq -r '.Certificate.InUseBy // [] | length' \
<<< "$certificate_json")" || die \
"Unable to read attachment state for managed MQTTS certificate '$MQTTS_CERTIFICATE_ARN'."
if [ "$in_use_count" -eq 0 ]; then
MQTTS_CERTIFICATE_JSON="$certificate_json"
log "Managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' is detached."
return
fi
last_error="certificate is still attached to $in_use_count AWS resource(s)"
elif [[ "$certificate_json" == *ResourceNotFoundException* ]]; then
MQTTS_CERTIFICATE_EXISTS=false
MQTTS_CERTIFICATE_JSON=""
log "Managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' is already absent."
return
else
last_error="$certificate_json"
fi
if [ "$now" -ge "$next_notice" ]; then
log "Managed MQTTS ACM certificate detachment is still pending."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "Managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' did not detach within '$TIMEOUT': $last_error"
}
delete_managed_mqtts_certificate() {
local delete_output
local delete_status
local describe_output
local describe_status
local deadline
local next_notice
local now
local timeout_seconds
[ "$MQTTS_CERTIFICATE_EXISTS" = true ] || return 0
require_managed_mqtts_certificate_ownership
log "Deleting managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN'."
set +e
delete_output="$(aws_acm delete-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" 2>&1)"
delete_status=$?
set -e
if [ "$delete_status" -ne 0 ]; then
case "$delete_output" in
*ResourceNotFoundException*)
MQTTS_CERTIFICATE_EXISTS=false
log "Managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' is already absent."
return
;;
*)
die "Unable to delete managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN': $delete_output"
;;
esac
fi
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
while [ "$now" -lt "$deadline" ]; do
set +e
describe_output="$(aws_acm describe-certificate \
--certificate-arn "$MQTTS_CERTIFICATE_ARN" \
--output json 2>&1)"
describe_status=$?
set -e
if [ "$describe_status" -ne 0 ] && \
[[ "$describe_output" == *ResourceNotFoundException* ]]; then
MQTTS_CERTIFICATE_EXISTS=false
log "Managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' was deleted."
return
fi
if [ "$now" -ge "$next_notice" ]; then
log "Managed MQTTS ACM certificate deletion is still propagating."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "Deletion of managed MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' was not confirmed within '$TIMEOUT'."
}
public_manager_url() {
printf 'https://%s' "$HOSTNAME_VALUE"
if [ "$EXPOSURE" = "haproxy" ] && [ "$HTTPS_PORT" != "443" ]; then
printf ':%s' "$HTTPS_PORT"
fi
printf '/manager/'
}
endpoint_is_ready() {
local endpoint_url="$1"
curl --disable --fail --silent --show-error --location \
--proto '=https' \
--proto-redir '=https' \
--max-redirs 5 \
--connect-timeout 10 \
--max-time 30 \
--output /dev/null \
"$endpoint_url"
}
proxy_certificate_state() {
local state
state="$(kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" exec deployment/proxy --container proxy \
-- sh -c '
lineage="$LE_DIR/live/$1"
if [ -s "$lineage/privkey.pem" ] && [ -s "$lineage/fullchain.pem" ]; then
printf "present\n"
elif [ -e "$lineage" ] || [ -L "$lineage" ]; then
printf "incomplete\n"
else
printf "absent\n"
fi
' sh "$HOSTNAME_VALUE")" || die \
"Unable to inspect HAProxy certificate state; refusing to infer that issuance is needed."
case "$state" in
present|incomplete|absent) printf '%s\n' "$state" ;;
*) die "Unexpected HAProxy certificate state '$state'; inspect deployment/proxy before retrying." ;;
esac
}
trigger_acme() {
local state
# Manager errors and connection failures say nothing about persisted TLS
# state. add rejects an existing lineage; renew would force fresh issuance.
# Leave existing certificates to the proxy's normal renewal lifecycle.
state="$(proxy_certificate_state)" || die "Unable to determine whether HAProxy needs ACME issuance."
case "$state" in
present)
log "Reusing HAProxy certificate state for '$HOSTNAME_VALUE'."
return
;;
incomplete)
die "HAProxy certificate state for '$HOSTNAME_VALUE' is incomplete. Inspect its persisted letsencrypt directory and proxy logs before retrying; refusing to overwrite existing state."
;;
esac
log "Triggering HAProxy ACME issuance for '$HOSTNAME_VALUE'."
if kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
exec deployment/proxy \
--container proxy \
-- /entrypoint.sh add "$HOSTNAME_VALUE"; then
return
fi
# Startup cert_init runs concurrently and may have created the certificate
# after our first inspection. Only accept the failure if both files now exist;
# trusted TLS and Manager readiness are still checked independently below.
state="$(proxy_certificate_state)" || die "Unable to inspect HAProxy state after ACME issuance failed."
if [ "$state" = present ]; then
log "HAProxy certificate state appeared during issuance; waiting for trusted HTTPS."
return
fi
die "HAProxy ACME issuance failed (certificate state: $state). DNS is ready; inspect deployment/proxy logs and retry apply."
}
wait_for_https() {
local deadline
local endpoint_url="$1"
local next_notice
local now
local timeout_seconds
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for a trusted certificate and Manager response at '$endpoint_url'."
while [ "$now" -lt "$deadline" ]; do
if endpoint_is_ready "$endpoint_url" 2>/dev/null; then
log "Trusted HTTPS is ready."
return
fi
if [ "$now" -ge "$next_notice" ]; then
log "The trusted HTTPS endpoint is still pending."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
if [ "$EXPOSURE" = "ingress" ]; then
die "Trusted HTTPS did not become ready at '$endpoint_url' within '$TIMEOUT'. Inspect ingress '$STACK_NAME/proxy', the shared ALB, and its ACM listener certificate."
fi
die "Trusted HTTPS did not become ready at '$endpoint_url' within '$TIMEOUT'. Inspect deployment/proxy logs and its persisted certificate state."
}
mqtts_endpoint_is_ready() {
local hostname
local probe_timeout="$1"
hostname="$HOSTNAME_VALUE"
if [ "$EXPOSURE" = ingress ]; then
hostname="$MQTTS_HOSTNAME_VALUE"
fi
# s_client can print "Verify return code: 0" without completing TLS, and SNI
# alone does not verify the hostname. Use the TLS API's verified handshake;
# no MQTT exchange is needed and peer shutdown after the handshake is harmless.
python3 - "$hostname" "$MQTTS_PORT" "$probe_timeout" <<'PY'
import signal
import socket
import ssl
import sys
def timed_out(signum, frame):
raise TimeoutError("MQTTS TLS probe timed out")
# Bound DNS resolution as well as connection/handshake on the Unix platforms
# supported by these scripts, without requiring GNU timeout on macOS.
signal.signal(signal.SIGALRM, timed_out)
signal.alarm(int(sys.argv[3]))
try:
# Pass resolved paths explicitly: macOS LibreSSL can ignore SSL_CERT_FILE
# when loading defaults, even though get_default_verify_paths resolves it.
trust = ssl.get_default_verify_paths()
context = ssl.create_default_context(cafile=trust.cafile, capath=trust.capath)
with socket.create_connection((sys.argv[1], int(sys.argv[2]))) as connection:
with context.wrap_socket(connection, server_hostname=sys.argv[1]):
pass
except (OSError, ValueError):
sys.exit(1)
finally:
signal.alarm(0)
PY
}
wait_for_mqtts() {
local deadline
local next_notice
local now
local timeout_seconds
local probe_timeout
local hostname="$HOSTNAME_VALUE"
local service_name=proxy
if [ "$EXPOSURE" = ingress ]; then
hostname="$MQTTS_HOSTNAME_VALUE"
service_name=proxy-mqtts
fi
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for trusted MQTTS at '$hostname:$MQTTS_PORT'."
while [ "$now" -lt "$deadline" ]; do
probe_timeout="$((deadline - now))"
[ "$probe_timeout" -le 10 ] || probe_timeout=10
if mqtts_endpoint_is_ready "$probe_timeout"; then
log "Trusted MQTTS is ready."
return
fi
now="$(date +%s)"
[ "$now" -lt "$deadline" ] || break
if [ "$now" -ge "$next_notice" ]; then
log "The trusted MQTTS endpoint is still pending."
next_notice="$((now + 30))"
fi
probe_timeout="$((deadline - now))"
[ "$probe_timeout" -le 5 ] || probe_timeout=5
sleep "$probe_timeout"
now="$(date +%s)"
done
die "Trusted MQTTS did not become ready at '$hostname:$MQTTS_PORT' within '$TIMEOUT'. Inspect service '$STACK_NAME/$service_name' and deployment/proxy logs."
}
apply_stack() {
local endpoint_url
require_command helm
require_command dig
require_command curl
if [ "$MQTTS_ENABLED" = true ]; then
require_command python3
fi
[ -x "$OR_STACK_COMMAND" ] || die \
"Portable stack command '$OR_STACK_COMMAND' is missing or not executable."
if [ "$EXPOSURE" = "ingress" ]; then
require_command jq
fi
require_eks_target
if [ "$EXPOSURE" = "ingress" ]; then
resolve_ingress_certificate
resolve_ingress_mqtts_configuration
if [ "$CERTIFICATE_MODE" = managed ] || \
[ "$MQTTS_CERTIFICATE_MODE" = managed ]; then
require_command cksum
fi
if [ "$CERTIFICATE_MODE" = managed ]; then
[ "${#HOSTNAME_VALUE}" -le 64 ] || die \
"Managed ACM mode currently requires a hostname of at most 64 characters; use an existing or shared certificate for a longer name."
fi
if [ "$MQTTS_CERTIFICATE_MODE" = managed ]; then
[ "${#MQTTS_HOSTNAME_VALUE}" -le 64 ] || die \
"Managed MQTTS ACM mode currently requires a hostname of at most 64 characters; use an existing or shared certificate for a longer name."
fi
fi
require_managed_external_dns
require_stack_hostname_subdomain
if [ "$EXPOSURE" = ingress ] && [ "$MQTTS_ENABLED" = true ]; then
require_stack_hostname_subdomain "$MQTTS_HOSTNAME_VALUE"
fi
if [ "$EXPOSURE" = "ingress" ] && [ "$CERTIFICATE_MODE" != "managed" ]; then
require_selected_acm_certificate
fi
if [ "$EXPOSURE" = ingress ] && [ "$MQTTS_ENABLED" = true ] && \
[ "$MQTTS_CERTIFICATE_MODE" != managed ]; then
require_selected_mqtts_acm_certificate
fi
# Reserve before ACM requests as well as before Helm, so a losing concurrent
# apply cannot issue certificates or install a competing public endpoint.
prepare_portable_stack || die "Namespace preparation or hostname reservation failed; no workloads were installed."
if [ "$EXPOSURE" = "ingress" ]; then
record_initial_certificate_mode
record_initial_mqtts_configuration
fi
if [ "$CERTIFICATE_MODE" = "managed" ]; then
resolve_managed_certificate
wait_for_managed_certificate_validation_records
apply_managed_certificate_validation_records
wait_for_managed_certificate
fi
if [ "$MQTTS_CERTIFICATE_MODE" = managed ]; then
resolve_managed_mqtts_certificate
wait_for_managed_mqtts_certificate_validation_records
apply_managed_mqtts_certificate_validation_records
wait_for_managed_mqtts_certificate
fi
apply_portable_stack
if [ "$MQTTS_CERTIFICATE_MODE" != managed ]; then
record_mqtts_certificate_configuration
fi
if [ "$CERTIFICATE_MODE" != "managed" ]; then
record_certificate_configuration
fi
wait_for_load_balancer
wait_for_dns "$LOAD_BALANCER_ADDRESS"
if [ "$EXPOSURE" = ingress ] && [ "$MQTTS_ENABLED" = true ]; then
wait_for_mqtts_load_balancer
wait_for_dns "$MQTTS_LOAD_BALANCER_ADDRESS" "$MQTTS_HOSTNAME_VALUE"
fi
endpoint_url="$(public_manager_url)"
if [ "$EXPOSURE" = "haproxy" ]; then
trigger_acme
fi
wait_for_https "$endpoint_url"
if [ "$MQTTS_ENABLED" = true ]; then
wait_for_mqtts
fi
log "Stack '$STACK_NAME' is ready at:"
printf ' %s\n' "$endpoint_url"
if [ "$MQTTS_ENABLED" = true ]; then
if [ "$EXPOSURE" = ingress ]; then
printf ' mqtts://%s:%s\n' "$MQTTS_HOSTNAME_VALUE" "$MQTTS_PORT"
else
printf ' mqtts://%s:%s\n' "$HOSTNAME_VALUE" "$MQTTS_PORT"
fi
fi
log "Retrieve its Manager login at any time with:"
printf ' %q credentials --name %q --cluster %q --region %q' \
"$0" "$STACK_NAME" "$CLUSTER_NAME" "$AWS_REGION_NAME"
if [ -n "$AWS_PROFILE_NAME" ]; then
printf ' --profile %q' "$AWS_PROFILE_NAME"
fi
printf '\n'
}
show_certificate_configuration() {
local certificate_arn
local certificate_mode
local certificate_status
local mqtts_certificate_arn
local mqtts_certificate_mode
local mqtts_certificate_status
local mqtts_hostname
certificate_mode="$(stack_certificate_mode 2>/dev/null || true)"
certificate_arn="$(stack_certificate_arn 2>/dev/null || true)"
log "Certificate ownership"
printf 'Certificate mode: %s\n' "${certificate_mode:-unrecorded}"
if [ -n "$certificate_arn" ]; then
printf 'Certificate ARN: %s\n' "$certificate_arn"
fi
if { [ "$certificate_mode" = "existing" ] || \
[ "$certificate_mode" = "shared" ] || \
[ "$certificate_mode" = "managed" ]; } && \
[ -n "$certificate_arn" ]; then
if certificate_status="$(aws_acm describe-certificate \
--certificate-arn "$certificate_arn" \
--query 'Certificate.Status' \
--output text 2>/dev/null)"; then
printf 'Certificate status: %s\n' "$certificate_status"
else
warn "Unable to read the current status of $certificate_mode ACM certificate '$certificate_arn'."
fi
fi
mqtts_certificate_mode="$(stack_mqtts_certificate_mode 2>/dev/null || true)"
mqtts_certificate_arn="$(stack_mqtts_certificate_arn 2>/dev/null || true)"
mqtts_hostname="$(stack_mqtts_hostname 2>/dev/null || true)"
if [ -n "$mqtts_certificate_mode" ] || [ -n "$mqtts_certificate_arn" ] || \
[ -n "$mqtts_hostname" ]; then
printf 'MQTTS certificate mode: %s\n' "${mqtts_certificate_mode:-unrecorded}"
[ -z "$mqtts_hostname" ] || printf 'MQTTS hostname: %s\n' "$mqtts_hostname"
[ -z "$mqtts_certificate_arn" ] || printf 'MQTTS certificate ARN: %s\n' "$mqtts_certificate_arn"
if { [ "$mqtts_certificate_mode" = existing ] || \
[ "$mqtts_certificate_mode" = shared ] || \
[ "$mqtts_certificate_mode" = managed ]; } && \
[ -n "$mqtts_certificate_arn" ]; then
if mqtts_certificate_status="$(aws_acm describe-certificate \
--certificate-arn "$mqtts_certificate_arn" \
--query 'Certificate.Status' \
--output text 2>/dev/null)"; then
printf 'MQTTS certificate status: %s\n' "$mqtts_certificate_status"
else
warn "Unable to read the current status of MQTTS ACM certificate '$mqtts_certificate_arn'."
fi
fi
fi
}
prepare_mqtts_destroy() {
local mqtts_reservation="$1"
local recorded_arn
local recorded_hostname
local recorded_mode
local service_certificate_arn
local service_hostname
local service_managed_dns
local service_load_balancer_type
local service_port
local service_ssl_ports
local service_type
local validation_endpoint
[ -n "$mqtts_reservation" ] || return 0
[ "$(jq -r '.target + "/" + .dns' <<< "$mqtts_reservation")" = "eks/external-dns" ] || die \
"The MQTTS hostname reservation requires external cleanup and or-stack release-hostname."
[ "$(jq -r '.exposure' <<< "$mqtts_reservation")" = ingress ] && \
[ "$(jq -r '.endpoint' <<< "$mqtts_reservation")" = mqtts ] || die \
"The MQTTS hostname reservation is not an EKS Ingress MQTTS endpoint."
MQTTS_ENABLED=true
MQTTS_HOSTNAME_VALUE="$(jq -r '.hostname' <<< "$mqtts_reservation")"
normalize_and_validate_mqtts_hostname
[ "$MQTTS_HOSTNAME_VALUE" != "$HOSTNAME_VALUE" ] || die \
"The web and MQTTS reservations unexpectedly use the same hostname."
if [ "$STACK_NAMESPACE_EXISTS" != true ]; then
log "Stack namespace '$STACK_NAME' is already absent; recovering retained MQTTS DNS ownership."
return
fi
require_reservation_namespace_identity "$mqtts_reservation"
recorded_mode="$(stack_mqtts_certificate_mode)" || die \
"Unable to read MQTTS certificate mode from namespace '$STACK_NAME'."
recorded_arn="$(stack_mqtts_certificate_arn)" || die \
"Unable to read MQTTS certificate ARN from namespace '$STACK_NAME'."
recorded_hostname="$(stack_mqtts_hostname)" || die \
"Unable to read MQTTS hostname from namespace '$STACK_NAME'."
STACK_MQTTS_SERVICE_JSON="$(kubectl --context "$KUBE_CONTEXT" \
--namespace "$STACK_NAME" \
get service proxy-mqtts \
--ignore-not-found \
-o json)" || die \
"Unable to inspect Service '$STACK_NAME/proxy-mqtts' before destroy."
case "$recorded_mode" in
existing|shared|managed) ;;
"")
# Prepare reserves hostnames before the facade records certificate mode.
# An interruption there leaves a valid claim but no MQTTS resources. Allow
# cleanup of that state without adopting an unowned deployed endpoint.
[ -z "$STACK_MQTTS_SERVICE_JSON" ] && [ -z "$recorded_arn" ] || die \
"Stack '$STACK_NAME' has a low-level or unowned Ingress MQTTS endpoint; refusing automated cleanup without a supported openremote.io/mqtts-certificate-mode."
[ -z "$recorded_hostname" ] || [ "$recorded_hostname" = "$MQTTS_HOSTNAME_VALUE" ] || die \
"Namespace '$STACK_NAME' records MQTTS hostname '$recorded_hostname', but reserves '$MQTTS_HOSTNAME_VALUE'."
validation_endpoint="$(kubectl --context "$KUBE_CONTEXT" --namespace "$STACK_NAME" \
get dnsendpoint "$ACM_MQTTS_VALIDATION_DNS_ENDPOINT_NAME" \
--ignore-not-found -o json)" || die \
"Unable to inspect validation resources for the incomplete MQTTS configuration."
[ -z "$validation_endpoint" ] || die \
"Stack '$STACK_NAME' has MQTTS validation resources without a recorded certificate mode; refusing destructive cleanup."
# Normal tagged-certificate recovery and NLB/DNS cleanup still follow;
# neither missing mode nor missing Service proves external cleanup.
log "Stack '$STACK_NAME' has an incomplete MQTTS prepare; continuing verified cleanup."
return
;;
*)
die "Stack '$STACK_NAME' has a low-level or unowned Ingress MQTTS endpoint; refusing automated cleanup without a supported openremote.io/mqtts-certificate-mode."
;;
esac
[ "$recorded_hostname" = "$MQTTS_HOSTNAME_VALUE" ] || die \
"Namespace '$STACK_NAME' records MQTTS hostname '$recorded_hostname', but reserves '$MQTTS_HOSTNAME_VALUE'."
MQTTS_CERTIFICATE_MODE="$recorded_mode"
MQTTS_CERTIFICATE_ARN="$recorded_arn"
[ -n "$STACK_MQTTS_SERVICE_JSON" ] || return 0
service_managed_dns="$(jq -r '.metadata.labels["openremote.io/managed-dns"] // empty' \
<<< "$STACK_MQTTS_SERVICE_JSON")"
service_hostname="$(jq -r '.metadata.annotations["external-dns.kubernetes.io/hostname"] // empty' \
<<< "$STACK_MQTTS_SERVICE_JSON")"
service_certificate_arn="$(jq -r '.metadata.annotations["service.beta.kubernetes.io/aws-load-balancer-ssl-cert"] // empty' \
<<< "$STACK_MQTTS_SERVICE_JSON")"
service_load_balancer_type="$(jq -r '.metadata.annotations["service.beta.kubernetes.io/aws-load-balancer-type"] // empty' \
<<< "$STACK_MQTTS_SERVICE_JSON")"
service_ssl_ports="$(jq -r '.metadata.annotations["service.beta.kubernetes.io/aws-load-balancer-ssl-ports"] // empty' \
<<< "$STACK_MQTTS_SERVICE_JSON")"
service_type="$(jq -r '.spec.type // empty' <<< "$STACK_MQTTS_SERVICE_JSON")"
service_port="$(jq -r '.spec.ports[]? | select(.name == "mqtts") | .port' \
<<< "$STACK_MQTTS_SERVICE_JSON")"
[ "$service_managed_dns" = true ] || die \
"Service '$STACK_NAME/proxy-mqtts' is not marked for OpenRemote-managed DNS."
[ "$service_hostname" = "$MQTTS_HOSTNAME_VALUE" ] || die \
"Service '$STACK_NAME/proxy-mqtts' records hostname '$service_hostname', not '$MQTTS_HOSTNAME_VALUE'."
[ "$service_type" = LoadBalancer ] && [ -n "$service_port" ] || die \
"Service '$STACK_NAME/proxy-mqtts' is not the expected MQTTS LoadBalancer."
[ "$service_load_balancer_type" = external ] && [ "$service_ssl_ports" = mqtts ] || die \
"Service '$STACK_NAME/proxy-mqtts' is not configured as the expected ACM-terminated AWS NLB."
if [[ ! "$service_certificate_arn" =~ ^arn:[^:]+:acm:[a-z0-9-]+:[0-9]{12}:certificate/[A-Za-z0-9-]+$ ]]; then
die "Service '$STACK_NAME/proxy-mqtts' does not carry a valid ACM certificate ARN."
fi
if [ -n "$recorded_arn" ] && [ "$recorded_arn" != "$service_certificate_arn" ]; then
if [ "$recorded_mode" = managed ]; then
die "Service '$STACK_NAME/proxy-mqtts' uses certificate '$service_certificate_arn', not owned managed certificate '$recorded_arn'."
fi
warn "Service '$STACK_NAME/proxy-mqtts' uses certificate '$service_certificate_arn', while the namespace records '$recorded_arn'. This can follow an interrupted rotation; both externally owned certificates will be retained."
fi
MQTTS_CERTIFICATE_ARN="$service_certificate_arn"
}
mqtts_load_balancer_exists() {
local load_balancers
local load_balancer_arns
local arn
local tags
local matches
# Service status disappears with the namespace (or an earlier uninstall).
# Query the controller's AWS identity tags on every destroy attempt so an
# interrupted cleanup cannot release reservations while its NLB still exists.
load_balancers="$(aws_elbv2 describe-load-balancers --output json 2>&1)" || die \
"Unable to inspect MQTTS load balancers before releasing hostname reservations: $load_balancers"
load_balancer_arns="$(jq -er '
.LoadBalancers | if type != "array" then error("missing load balancers") else
[.[] | select(.Type == "network") | .LoadBalancerArn |
if type == "string" and length > 0 then . else error("missing NLB ARN") end] |
join("\n")
end' <<< "$load_balancers")" || die "Unable to parse the AWS load-balancer response."
while IFS= read -r arn; do
[ -n "$arn" ] || continue
if ! tags="$(aws_elbv2 describe-tags --resource-arns "$arn" --output json 2>&1)"; then
case "$tags" in
# The controller may delete an NLB between the list and tag queries.
*'(LoadBalancerNotFound)'*) continue ;;
*) die "Unable to inspect load-balancer ownership for '$arn': $tags" ;;
esac
fi
matches="$(jq -er --arg arn "$arn" --arg cluster "$CLUSTER_NAME" \
--arg service "$STACK_NAME/proxy-mqtts" '
.TagDescriptions |
if type != "array" or length != 1 then error("missing tag description") else
.[0] | if .ResourceArn != $arn or (.Tags | type) != "array" then
error("invalid tag description") else
.Tags | from_entries |
.["elbv2.k8s.aws/cluster"] == $cluster and
.["service.k8s.aws/stack"] == $service
end
end | tostring' <<< "$tags")" || die "Unable to parse AWS load-balancer ownership for '$arn'."
[ "$matches" != true ] || return 0
done <<< "$load_balancer_arns"
return 1
}
wait_for_mqtts_load_balancer_removed() {
local deadline
local next_notice
local now
local timeout_seconds
[ "$MQTTS_ENABLED" = true ] && [ "$EXPOSURE" = ingress ] || return 0
timeout_seconds="$(duration_seconds "$TIMEOUT")"
now="$(date +%s)"
deadline="$((now + timeout_seconds))"
next_notice="$((now + 30))"
log "Waiting for MQTTS NLBs for '$CLUSTER_NAME/$STACK_NAME/proxy-mqtts' to be deleted."
while [ "$now" -lt "$deadline" ]; do
if ! mqtts_load_balancer_exists; then
log "No MQTTS NLB remains for the stack."
return
fi
if [ "$now" -ge "$next_notice" ]; then
log "The MQTTS NLB is still present."
next_notice="$((now + 30))"
fi
sleep 5
now="$(date +%s)"
done
die "MQTTS NLB deletion for '$CLUSTER_NAME/$STACK_NAME/proxy-mqtts' was not confirmed within '$TIMEOUT'. Hostname reservations and certificate state were retained so destroy can be retried safely."
}
wait_for_public_hostname_removed() {
local description="$2"
local hostname="$1"
local records
select_managed_dns_domain "$hostname"
discover_authoritative_name_servers
records="$(printf '%s\t%s\n' "$hostname" A "$hostname" AAAA "$hostname" CNAME)"
wait_for_dns_records_removed "$description" "$records"
}
destroy_stack_with_managed_resources() {
local mqtts_reservation
local reservation
require_command dig
require_command helm
require_command jq
reservation="$(stack_hostname_reservation)" || die "Unable to inspect hostname reservations before destroy."
mqtts_reservation="$(stack_mqtts_hostname_reservation)" || die \
"Unable to inspect MQTTS hostname reservations before destroy."
# Every public EKS stack reserves its hostname before creating external
# resources. Never bypass DNS cleanup when that required state is missing.
[ -n "$reservation" ] || die \
"Stack '$STACK_NAME' has no hostname reservation; refusing EKS destruction without verifiable DNS cleanup state. Inspect the stack and hostname registry before retrying."
[ "$(jq -r '.target + "/" + .dns' <<< "$reservation")" = "eks/external-dns" ] || die \
"This reservation requires external cleanup and or-stack release-hostname."
HOSTNAME_VALUE="$(jq -r '.hostname' <<< "$reservation")"
EXPOSURE="$(jq -r '.exposure' <<< "$reservation")"
inspect_stack_namespace_for_destroy
if [ "$STACK_NAMESPACE_EXISTS" = true ]; then
require_reservation_namespace_identity "$reservation"
fi
MQTTS_ENABLED=false
if [ -n "$mqtts_reservation" ]; then
prepare_mqtts_destroy "$mqtts_reservation"
fi
capture_stack_ingress_for_destroy
resolve_managed_certificate_for_destroy
resolve_managed_mqtts_certificate_for_destroy
if [ "$MANAGED_MQTTS_CERTIFICATE_DESTROY" = true ] && \
[ -z "$mqtts_reservation" ]; then
die "Stack '$STACK_NAME' owns a managed MQTTS certificate without an MQTTS hostname reservation; refusing unverifiable DNS cleanup."
fi
if [ "$MANAGED_CERTIFICATE_DESTROY" = true ]; then
CERTIFICATE_MODE=managed
EXPOSURE=ingress
else
CERTIFICATE_MODE=""
fi
if [ "$MANAGED_MQTTS_CERTIFICATE_DESTROY" = true ]; then
MQTTS_CERTIFICATE_MODE=managed
fi
require_managed_external_dns
if [ "$MANAGED_CERTIFICATE_DESTROY" = true ]; then
validate_stack_ingress_for_managed_destroy
capture_validation_dns_records_for_destroy
fi
if [ "$MANAGED_MQTTS_CERTIFICATE_DESTROY" = true ]; then
capture_mqtts_validation_dns_records_for_destroy
fi
# The ALB Ingress can be removed independently, allowing its managed web
# certificate to detach before the namespace is deleted. The MQTTS
# certificate remains attached to the NLB until Service finalizer cleanup.
if [ "$MANAGED_CERTIFICATE_DESTROY" = true ]; then
remove_managed_stack_ingress
wait_for_public_hostname_removed "$HOSTNAME_VALUE" "stack hostname '$HOSTNAME_VALUE'"
wait_for_managed_certificate_detachment
fi
if [ "$STACK_NAMESPACE_EXISTS" = true ]; then
run_or_stack "$@"
STACK_NAMESPACE_EXISTS=false
elif [ "$MANAGED_CERTIFICATE_DESTROY" = true ] || \
[ "$MANAGED_MQTTS_CERTIFICATE_DESTROY" = true ]; then
log "Stack namespace '$STACK_NAME' is already absent; continuing external cleanup."
fi
wait_for_mqtts_load_balancer_removed
if [ "$MANAGED_CERTIFICATE_DESTROY" = false ]; then
wait_for_public_hostname_removed "$HOSTNAME_VALUE" "stack hostname '$HOSTNAME_VALUE'"
fi
if [ -n "$mqtts_reservation" ]; then
wait_for_public_hostname_removed "$MQTTS_HOSTNAME_VALUE" \
"MQTTS hostname '$MQTTS_HOSTNAME_VALUE'"
fi
# Managed mode assumes each certificate and its validation CNAMEs are
# exclusive to its endpoint. ACM can reuse CNAMEs for other certificates;
# ownership tags do not prove DNS-record exclusivity. Deleting the namespace
# lets ExternalDNS remove both validation DNSEndpoints before ACM deletion.
if [ "$MANAGED_MQTTS_CERTIFICATE_DESTROY" = true ]; then
wait_for_managed_mqtts_certificate_detachment
select_managed_dns_domain "$MQTTS_HOSTNAME_VALUE"
discover_authoritative_name_servers
wait_for_dns_records_removed "MQTTS ACM validation" \
"$MQTTS_VALIDATION_DNS_RECORDS"
delete_managed_mqtts_certificate
fi
if [ "$MANAGED_CERTIFICATE_DESTROY" = true ]; then
select_managed_dns_domain "$HOSTNAME_VALUE"
discover_authoritative_name_servers
wait_for_dns_records_removed "ACM validation" "$VALIDATION_DNS_RECORDS"
delete_managed_certificate
fi
if [ -n "$mqtts_reservation" ]; then
release_portable_mqtts_hostname
if [ "$MANAGED_MQTTS_CERTIFICATE_DESTROY" = false ] && [ -n "$MQTTS_CERTIFICATE_ARN" ]; then
log "Externally owned $MQTTS_CERTIFICATE_MODE MQTTS ACM certificate '$MQTTS_CERTIFICATE_ARN' was retained."
fi
fi
# Release the required web claim last. If its release fails, destroy can
# still be retried; releasing it first would strand a remaining optional
# MQTTS claim behind the web-reservation preflight.
release_portable_hostname
if [ "$MANAGED_CERTIFICATE_DESTROY" = true ] || \
[ "$MANAGED_MQTTS_CERTIFICATE_DESTROY" = true ]; then
log "Stack '$STACK_NAME' and its managed external certificate resources are destroyed."
fi
}
release_portable_hostname() {
run_or_stack release-hostname --name "$STACK_NAME" --kube-context "$KUBE_CONTEXT" \
--hostname "$HOSTNAME_VALUE" --confirm "$STACK_NAME"
}
release_portable_mqtts_hostname() {
run_or_stack release-hostname --name "$STACK_NAME" --kube-context "$KUBE_CONTEXT" \
--hostname "$MQTTS_HOSTNAME_VALUE" --confirm "$STACK_NAME"
}
delegate_stack_command() {
local stack_args=(
"$COMMAND"
--name "$STACK_NAME"
--kube-context "$KUBE_CONTEXT"
)
case "$COMMAND" in
uninstall)
stack_args+=(--timeout "$TIMEOUT")
;;
destroy)
stack_args+=(--confirm "$CONFIRM_STACK_NAME" --timeout "$TIMEOUT")
;;
esac
if [ "$COMMAND" = "destroy" ]; then
destroy_stack_with_managed_resources "${stack_args[@]}"
return
fi
run_or_stack "${stack_args[@]}"
if [ "$COMMAND" = "status" ]; then
show_certificate_configuration
fi
}
main() {
parse_arguments "$@"
validate_common_options
validate_command_options
case "$COMMAND" in
apply)
apply_stack
;;
status|credentials|uninstall|destroy)
[ -x "$OR_STACK_COMMAND" ] || die \
"Portable stack command '$OR_STACK_COMMAND' is missing or not executable."
require_eks_target
delegate_stack_command
;;
esac
}
if [ "${BASH_SOURCE[0]}" = "$0" ]; then
main "$@"
fi