ec2-create-ce

#!/bin/bash

export TZ="UTC"

# defaults
[ -n "${KEY_NAME}" ] || KEY_NAME="jenkins-ngcp-create-ami"
[ -n "${KEY_FILE}" ] || KEY_FILE="${HOME}/.ssh/${KEY_NAME}.pem"
[ -n "${SECURITY_GROUP}" ] || SECURITY_GROUP="sipwise-only"
[ -n "${TAG_PURPOSE}" ] || TAG_PURPOSE="ngcp-create-ce-ami"
DATE_STRING=$(date +%Y-%m-%d_%H:%M)

rm -f ec2_report.txt

usage() {
  echo "$0 - script to generate EC2 AMI for sip:provider ngcp CE

Usage:

  $0 <mandatory_options> [OPTIONS]

Supported OPTIONS:

  --ami-name <STRING>       set AMI name to specified string
                            (if unset defaults to ngcp-ce-\$release)
  --base-ami <AMI-ID>       AMI ID that should be used as base for running
                            ngcp installation process (recommended: Debian 64bit)
                            (if unset defaults to ami-8a745cf3)
  --copy-to-all-regions     Copy resulting AMI to all available EC2 regions
  --help                    display this help text
  --installer-url <URL>     Use ngcp-installer.deb, retrieved from specified URL
                            (if unset defaults to http://deb.sipwise.com/spce/ngcp-installer-\$NGCP_RELEASE.deb)
  --instance-type <TYPE>    EC2 instance type that should be used for launching
                            (if unset defaults to t2.medium)
  --keep-ami-snapshot       do not remove snapshot used by a possibly existing AMI
                            which gets removed during the run (because of identical names)
                            (only considered in combination with --remove-existing-ami)
  --ngcp-release <VERSION>  sip:provider release version
                            (if unset defaults to <latest>)
  --public                  Make AMI available to the public
                            (if unset defaults to private)
  --remove-existing-ami     If an AMI with the according name already exists get
                            rid of it before trying to creating the AMI
  --region <REGION>         specify region that should be used
                            (if unset defaults to the eu-west-1 zone)
  --skip-reboot             do not reboot ngcp system after installation finished

Mandatory options for EC2-Classic (non-VPC/older generations) usage:

  --elastic-ip <IP>         associate started instance ID with provided Elastic IP address

Mandatory options for VPC usage:

  --allocation-id <A-ID>    specify allocation ID for Elastic IP
  --elastic-ip <IP>         associate started instance ID with provided Elastic IP address
  --subnet <Subnet-ID>      specify subnet ID that should be used
"
}

ssh_wrapper() {
  # use keepalives of 10 seconds (if 3 fail, connection terminated) and
  # connection timeout of 15 secs instead of default system TCP timeout (15
  # minutes)
  # shellcheck disable=SC2029
  ssh -o "ServerAliveInterval 10" \
    -o "ConnectTimeout 15" \
    -o "StrictHostKeyChecking=no" \
    -o "UserKnownHostsFile=/dev/null" \
    -i "${KEY_FILE}" \
    "admin@${HOSTNAME}" \
    "$@"
}


# generate report for usage in other scripts/jenkins jobs
generate_ec2_report() {
  cat > ec2_report.txt << EOF
instance_hostname=${HOSTNAME}
instance_id=${INSTANCE_ID}
aws_region=${AWS_REGION}
ami_id=${AMI_ID}
ami_name=${AMI_NAME}
ami_description=${AMI_DESCRIPTION}
ami_public=${_opt_public}
ami_instance_type=${INSTANCE_TYPE}
ngcp_version=${NGCP_VERSION}

# for usage in Jenkins
testhost=${TESTHOST}
vmversion=${VMVERSION}
EOF
}

bailout() {
  [ -n "$1" ] && EXIT="$1" || EXIT="1"

  # ensure we get ec2_report.txt even if script execution fails,
  # otherwise Jenkins jobs like ec2-ami-ec can't provide ec2_report.txt
  # to Jenkins downstream job ec2-ami-stop
  generate_ec2_report

  exit "${EXIT}"
}
trap bailout 1 2 3 3 6 14 15

# NOTE: this is non-blocking currently, so we're not waiting until everything is finished here
copy_ami() {
  local logfile
  logfile=$(mktemp)

  local source_region="$1"
  local ami_id="$2"
  local ami_name="$3"
  local ami_description="$4"

  local target_region
  for target_region in \
    eu-west-1
  do
    case "${target_region}" in
      "${source_region}")
        echo "*** Skipping region ${target_region} as source and destination region are the same ones. ***"
        echo "AMI ID for region ${target_region}: ${ami_id}" >> ec2_report_amis.txt
        ;;
      *)
        remove_ami "${target_region}" "${ami_name}"

        echo "*** Copying AMI ID ${ami_id} from ${source_region} to ${target_region} (note: we are NOT blocking/waiting until this is fully finished!) ***"
        aws ec2 copy-image --source-image-id "${ami_id}" --source-region "${source_region}" --region "${target_region}" --name "${ami_name}" --description "${ami_description}" > "${logfile}"

        if [ $? -ne 0 ]; then
          echo "Error copying AMI ID ${ami_id} from ${source_region} to ${target_region}" >&2
          return 1
        fi

        local ami_id_copy
        ami_id_copy="$(jq -r '.ImageId' "${logfile}")"

        if [ -z "${ami_id_copy}" ] ; then
          echo "Error retrieving AMI ID ${ami_id} for region ${target_region}:"
          cat "${logfile}" >&2
          return 1
        fi

        # for usage as global report
        echo "AMI ID for region ${target_region}: ${ami_id_copy}" >> ec2_report_amis.txt
        ;;
    esac
  done
}

remove_ami() {
  local aws_region="$1"
  local ami_name="$2"

  if ! aws ec2 describe-images --region "${aws_region}" --filter "Name=name,Values=${ami_name}" | jq -r '.Images[].ImageId' | grep -q . ; then
    return 0 # no AMIs with given name present
  fi

  echo "*** Warning, AMI with name ${ami_name} exists already."
  if ! ${_opt_remove_existing_ami} ; then
    return 0
  fi

  local existing_ami_id
  existing_ami_id=$(aws ec2 describe-images --region "${aws_region}" --filter "Name=name,Values=${ami_name}" | jq -r '.Images[].ImageId')

  if [ -z "${existing_ami_id}" ] ; then
    echo "Problem retrieving AMI ID for AMI with name ${ami_name}." >&2
    return 1
  fi

  # we need to identify which snapshot is used by the AMI so we can remove it after "deregister"-ing the AMI
  local ami_snapshot
  ami_snapshot=$(aws ec2 describe-images --region "${aws_region}" --filter "Name=name,Values=${ami_name}" | jq -r '.Images[].BlockDeviceMappings[].Ebs.SnapshotId')

  if [ -z "${ami_snapshot}" ] ; then
    echo "Problem retrieving snapshot ID for AMI with name ${ami_name}." >&2
    return 1
  fi

  echo "*** Option --remove-existing-ami is set, removing existing AMI with ID ${existing_ami_id} ***"
  aws ec2 deregister-image --region "${aws_region}" --image-id "${existing_ami_id}"
  if [ $? -ne 0 ] ; then
    echo "Noticed problem when trying to delete AMI with name ${ami_name}." >&2
    return 1
  fi

  if ${_opt_keep_ami_snapshot} ; then
    echo "*** Option --keep-ami-snapshot is set, not removing AMI snapshot ${ami_snapshot}. ***"
  else
    aws ec2 delete-snapshot --region "${aws_region}" --snapshot-id "${ami_snapshot}"
    if [ $? -ne 0 ] ; then
      echo "Noticed problem when trying to delete snapshot ${ami_snapshot} for 'deregister'-ed AMI with name ${ami_name}." >&2
      return 1
    fi
  fi
}

wait_for_ssh() {
  local max_attempt=360
  local attempt=0

  while [[ "${attempt}" -lt "${max_attempt}" ]]; do
    if ssh_wrapper exit; then
      return
    fi
    sleep 1
    attempt=$((attempt + 1))
  done

  return 1
}

get_status() {
   case "${NGCP_VERSION}" in
    2.*|3.*|mr[3-6].*|mr7.[0-4]*)
      if ssh_wrapper "grep 'System configured' -q /etc/sipwise_ngcp_version"; then
        echo "finished"
      fi
    ;;
    *)
      local status=''
      status=$(wget --timeout=30 -q -O- "${HOSTNAME}:4242/status" || true)
      echo "${status}"
    ;;
  esac
}

wait_for_configuration() {
  local max_attempt=360
  local attempt=0
  local status=''

  if [[ "$-" == *x* ]]; then
    trace_enabled=true
    set +x
  fi

  echo "Waiting when system is configured"
  while [[ "${attempt}" -lt "${max_attempt}" ]]; do
    status=$(get_status)
    if [[ "${status}" == "finished" ]]; then
      echo "System was successfully configured"
      return
    elif [[ "${status}" == "error" ]]; then
      echo "System configuration was failed"
      break
    fi
    echo "Attempt ${attempt} of ${max_attempt}, sleeping for a second. Status: '${status}'"
    sleep 1
    attempt=$((attempt + 1))
  done

  if "${trace_enabled}"; then
    set -x
  fi

  return 1
}

CMDLINE_OPTS=allocation-id:,ami-name:,base-ami:,copy-to-all-regions,elastic-ip:,help,installer-url:,instance-type:,keep-ami-snapshot,ngcp-release:,public,region:,remove-existing-ami,subnet:,skip-reboot

_opt_temp=$(getopt --name "$0" -o +bch --long ${CMDLINE_OPTS} -- "$@")
if [ $? -ne 0 ]; then
  echo "Try '$0 --help' for more information." >& 2
  exit 1
fi
eval set -- "${_opt_temp}"

_opt_keep_ami_snapshot=false
_opt_public=false
_opt_remove_existing_ami=false
_opt_skip_reboot=false
_opt_copy_to_all_regions=false

while :; do
  case "$1" in
  --allocation-id)
    shift; ALLOCATION_ID="$1"
    ;;
  --ami-name)
    shift; AMI_NAME="$1"
    ;;
  --base-ami)
    shift; BASE_AMI="$1"
    ;;
  --copy-to-all-regions)
    _opt_copy_to_all_regions=true
    ;;
  --elastic-ip)
    shift; ELASTIC_IP="$1"
    ;;
  --help)
    usage ; exit 0;
    ;;
  --installer-url)
    shift; INSTALLER_URL="$1"
    ;;
  --instance-type)
    shift; INSTANCE_TYPE="$1"
    ;;
  --keep-ami-snapshot)
    _opt_keep_ami_snapshot=true
    ;;
  --ngcp-release)
    shift; NGCP_RELEASE="$1"
    ;;
  --public)
    _opt_public=true
    ;;
  --region)
    shift; AWS_REGION="$1"
    ;;
  --remove-existing-ami)
    _opt_remove_existing_ami=true
    ;;
  --skip-reboot)
    _opt_skip_reboot=true
    ;;
  --subnet)
    shift; SUBNET="$1"
    ;;
  --)
    shift; break
    ;;
  *)
    echo "Internal getopt error! $1" >&2
    exit 1
    ;;
  esac
  shift
done

# if unset set sane defaults
[ -n "${AWS_REGION}" ] || AWS_REGION="eu-west-1"
[ -n "${BASE_AMI}" ] || BASE_AMI="ami-0e9cc061cd3259f22"
[ -n "${INSTANCE_TYPE}" ] || INSTANCE_TYPE="m5.large"

if ! [ -r "${HOME}"/.aws/config ] ; then
  echo "awscli seems to be unconfigured, ensure ${HOME}/.aws/config exists." >&2
  exit 1
fi

if ! [ -r "${KEY_FILE}" ] ; then
  echo "Could not read key file ${KEY_FILE}, can not continue." >&2
  exit 1
fi

check4progs(){
  local RC=''
  for arg in "$@" ; do
    which "${arg}" >/dev/null 2>&1 || RC="${arg}"
  done
  if [ -n "${RC}" ] ; then
    echo "${RC} not found/executable" >&2
    return 1
  fi
}

if ! check4progs jq ; then
  echo "Required tool jq not found, forgot to install jq?" >&2
  exit 1
fi

if ! check4progs aws ; then
  echo "Required tool aws not found, forgot to install awscli?" >&2
  exit 1
fi

if [ -z "${NGCP_RELEASE}" ] ; then
  NGCP_RELEASE="latest"
  echo "No ngcp release set, defaulting to ${NGCP_RELEASE}"
fi

if [ -z "${INSTALLER_URL}" ] ; then
  INSTALLER_URL="http://deb.sipwise.com/spce/ngcp-installer-${NGCP_RELEASE}.deb"
fi

if ! wget --quiet -O /tmp/ngcp-installer.deb "${INSTALLER_URL}" ; then
  echo "Couldn't retrieve ${INSTALLER_URL} - exiting" >&2
  exit 1
fi

if dpkg --info /tmp/ngcp-installer.deb | grep -q 'mr' ; then
  NGCP_VERSION=$(dpkg --info /tmp/ngcp-installer.deb | awk '/^ Version: / {print $2}' | sed 's/.*mr/mr/')
else # older versions don't have the mr string in the version information, so let's reuse provided ngcp release
  NGCP_VERSION="${NGCP_RELEASE}"
  echo "*** No 'mr' string inside version information of installer found... ***"
  echo "*** ... falling back to ngcp release version information [${NGCP_VERSION}] ***"
fi

if [ -z "${NGCP_VERSION}" ] ; then
  echo "Couldn't identify ngcp version, exiting." >&2
  bailout 1
fi

# ensure we don't have any builds running already, see TT#30565
NUM_RUNNING_INSTANCES=$(aws ec2 describe-instances --region "${AWS_REGION}"   \
                       --filter "Name=instance-state-name,Values=running"     \
                                "Name=tag:Purpose,Values=${TAG_PURPOSE}"    | \
                                jq '.Reservations[].Instances[].InstanceId' | \
                                wc -l)
if [ "${NUM_RUNNING_INSTANCES}" -gt 0 ] ; then
  echo "Error: there are running instances with tag ${TAG_PURPOSE} in region ${AWS_REGION}"  >&2
  echo "Exiting to ensure we don't have instances running due to Jenkins job failures."  >&2
  echo "Please ensure that those instances are stopped before trying to build new ones." >&2
  echo "In case of questions contact someone with according AWS permissions."            >&2
  bailout 1
fi

# if AMI_NAME is set to "none" then we get defaults from Jenkins, if
# so then use the ngcp-ce-... naming schema
if [ -n "${AMI_NAME}" ] && [ "${AMI_NAME}" != "none" ] ; then
  echo "*** AMI_NAME is set to ${AMI_NAME} ***"
else
  # use just "ngcp-ce-mr3.3.1.0" if we get something like
  # "ngcp-ce-mr3.3.1.0+0~20140528091259.443+wheezy~1.gbpf74599"
  # as NGCP_VERSION
  AMI_NAME="ngcp-ce-${NGCP_VERSION%%\+*}"
  echo "*** AMI_NAME is unset yet, defaulting to ${AMI_NAME}"

  case "${AMI_NAME}" in
    ngcp-ce-mr*.*.*.*)
      echo "*** Identified hotfix version string in AMI name [${AMI_NAME}]... ***"
      AMI_NAME="${AMI_NAME%.*}"
      echo "*** ... stripping AMI name to use ${AMI_NAME} instead ***"
      ;;
  esac
fi


AMI_DESCRIPTION="Official sip:provider CE AMI for release ${NGCP_VERSION} [${DATE_STRING}]"

case "${NGCP_RELEASE}" in
  2.*|3.*|mr3.*|mr4.*|mr5.*|mr6.0*|mr6.1*)
    echo "*** ngcp release ${NGCP_RELEASE} doesn't support systemd"
    SYSTEMD_SUPPORT=false
    ;;
  *)
    echo "*** ngcp release ${NGCP_RELEASE} supports systemd"
    SYSTEMD_SUPPORT=true
    ;;
esac

USER_DATA_FILE="$(mktemp)"
cat > "${USER_DATA_FILE}" << EOF
#!/bin/bash
wget -O /tmp/ngcp-installer.deb ${INSTALLER_URL}
dpkg -i /tmp/ngcp-installer.deb

if [ -f /etc/apt/sources.list.d/backports.list ] && grep -q 'jessie-backports' /etc/apt/sources.list.d/backports.list ; then
  echo "jessie-backports no longer exists, removing /etc/apt/sources.list.d/backports.list"
  rm -f /etc/apt/sources.list.d/backports.list
fi

# workaround for mr13.2.1 and newer with bookworm-backports enabled by default:
if [ -f /etc/apt/sources.list.d/debian.sources ] && grep -q -- '-backports' /etc/apt/sources.list.d/debian.sources ; then
  echo "backports is unexpected, disabling from /etc/apt/sources.list.d/debian.sources"
  sed -i -E 's/ [a-z]+-backports//' /etc/apt/sources.list.d/debian.sources
fi

if [ "${SYSTEMD_SUPPORT}" = "false" ] && [ -d /run/systemd/system ] ; then
  echo "Switching from systemd to sysvinit (pre-reboot)"
  apt update
  apt install --yes sysvinit-core
  echo "Rebooting now to switch from systemd to sysvinit"
  #Temporary crontab to restart installer after reboot
  (crontab -l ; echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") | crontab -
  (crontab -l ; echo "@reboot FORCE=yes /usr/sbin/ngcp-installer 2>&1 | tee -a /var/log/ngcp-installer-debug.log") | crontab -
  reboot
fi

# workaround for mr4.2.1
export HOME=/root/
EOF



init_configuration=false
case "${NGCP_RELEASE}" in
  2.8|3.*|mr[3-5].*|mr6.[0-4]*)
    echo "*** ngcp release ${NGCP_RELEASE} doesn't have configuration tools"
    ;;
  *)
    echo "*** ngcp release ${NGCP_RELEASE} has configuration tools"
    init_configuration=true
    ;;
esac

if "${init_configuration}" ; then
  cat >> "${USER_DATA_FILE}" << EOF
sed_regexp=''
sed_regexp+='s/^(export FORCE)=.+$/\1=yes/;'
sed_regexp+='s/^(export DHCP)=.+$/\1=true/;'
sed -ri "\${sed_regexp}" /etc/ngcp-installer/config_deploy.inc
ngcp-installer
attempt=0
max_attempt=1000
rc=1
while [[ "\${attempt}" -lt "\${max_attempt}" ]]; do
  if grep 'System configured' -q /etc/sipwise_ngcp_version; then
    rc=0
    break
  fi
  sleep 1
  attempt=\$((attempt + 1))
done
exit \${rc}
EOF
else
  cat >> "${USER_DATA_FILE}" << EOF
FORCE=yes ngcp-installer 2>&1 | tee -a /var/log/ngcp-installer-debug.log
[[ -f /tmp/ngcp-installer.log ]] && cp /tmp/ngcp-installer.log /var/log/ngcp-installer.log
EOF
fi

echo "*** Generated the following user-data file:"
cat "${USER_DATA_FILE}"
echo "*** end of user-data-file ***"

LOGFILE_RUN="$(mktemp)"
if [ -n "${SUBNET}" ] ; then
  echo "Starting EC2 instance in region ${AWS_REGION}, using VPC with subnet ${SUBNET}, instance-type ${INSTANCE_TYPE} + AMI ${BASE_AMI}"
  aws ec2 run-instances --region "${AWS_REGION}" \
               --user-data "file://${USER_DATA_FILE}" \
               --subnet "${SUBNET}" \
               --key "${KEY_NAME}" \
               --instance-type "${INSTANCE_TYPE}" \
               --image-id "${BASE_AMI}" >"${LOGFILE_RUN}"
else
  echo "Starting EC2 instance in region ${AWS_REGION}, using EC2-Classic with security-group ${SECURITY_GROUP}, instance-type ${INSTANCE_TYPE} + AMI ${BASE_AMI}"
  aws ec2 run-instances --region "${AWS_REGION}" \
               --user-data "file://${USER_DATA_FILE}" \
               --group "${SECURITY_GROUP}" \
               --key "${KEY_NAME}" \
               --instance-type "${INSTANCE_TYPE}" \
               --image-id "${BASE_AMI}" >"${LOGFILE_RUN}"
fi

INSTANCE_ID="$(jq -r '.Instances[].InstanceId'  "${LOGFILE_RUN}")"
if [ -z "${INSTANCE_ID}" ] ; then
  echo "Error: Could not identify instance ID, exiting." >&2
  exit 1
fi

# assign tag so we de-register again the AMI if needed
aws ec2 create-tags --region "${AWS_REGION}" --resources "${INSTANCE_ID}" --tag "Key=Purpose,Value=${TAG_PURPOSE}"

# assign name (useful for web management console browsing)
[ -z "${JOB_NAME:-}" ]     && JOB_NAME="unset"                     # running outside of Jenkins
[ -z "${BUILD_NUMBER:-}" ] && BUILD_NUMBER="executed-by-$(whoami)" # running outside of Jenkins
aws ec2 create-tags --region "${AWS_REGION}" --resources "${INSTANCE_ID}" --tag "Key=Name,Value=${AMI_NAME}/${DATE_STRING}/${JOB_NAME}/build_${BUILD_NUMBER}"

echo "Starting instance ID ${INSTANCE_ID} for ngcp version ${NGCP_VERSION}"

STATUS="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[].State.Name')"

retry=120 # up to 10 minutes
while [ "${STATUS}" != "running" ] && [ ${retry} -ne 0 ] ; do
  STATUS="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[].State.Name')"

  retry=$((retry - 1))
  echo "Instance ID ${INSTANCE_ID} not running yet, checking again in 5 seconds (${retry} retries left)."
  sleep 5
done

if [ "${STATUS}" != "running" ] ; then
  echo "No further retries left and couldn't bring up system for AMI ${BASE_AMI}, giving up." >&2
  bailout 1
fi

# assign elastic IP address to instance,
# enabling us to enable access to restricted
# webserver areas
if [ -n "${ALLOCATION_ID}" ] ; then
  echo "Associating elastic IP ${ELASTIC_IP} allocation ID ${ALLOCATION_ID} with instance ID ${INSTANCE_ID}"
  aws ec2 associate-address --region "${AWS_REGION}" --allocation-id "${ALLOCATION_ID}" --instance "${INSTANCE_ID}" --allow-reassociation
elif [ -n "${ELASTIC_IP}" ] ; then
  echo "Associating elastic IP ${ELASTIC_IP} with instance ID ${INSTANCE_ID}"
  aws ec2 associate-address --region "${AWS_REGION}" "${ELASTIC_IP}" -i "${INSTANCE_ID}" --allow-reassociation
fi

HOSTNAME="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[] | {PublicDnsName} | [.[]] | .[0]')"

retry=360 # up to 30 minutes
FINISHED_NGCP_CE_INSTALLATION=false
install_log_file='/var/log/ngcp-installer-debug.log'
if "${init_configuration}" ; then
  install_log_file='/var/log/ngcp-installer.log'
fi
while [ "${FINISHED_NGCP_CE_INSTALLATION}" != "true" ] && [ ${retry} -ne 0 ] ; do
  ssh_wrapper "grep -q 'Installation finished. Thanks for choosing' ${install_log_file}"
  RC=$?
  if [ ${RC} -eq 0 ] ; then
    FINISHED_NGCP_CE_INSTALLATION=true
  else
    echo "ngcp installation not yet finished, checking again in 5 seconds (${retry} retries left)."
    sleep 5
    retry=$((retry - 1))
  fi
done

echo "Trying to retrieve ngcp-installer log files"
scp -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" -i "${KEY_FILE}" "admin@${HOSTNAME}":/var/log/ngcp-installer\*log ./ || true

if [ "${FINISHED_NGCP_CE_INSTALLATION}" = "true" ] ; then
  echo "Successfully finished ngcp CE installation (version ${NGCP_VERSION})."
else
  echo "No further retries left and couldn't finish ngcp CE installation (version ${NGCP_VERSION}), giving up." >&2
  bailout 1
fi

#Remove temporary crontab
ssh_wrapper "sudo crontab -l || sudo crontab -r"

# reboot CE system
if ${_opt_skip_reboot} ; then
  echo "*** Skipping system reboot of instance ID ${INSTANCE_ID} as requested via --skip-reboot ***"
else
  ssh_wrapper "sudo reboot"
  retry=120 # up to 10 minutes
  STATUS=""
  while [ "${STATUS}" != "running" ] && [ ${retry} -ne 0 ] ; do
    STATUS="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[].State.Name')"
    retry=$((retry - 1))
    echo "Instance ID not yet running (status: ${STATUS}), checking again in 5 seconds (${retry} retries left)."
    sleep 5
  done

  if [ "${STATUS}" != "running" ] ; then
    echo "No further retries left and couldn't finish reboot of instance ID ${INSTANCE_ID}, giving up." >&2
    bailout 1
  fi

  # we probably have a new hostname after reboot
  HOSTNAME="$(aws ec2 describe-instances --region ${AWS_REGION} --filter "Name=instance-id,Values=${INSTANCE_ID}" | jq -r '.Reservations[].Instances[] | {PublicDnsName} | [.[]] | .[0]')"

  echo "Waiting for ssh access..."
  wait_for_ssh
fi

success=true
if "${init_configuration}"; then
  echo "Configuring system..."
  ssh_wrapper "sudo screen -dm bash -c 'ngcp-initial-configuration'"
  wait_for_configuration || success=false
  echo "Trying to retrieve ngcp-installer.log file after configuration"
  scp \
    -o "StrictHostKeyChecking=no" \
    -o "UserKnownHostsFile=/dev/null" \
    -i "${KEY_FILE}" \
    "admin@${HOSTNAME}":/var/log/ngcp-installer\*log ./ || true
fi

# now check for available ssh connection and nginx listening on port 1443
retry=120 # up to 10 minutes
STATUS=""
while [[ "${STATUS}" != "ok" && "${retry}" -ne 0 ]] && "${success}"; do
  ssh_wrapper "sudo lsof -i -n -P  | grep -q '.*:1443 (LISTEN)'"
  if [ $? -eq 0 ] ; then
    STATUS=ok
  else
    retry=$((retry - 1))
    echo "Host ${HOSTNAME} can't be reached via ssh login yet or nginx not listening on port 1443."
    echo "Retrying ssh access and checking port 1443 LISTEN state in 5 seconds (${retry} retries left)."
    sleep 5
  fi
done

if [ "${STATUS}" != "ok" ] ; then
  echo "Host ${HOSTNAME} can't be reached via ssh or port 1443 not in state LISTEN, giving up." >&2
  bailout 1
fi

remove_ami "${AWS_REGION}" "${AMI_NAME}" || bailout 1

echo "*** Getting rid of all authorized_keys files. ***"
ssh_wrapper 'sudo find / -name authorized_keys -exec rm -f {} \;'

LOGFILE_AMI=$(mktemp)
aws ec2 create-image                    \
     --region "${AWS_REGION}"           \
     --name "${AMI_NAME}"               \
     --description "${AMI_DESCRIPTION}" \
     --instance-id "${INSTANCE_ID}"     \
     > "${LOGFILE_AMI}"

AMI_ID=$(jq -r '.ImageId' "${LOGFILE_AMI}")

if [ -n "${AMI_ID}" ] ; then
  echo "Received AMI ID ${AMI_ID}, now waiting until AMI is available."
else
  echo "Error creating AMI :(" >&2
  cat "${LOGFILE_AMI}" >&2
  bailout 1
fi

retry=120 # up to 10 minutes
STATUS=""
while [ "${STATUS}" != "available" ] && [ ${retry} -ne 0 ] ; do
  STATUS="$(aws ec2 describe-images --region "${AWS_REGION}" --image-ids "${AMI_ID}" | jq -r '.Images[].State')"

  if [ "${STATUS}" = "failed" ] ; then
    echo "Fatal error while creating AMI ID ${AMI_ID} (status: ${STATUS}), exiting." >&2
    bailout 1
  fi

  retry=$((retry - 1))
  echo "AMI ID ${AMI_ID} not yet finished (status: ${STATUS}), checking again in 5 seconds (${retry} retries left)."
  sleep 5
done

if [ "${STATUS}" != "available" ] ; then
  echo "No further retries left and couldn't finish creation of AMI ID ${AMI_ID} for instance ID ${INSTANCE_ID}, giving up." >&2
  bailout 1
fi

echo "Successfully created AMI with ID ${AMI_ID}"

if ${_opt_public} ; then
  echo "Marking AMI as public"
  aws ec2 modify-image-attribute --region "${AWS_REGION}" --image-id "${AMI_ID}" --launch-permission "Add=[{Group=all}]"
  if [ $? -ne 0 ] ; then
    echo "Noticed problem when trying to mark AMI with ID ${AMI_ID} as public." >&2
    bailout 1
  fi
fi

if ${_opt_copy_to_all_regions} ; then
  copy_ami "${AWS_REGION}" "${AMI_ID}" "${AMI_NAME}" "${AMI_DESCRIPTION}" || bailout 1
fi

echo "*** Debugging information ***"
echo "--- user-data-file ---"
cat "${USER_DATA_FILE}"

echo "--- log file for instance run ---"
cat "${LOGFILE_RUN}"

echo "--- log file for AMI creation ---"
cat "${LOGFILE_AMI}"
echo "*** End of Debugging information ***"

echo
echo "*** Status report for AMI ***
AWS region:       ${AWS_REGION}
AMI ID:           ${AMI_ID}
AMI name:         ${AMI_NAME}
AMI description:  ${AMI_DESCRIPTION}
AMI public:       ${_opt_public}
Instance type:    ${INSTANCE_TYPE}
NGCP version:     ${NGCP_VERSION}
*** End of Status report ***"

if [ -r ec2_report_amis.txt ] ; then
  echo "*** Status report for AMIs in all regions ***"
  cat ec2_report_amis.txt
fi

# Jenkins setup specific
TESTHOST="${HOSTNAME}"
case "${NGCP_RELEASE}" in
  latest)
    VMVERSION="trunk"
    ;;

  *)
    VMVERSION="${NGCP_RELEASE}"
    ;;
esac

generate_ec2_report

rm -f "${USER_DATA_FILE}" "${LOGFILE_RUN}" "${LOGFILE_AMI}"