#!/bin/bash
#  Copyright Contributors to the Feilong Project.
#  SPDX-License-Identifier: Apache-2.0

###############################################################################
# Copyright 2017,2020 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#                            #
###############################################################################
# COMPONENT: unpackdiskimage                                                  #
#                                                                             #
# Deploys a disk iamge to the disk at the specified channel ID on the         #
# specified z/VM guest system.                                                #
###############################################################################

source /opt/zthin/lib/zthinshellutils

###############################################################################
### FUNCTIONS #################################################################
###############################################################################

function printCMDDescription {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Prints a short description of this command.
  # @Overrides:
  #   printCMDDescription{} in "zthinshellutils".
  # @Code:
  echo -n "Deploys a disk image to the disk at the specified channel ID on "
  echo    "the specified z/VM guest system."
} #printCMDDescription{}

###############################################################################

function parseArgs {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Parses and checks command-line arguments.
  # @Code:
  # Non-local variables in this function are intentionally non-local.
  isOption -h --help "     Print this help message."   && printHelp='true'
  isOption -v --verbose "  Print verbose output."      && verbose='-v'
  isOption -x --debug "    Print debugging output."    && debug='-x'

  if [[ ${#args[@]} == 4 ]]; then
    getPositionalArg 1 fcpChannel
    getPositionalArg 2 wwpn
    getPositionalArg 3 lun
    getPositionalArg 4 imageFile

    # Make channel, WWPN, and LUN lower case
    fcpChannel=$(echo ${fcpChannel} | tr '[:upper:]' '[:lower:]')
    wwpn=$(echo ${wwpn} | tr '[:upper:]' '[:lower:]')
    lun=$(echo ${lun} | tr '[:upper:]' '[:lower:]')
  elif [[ ${#args[@]} == 6 ]]; then
    getPositionalArg 1 fcpChannel
    getPositionalArg 2 wwpn
    getPositionalArg 3 lun
    getPositionalArg 4 ignitionUrl
    getPositionalArg 5 nicID
    getPositionalArg 6 ipConfig
    diskType='SCSI'
    skipcopy='YES'
    # Make channel, WWPN, and LUN lower case
    fcpChannel=$(echo ${fcpChannel} | tr '[:upper:]' '[:lower:]')
    wwpn=$(echo ${wwpn} | tr '[:upper:]' '[:lower:]')
    lun=$(echo ${lun} | tr '[:upper:]' '[:lower:]')
  elif [[ ${#args[@]} == 7 ]]; then
    getPositionalArg 1 userID
    getPositionalArg 2 channelID
    getPositionalArg 3 imageFile
    getPositionalArg 4 ignitionUrl
    getPositionalArg 5 diskType
    getPositionalArg 6 nicID
    getPositionalArg 7 ipConfig
  elif [[ ${#args[@]} == 8 ]]; then
    getPositionalArg 1 fcpChannel
    getPositionalArg 2 wwpn
    getPositionalArg 3 lun
    getPositionalArg 4 imageFile
    getPositionalArg 5 ignitionUrl
    getPositionalArg 6 diskType
    getPositionalArg 7 nicID
    getPositionalArg 8 ipConfig

    # Make channel, WWPN, and LUN lower case
    fcpChannel=$(echo ${fcpChannel} | tr '[:upper:]' '[:lower:]')
    wwpn=$(echo ${wwpn} | tr '[:upper:]' '[:lower:]')
    lun=$(echo ${lun} | tr '[:upper:]' '[:lower:]')
  else
    getPositionalArg 1 userID
    getPositionalArg 2 channelID
    getPositionalArg 3 imageFile
  fi

  if [[ $printHelp ]]; then
    printHelp
    exit 0
  fi
  local badOptions=$(getBadOptions)
  if [[ $badOptions ]]; then
    echo "ERROR: ${badOptions}"
    printCMDUsage
    exit 1
  fi

  if [[ ${#args[@]} == 4 && (! $fcpChannel || ! $wwpn || ! lun || ! $imageFile) ]]; then
    echo 'ERROR: Missing required parameter.'
    printCMDUsage
    exit 1
  elif [[ ${#args[@]} == 3 && (! $userID || ! $channelID || ! $imageFile) ]]; then
    echo 'ERROR: Missing required parameter.'
    printCMDUsage
    exit 1
  elif [[ ${#args[@]} == 7 && (! $userID || ! $channelID || ! $imageFile || ! $ignitionUrl || ! $diskType || ! $nicID || ! $ipConfig) ]]; then
    echo 'ERROR: Missing required parameter.'
    printCMDUsage
    exit 1
  elif [[ ${#args[@]} == 6 && (! $fcpChannel || ! $wwpn || ! $lun || ! $ignitionUrl || ! $nicID || ! $ipConfig) ]]; then
    echo 'ERROR: Missing required parameter.'
    printCMDUsage
    exit 1
  elif [[ ${#args[@]} == 8 && (! $fcpChannel || ! $wwpn || ! $lun  || ! $imageFile || ! $ignitionUrl || ! $diskType || ! $nicID || ! $ipConfig) ]]; then
    echo 'ERROR: Missing required parameter.'
    printCMDUsage
    exit 1
  fi

  # Remove old traces beyond the number specified in /var/opt/zthin/settings.conf. If the
  # specified number is 0, less than 0, or not actually a number, then we skip
  # this cleanup.
  if [[ $keepOldTraces -gt 0 ]]; then
    local removeTraces=$(($(ls -1 /var/log/zthin/unpackdiskimage_trace_* | wc -l) -
                          ${keepOldTraces}))
    if [[ $removeTraces -gt 0 ]]; then
      for trace in $(ls -1 /var/log/zthin/unpackdiskimage_trace_* |
                     head -${removeTraces}); do
        rm -f $trace
      done
    fi
  fi

  timestamp=$(date -u --rfc-3339=ns | sed 's/ /-/;s/\.\(...\).*/.\1/' | sed 's/:/_/g')
  logFile=/var/log/zthin/unpackdiskimage_trace_${timestamp}.txt
  if [[ $debug ]]; then
    exec 2> >(tee -a $logFile)
    set -x
  else
    exec 2> $logFile
    set -x
  fi
  inform "unpackdiskimage ${args} start time: ${timestamp}"

  if [[ ${#args[@]} == 4 && (${#wwpn} < 18 || ${#lun} < 18) ]]; then
    echo 'ERROR: WWPN and LUN must be a 16 byte number.'
    exit 1
  fi

  if [[ ${#args[@]} == 4 ]]; then
    echo "FCP CHANNEL:    \"$fcpChannel\""
    echo "WWPN:           \"$wwpn\""
    echo "LUN:            \"$lun\""
    echo "IMAGE FILE:     \"$imageFile\""
    echo ""
  elif [[ ${#args[@]} == 7 ]]; then
    echo "SOURCE USER ID: \"$userID\""
    echo "DISK CHANNEL:   \"$channelID\""
    echo "IMAGE FILE:     \"$imageFile\""
    echo "IGNITION URL:   \"$ignitionUrl\""
    echo "DISK TYPE:      \"$diskType\""
    echo "NIC ID:         \"$nicID\""
    echo "IP config:      \"$ipConfig\""
    echo ""
  elif [[ ${#args[@]} == 8 ]]; then
    echo "FCP CHANNEL:    \"$fcpChannel\""
    echo "WWPN:           \"$wwpn\""
    echo "LUN:            \"$lun\""
    echo "IMAGE FILE:     \"$imageFile\""
    echo "IGNITION URL:   \"$ignitionUrl\""
    echo "DISK TYPE:      \"$diskType\""
    echo "NIC ID:         \"$nicID\""
    echo "IP config:      \"$ipConfig\""
    echo ""
  elif [[ ${#args[@]} == 6 ]]; then
    echo "FCP CHANNEL:    \"$fcpChannel\""
    echo "WWPN:           \"$wwpn\""
    echo "LUN:            \"$lun\""
    echo "IGNITION URL:   \"$ignitionUrl\""
    echo "DISK TYPE:      \"$diskType\""
    echo "NIC ID:         \"$nicID\""
    echo "IP config:      \"$ipConfig\""
    echo ""
  else
    echo "SOURCE USER ID: \"$userID\""
    echo "DISK CHANNEL:   \"$channelID\""
    echo "IMAGE FILE:     \"$imageFile\""
    echo ""
  fi
} #parseArgs{}

###############################################################################

function checkSanity {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Performs basic checks to ensure that a successful deploy can reasonably
  #   be expected.
  # @Code:
  # Make sure the specified image file exists.
  if [[ $skipcopy != 'YES' && ! -f $imageFile ]]; then
    printError "The specified image file was not found: $imageFile."
    exit 3
  fi

  if [[ -n "$diskType" ]]; then
    inform "The image file is for RHCOS, no need to check image header. "
    format='RHCOS'
    return
  fi

  # Determine type of source disk from image header.
  local diskTag="$(dd if=$imageFile bs=1 count=20 2>/dev/null)"
  if [[ $diskTag = 'xCAT FCP Disk Image:' ]]; then
    # Intentionally non-local variable.
    format='FCP'
  elif [[ $diskTag = 'xCAT FBA Disk Image:' ]]; then
    # Intentionally non-local variable.
    format='FBA'
  elif [[ $diskTag = 'xCAT FBA Part Image:' ]]; then
    # Intentionally non-local variable.
    format='FBApt'
  elif [[ $diskTag = 'xCAT CKD Disk Image:' ]]; then
    # Intentionally non-local variable.
    format='CKD'
  else
    printError "Disk image type not recognized"
    exit 3
  fi

  # Determine header length and header and save in non-local variables.
  header="$(dd if=$imageFile bs=1 count=10 skip=37 2>/dev/null)"
  if [[ $header == HLen:* ]]; then
    # Get length value and remove leading zeroes
    headerLen="$(dd if=$imageFile bs=1 count=4 skip=43 2>/dev/null | sed 's/0*//')"
    imageBlockSize=$blockSize
    headerBlocks=$(( $headerLen / $blockSize + 1))
  else
    headerLen=36
    imageBlockSize=$headerLen
    headerBlocks=1
  fi
  header="$(dd if=$imageFile bs=1 count=$headerLen 2>/dev/null)"

  # Determine gzip compression level and save in non-local variable.
  if (( headerLen >= 55 )); then
    gzipCompression=${header:54:1}
  else
    inform "Version 1 image file detected."
    gzipCompression=6
  fi
  inform "Image file compression level: $gzipCompression"

  if [[ $userID && $(isSystemActive $userID) ]]; then
    printError 'The specified target system is currently running.'
    exit 3
  fi

  # Intentionally non-local variable.
  passedSanityChecks='true'
} #checkSanity{}

###############################################################################

function resizePartition {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Resize a Linux ECKD/FBA disk by increasing the last partition of this disk
  #   to use the additional space and resize the filesystem to use the new
  #   partition size.
  # @Returns:
  #   0 if the partition resize was successful.
  #   1 if error
  # @Parameters:
  local devNode=$1
  # @Code:

  # Identify the last partition on the specific disk
  partitionNum=`blkid | grep ${devNode} | wc -l`
  lastPartition=${devNode}${partitionNum}

  # Find the filesystem of the last partition on the disk
  fs=`blkid | grep ${lastPartition} | awk '{print $NF}' | sed -e 's/^TYPE=//' | sed 's/\"//g'`
  rc=$?
  if (( rc != 0 )); then
    printError "Unable to obtain the file system type on disk: ${userID}:${channelID} returns rc: $rc, out: $out"
    return 1
  fi

  if [[ $fs == "xfs" ]]; then
    # the file system will be resized by cloud-init
    inform "The file system type is XFS."

  # Resize the ext filesystem
  elif [[ $fs =~ "ext" ]]; then
    #Resize the ext filesystem.
    out=`e2fsck -yf ${lastPartition} 2>&1`
    rc=$?

    #Considering that the e2fsck version of rhel77 is too low to support 
    #the rhel8 guest VM, so the return value of 8 is ignored.
    if (( rc != 0 && rc != 1 && rc != 2 && rc != 8 )); then
      printError "'e2fsck -f ${lastPartition}' returns rc: $rc, out: $out"
      return 1
    fi
  elif [[ $fs =~ "LVM2_member" ]]; then
    inform "LVM partitions found, Resize physical volume(s) by pvresize."
    output=$(which pvresize)
    if [[ $? -ne 0 ]]; then
      inform "(Warning):Resize abort for pvresize command not found. Please manually extend LVM partition"
      return 0
    fi

    inform "Resize physical volume(s) type $fs lastPartition: $lastPartition with pvresize."
    out=`pvresize  -v ${lastPartition}`
    if [[ $? -ne 0 ]]; then
      inform "(Warning):pvresize execution failed. Please check and manually resize LVM partition"
    else
      inform "Successful Resize physical volume(s), type $fs lastPartition: $lastPartition, out: $out."
    fi
  else
    inform "(Warning) The file system type $fs is not support to be increased"
  fi

  return 0
} #resizePartition{}

###############################################################################

function resizeECKD {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description: 
  #   Resize a Linux ECKD disk by increasing the partition to use the 
  #   additional space and resize the filesystem to use the new 
  #   partition size.
  # @Returns:
  #   0 if the resize was successful.
  #   1 if there was an error with setting the disk offline.
  # @Parameters:
  #   None
  # @Code:
  local alias
  local devNode
  local out

  # Increase the partition to use the rest of the disk.
  inform "Increasing partition space and file system to use the larger target disk."
  
  # reconnect in normal mode (non-raw track access mode)
  disconnectDisk $userID $channelID 0
  connectDisk $userID $channelID 'w' 0
  if (( $? )); then
    printError "Failed to connect disk: ${userID}:${channelID}"
    return 1
  fi
  alias=$(getDiskAlias $userID $channelID)
  devNode=$(findLinuxDeviceNode $alias)

  local partitionsCount=`blkid | grep ${devNode} | wc -l`
  if [[ $partitionsCount -gt 1 ]]; then
    inform "Multiple partitions are detected on root disk. The last partition will be extended."
    partnum=`parted -m ${devNode} print | tail -n 1 | cut -d':' -f1`
    # Input redirection needed because fdasd does not have a command line parameter for
    #    u   re-create VTOC re-using existing partition sizes
    out=`fdasd ${devNode} <<-END
	u
	y
	w
	END`
    rc=$?
    if (( rc != 0 )); then
      printError "When recreating the VTOC, 'fdasd' to  ${devNode} returns rc: $rc, out: $out"
      return 1
    fi
    out=`parted ${devNode} resizepart $partnum 100%`
    rc=$?
    if (( rc != 0 )); then
      printError "When resizing the dasd, 'parted' to  ${devNode} returns rc: $rc, out: $out"
      return 1
    fi
  else
    # Resize the partition to take the whole disk.
    # Note: We will see a message indicating that "device is not fully formatted".
    #       We will ignore that because we are redoing the parition to take 
    #       advantage of the increased size.
    # Add dasdview to debug, currently zCC only supports CDL disk layout.
    dasdview -x ${devNode}
    out=`fdasd -a ${devNode}`
    rc=$?
    if (( rc != 0 )); then
      if (( rc == 255 )); then
        sleep 0.5
        out=`blockdev --rereadpt ${devNode}`
        rc=$?
        if (( rc != 0 )); then
          printError "When resizing the dasd, 'blockdev --rereadpt' to  ${devNode} returns rc: $rc, out: $out"
          return 1
        fi
      else
        printError "When resizing the dasd, 'fsdasd -a' to  ${devNode} returns rc: $rc, out: $out"
        return 1
      fi
    fi
  fi

  # Reset the device by disconnecting and then reconnecting
  disconnectDisk $userID $channelID 0
  connectDisk $userID $channelID 'w' 0
  if (( $? )); then
    printError "Failed to connect disk: ${userID}:${channelID}"
    return 1
  fi
  alias=$(getDiskAlias $userID $channelID)
  devNode=$(findLinuxDeviceNode $alias)

  # Get the current geometry of the partition
  out=`fdasd -sp ${devNode}`
  rc=$?
  if (( rc != 0 )); then
    printError "unable to obtain partition geometry, 'fsdasd -sp' to  ${devNode} returns rc: $rc, out: $out"
    return 1
  fi

  #Resize the file system of last partition on specific disk to use the additional disk space
  resizePartition ${devNode}
  if (( $? )); then
    printError "Failed to resize the last partition on disk: ${userID}:${channelID}"
    return 1
  fi
  return 0

} #resizeECKD{}

###############################################################################

function resizeFBA {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Resize a Linux FBA disk by increasing the last partition to use the
  #   additional space and resize the filesystem to use the new
  #   partition size.
  # @Returns:
  #   0 if the resize was successful.
  #   1 if there was an error with setting the disk offline.
  # @Parameters:
  #   None
  # @Code:
  local alias
  local devNode

  # Since FBA disk only have single partition, increase the file system on it to use the entire disk space
  inform "Increasing partition space and file system to use the larger target disk."

  # Reconnect disk
  disconnectDisk $userID $channelID 0
  connectDisk $userID $channelID 'w' -1
  if (( $? )); then
    printError "Failed to connect disk: ${userID}:${channelID}"
    return 1
  fi

  alias=$(getDiskAlias $userID $channelID)
  devNode=$(findLinuxDeviceNode $alias)

  local partitionsCount=`blkid | grep ${devNode} | wc -l`
  if [[ $partitionsCount -gt 1 ]]; then
    printError "Multiple partitions are detected on root disk, extend is not supported. Only single partition can be extended."
    return 1
  fi
  # Resize the file system of last partition on specific disk to use the additional disk space
  resizePartition ${devNode}
  if (( $? )); then
    printError "Failed to resize the last partition on disk: ${userID}:${channelID}"
    return 1
  fi
  return 0
} #resizeFBA{}

function mount_boot_partition {
    # The location of the mountpoint
    mountpoint=$1

    if [ ! -d $mountpoint ]; then
        printError "mountpoint $mountpoint must be a directory"
        exit 1
    fi

    # Get the boot partition
    set -o pipefail
    let retry=0
    while true
    do
        eval $(lsblk "${DEST_DEV}" --output LABEL,NAME --pairs | grep 'LABEL="boot' | tr ' ' '\n' | tail -n 1)
        if [[ ${NAME} =~ "mpath" ]]; then
            mount "/dev/mapper/${NAME}" "$mountpoint"
        else
            mount "/dev/${NAME}" "$mountpoint"
        fi
        RETCODE=$?
        if [[ $RETCODE -ne 0 ]]; then
            if [[ $retry -lt 30 ]]; then
                # retry and sleep to allow udevd to populate the disk
                sleep 1
                let retry=$retry+1
                continue
            fi
            printError "failed mounting boot partition"
            exit 1
        fi
        break;
    done
    set +o pipefail
} #mount_boot_partition{}

### DEPLOY FUNCTIONS ##########################################################
###############################################################################

function deployDiskImage {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Deploy the disk image.
  # @Code:

  function deployCKDImage {
    : SOURCE: ${BASH_SOURCE}
    : STACK:  ${FUNCNAME[@]}
    # @Description:
    #   Deploy the specified count-key-data disk image.
    # @Code:
    local out
    local rc
    local errorFile

    connectDisk $userID $channelID 'w' 1
    if (( $? )); then
      printError "Failed to connect disk: ${userID}:${channelID}"
      exit 3
    fi

    local alias=$(getDiskAlias $userID $channelID)
    if [[ $(vmcp q v $alias | grep 'CYL ON DASD') ]]; then
      local diskSize="$(dd if=$imageFile bs=1 count=16 skip=20 2>/dev/null |
                        awk '{print $1}')"
      local targetDiskSize=`vmcp q v $alias | awk '{print $6}'`
      if [[ $diskSize -eq 0 ]]; then
          printError "$imageFile does not contain cylinder/record information.  This appears to be a dummy image file."
          exit 3
      fi
      if [[ $diskSize -le $targetDiskSize ]]; then
        # Deploy the image into the target disk
        errorFile=`mktemp -p /var/log/zthin -t unpackStderr.XXXXXXXXXX`
        if (( $? )); then
          printError "Failed to create a temporary file in the /var/log/zthin directory"
          exit 3
        fi
        wipefs --all --force "/dev/disk/by-path/ccw-0.0.${alias}"
        if (( gzipCompression == 0 )); then
          declare -a stages=('overall_placeholder' 'dd' 'ckddecode')
          dd if=$imageFile bs=$imageBlockSize skip=$headerBlocks 2>>$errorFile |
            ckddecode /dev/disk/by-path/ccw-0.0.${alias} $targetDiskSize 2>>$errorFile
        else
          declare -a stages=('overall_placeholder' 'dd' 'zcat' 'ckddecode')
          dd if=$imageFile bs=$imageBlockSize skip=$headerBlocks 2>>$errorFile |
            zcat  2>>$errorFile|
              ckddecode /dev/disk/by-path/ccw-0.0.${alias} $targetDiskSize 2>>$errorFile
        fi
        declare -a pipeRC=($PIPESTATUS ${PIPESTATUS[@]})
        out=`cat $errorFile | tr '\n' ' '`
        rm $errorFile

        if (( ${pipeRC[0]} != 0 )); then
          getStageFailures stages[@] pipeRC[@]
          printError "Failed deploying disk image $(basename $imageFile) at stage(rc): $getStageFailuresOut $out"
          exit 3
        fi
      else
        printError "Target disk is too small for specified image, deploy to target disk:$targetDiskSize cylinders, image need minimum:$diskSize cylinders"
        exit 3
      fi

      if (( diskSize < targetDiskSize )); then
        resizeECKD
        rc=$?
        if (( rc != 0 )); then
          exit 3
        fi
      fi
    else
      printError "Specified image is of a Count-Key-Data volume, but specified\
                  disk is not a Count-Key-Data volume."
      exit 3
    fi
  } #deployCKDImage{}

  function deployFBAImage {
    : SOURCE: ${BASH_SOURCE}
    : STACK:  ${FUNCNAME[@]}
    # @Description:
    #   Deploy the specified fixed-block disk image.
    # @Code:
    local out
    local rc
    local errorFile

    connectDisk $userID $channelID 'w' -1
    if (( $? )); then
      printError "Failed to connect disk: ${userID}:${channelID}"
      exit 3
    fi
    local alias=$(getDiskAlias $userID $channelID)
    if [[ $(vmcp q v $alias | grep 'BLK ON DASD') ]]; then
      local diskSize="$(dd if=$imageFile bs=1 count=16 skip=20 2>/dev/null |
                        awk '{print $1}')"
      local targetDiskSize=`vmcp q v $alias | awk '{print $6}'`
      if [[ $diskSize -eq 0 ]]; then
          printError "$imageFile does not contain block information.  This appears to be a dummy image file."
          exit 3
      fi
      if [[ ${diskSize} -le ${targetDiskSize} ]]; then
        errorFile=`mktemp -p /var/log/zthin -t unpackStderr.XXXXXXXXXX`
        if (( $? )); then
          printError "Failed to create a temporary file in the /var/log/zthin directory"
          exit 3
        fi
        wipefs --all --force "/dev/disk/by-path/ccw-0.0.${alias}"
        if (( gzipCompression == 0 )); then
          dd if=$imageFile bs=$imageBlockSize skip=$headerBlocks 2>>$errorFile \
            > /dev/disk/by-path/ccw-0.0.${alias}
          rc=$?
          if (( rc != 0 )); then
            out=`cat $errorFile | tr '\n' ' '`
          fi
        else
          declare -a stages=('overall_placeholder' 'dd' 'zcat')
          dd if=$imageFile bs=$imageBlockSize skip=$headerBlocks 2>>$errorFile |
            zcat 2>>$errorFile > /dev/disk/by-path/ccw-0.0.${alias}
          declare -a pipeRC=($PIPESTATUS ${PIPESTATUS[@]})
          rc=${pipeRC[0]}
          if (( pipeRC[0] != 0 )); then
            out=`cat $errorFile | tr '\n' ' '`
            getStageFailures stages[@] pipeRC[@]
            out="at stage(rc):$getStageFailuresOut $out"
          fi
        fi

        rm $errorFile

        syncfileutil /dev/disk/by-path/ccw-0.0.${alias}

        if (( rc != 0 )); then
          printError "Failed deploying disk image $(basename $imageFile) $out"
          exit 3
        fi
      else
        printError "Target disk is too small for specified image. The image has $diskSize blocks in the header, deploy to a disk with $targetDiskSize blocks, either deploy to a bigger size target disk or create a smaller image."
        exit 3
      fi
    else
      printError "Specified image is of a fixed-block volume, but specified\
                  disk is not a fixed-block volume."
      exit 3
    fi
  } #deployFBAImage{}

  function execZIPL {
    : SOURCE: ${BASH_SOURCE}
    : STACK:  ${FUNCNAME[@]}
    # @Description:
    #   Retrieves initial information for the IPL from disk
    # @Parameters:
    # None
    # @Returns:
    #   0 - If the zipl is executed successfully on target disk
    #   1 - If the zipl failed to be executed on target disk
    # @Code:

    local out
    local rc
    local os

    # Reconnect disk
    disconnectDisk $userID $channelID 0
    connectDisk $userID $channelID 'w' -1
    if (( $? )); then
      printError "Failed to connect disk: ${userID}:${channelID}"
      return 1
    fi
    local alias=$(getDiskAlias $userID $channelID)
    local devNode=$(findLinuxDeviceNode $alias)
    local deviceMountPoint=/mnt/${alias}

    # Identify the last partition on the specific disk
    local partitionNum=`blkid | grep ${devNode} | wc -l`
    local lastPartition=${devNode}${partitionNum}

    # Find the filesystem of the last partition on the disk
    local fs=`blkid | grep ${lastPartition} | awk '{print $NF}' | sed -e 's/^TYPE=//' | sed 's/\"//g'`
    rc=$?
    if (( rc != 0 )); then
      printError "Unable to obtain the file system type on disk: ${userID}:${channelID}"
      return 1
    fi

    mkdir -p $deviceMountPoint
    if [[ $fs == "xfs" ]]; then
      # Mount with nouuid option
      mount -o nouuid /dev/disk/by-path/ccw-0.0.${alias}-part1 $deviceMountPoint
      rc=$?
    else
      mount /dev/disk/by-path/ccw-0.0.${alias}-part1 $deviceMountPoint
      rc=$?
    fi
    if (( rc != 0 )); then
      printError "Unable to mount the FBA disk partition of ${userID}:${channelID}"
      rm -rf $deviceMountPoint
      return 1
    fi

    #Get target os version
    local osRelease="$deviceMountPoint/etc/os-release"
    local slesRelease="$deviceMountPoint/etc/SuSE-release"
    local rhelRelease="$deviceMountPoint/etc/redhat-release"

    if [[ -e $osRelease ]]; then
      os=`cat $osRelease | grep "^ID=" | sed \
        -e 's/ID=//' \
        -e 's/"//g'`
      version=`cat $osRelease | grep "^VERSION_ID=" | sed \
        -e 's/VERSION_ID=//' \
        -e 's/"//g' \
        -e 's/\.//'`
      os=$os$version

    #The /etc/SuSE-release file will be deprecated in sles11.4 and later release
    elif [[ -e $slesRelease ]]; then
      os='sles'
      version=`cat $slesRelease | grep "VERSION =" | sed \
        -e 's/^.*VERSION =//' \
        -e 's/\s*$//' \
        -e 's/.//' \
        -e 's/[^0-9]*([0-9]+).*/$1/'`
      os=$os$version

      # Append service level
      level=`echo $slesRelease | grep "LEVEL =" | sed \
        -e 's/^.*LEVEL =//' \
        -e 's/\s*$//' \
        -e 's/.//' \
        -e 's/[^0-9]*([0-9]+).*/$1/'`
      os=$os'sp'$level

    #The /etc/redhat-release file will be deprecated in rhel7 and later release
    elif [[ -e $rhelRelease ]]; then
      os='rhel'
      version=`cat $rhelRelease | grep -i "Red Hat Enterprise Linux Server" | sed \
        -e 's/[A-Za-z\/\.\(\)]//g' \
        -e 's/^ *//g' \
        -e 's/ *$//g' \
        -e 's/\s.*$//'`
      os=$os$version
    fi

    # Exec zipl command to prepare device for initial problem load
    if [[ $os == sles12* || $os == sles15* ]]; then
      out=`chroot $deviceMountPoint /sbin/zipl -c /boot/zipl/config 2>&1`
      rc=$?
      if (( rc != 0 )); then
        printError "Failed to execute zipl on $os due to $out"
        umount $deviceMountPoint
        rm -rf $deviceMountPoint
        return 1
      fi
    elif [[ $os == sles11* || $os == rhel* || $os == ubuntu* ]]; then
      out=`chroot $deviceMountPoint /sbin/zipl 2>&1`
      rc=$?
      if (( rc != 0 )); then
        printError "Failed to execute zipl on $os due to $out"
        umount $deviceMountPoint
        rm -rf $deviceMountPoint
        return 1
      fi
    elif [[ $os == "" ]]; then
      inform "This is not the root disk, zipl will not be executed on it"
    else
      inform "The os version is: $os, this is not a supported linux distro"
    fi


    #Unmount the target disk
    umount $deviceMountPoint
    rm -rf $deviceMountPoint
    return 0
  } #execZIPL

  function update_zipl_bootloader_fcp {
    inform "Updating zipl"

    local rd_zfcp=""

    # discover other valid wwpns
    local ActiveWWPNs=$(ls /sys/bus/ccw/drivers/zfcp/0.0.$fcp/ | grep 0x)
    inform "All wwpns: ${ActiveWWPNs}"
    for w in ${ActiveWWPNs}
    do
      echo "$lun" > /sys/bus/ccw/drivers/zfcp/0.0.$fcp/${w}/unit_add
    done
    # wait paths ready
    echo add > /sys/bus/ccw/devices/0.0.${fcpChannel}/uevent
    which udevadm &> /dev/null && udevadm settle || udevsettle

    local lszfcp_output=$(lszfcp -HD 2>&1)
    inform "lszfcp output: ${lszfcp_output}"

    local valid_wwpns=`lszfcp -HD | grep ${fcpChannel} | tail -n +2 | awk -F'/' '{print $2}'`
    inform "All valid wwpns: ${valid_wwpns}"
    for ww in ${valid_wwpns}
    do
      rd_zfcp+="rd.zfcp=0.0."$fcpChannel,$ww,$lun" "
    done

    mkdir -p /tmp/$fcpChannel/boot_partition
    mount_boot_partition /tmp/$fcpChannel/boot_partition
    trap 'umount /tmp/$fcpChannel/boot_partition; trap - RETURN' RETURN

    #Get ignition config file
    mkdir -p "/tmp/$fcpChannel/boot_partition/ignition"
    if [ -f "$ignitionUrl" ]; then
        inform "$ignitionUrl is FILE"
        cat $ignitionUrl >/tmp/$fcpChannel/boot_partition/ignition/config.ign 2>/dev/null
    else
        inform "$ignitionUrl is URL"
        let retry=0
        while true
        do
            curl -Lf $ignitionUrl >/tmp/$fcpChannel/boot_partition/ignition/config.ign 2>/dev/null
            RETCODE=$?
            if [ $RETCODE -ne 0 ]
            then
                if [[ $RETCODE -eq 22 && $retry -lt 5 ]]
                then
                    # Network isn't up yet, sleep for a sec and retry
                    sleep 1
                    let retry=$retry+1
                    continue
                else
                    printError "failed fetching ignition config from ${ignitionUrl}: ${RETCODE}"
                    exit 1
                fi
                inform "fetching ignition config from ${ignitionUrl}: ${RETCODE}"
            else
                inform "Successfully get the ignition config"
                break;
            fi
        done
    fi

    blsfile=$(ls /tmp/$fcpChannel/boot_partition/loader/entries/*.conf)
    #Get zipl config file for parameters

    # ipConfig format: <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none[:[<dns1>][:<dns2>]];<mtu>
    # ipsetting format here: <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none[:[<dns1>][:<dns2>]]
    ipsetting=$(echo ${ipConfig%;*})
    # mtu format: <mtu> , example, "1500"
    mtu=$(echo ${ipConfig#*;})
    # nameserver format: nameserver=<nameserver>
    nameserver1=$(echo $ipsetting| awk -F: '{print $8}')
    nameserver2=$(echo $ipsetting| awk -F: '{print $9}')
    # ipsetting format here: <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none:<mtu>
    ipsetting=$(echo ${ipsetting/%none:*/none:}${mtu})

    if [ $nameserver1  ]; then 
        nameserver1='nameserver='${nameserver1}
    fi
    if [ $nameserver2  ]; then 
        nameserver2='nameserver='${nameserver2}
    fi

    echo "$(grep options $blsfile | cut -d' ' -f2-) zfcp.allow_lun_scan=0 rw $rd_zfcp rd.znet=qeth,$nicID,layer2=1,portno=0 ignition.firstboot=1 rd.neednet=1 ip=$ipsetting $nameserver1 $nameserver2" > /tmp/$fcpChannel/zipl_prm
    #Run zipl to update bootloader
    zipl --verbose -p /tmp/$fcpChannel/zipl_prm -i $(ls /tmp/$fcpChannel/boot_partition/ostree/*/*vmlinuz*) -r $(ls /tmp/$fcpChannel/boot_partition/ostree/*/*initramfs*) --target /tmp/$fcpChannel/boot_partition
    if [[ $? -ne 0 ]]; then
        printError "failed updating bootloder"
        exit 1
    fi
    #Update BLS config file for reboot
    sed -i -e 's/ignition.firstboot=1.*//' /tmp/$fcpChannel/zipl_prm
    sed -e '/options/d' $blsfile > /tmp/$fcpChannel/blsfile
    echo "options $(cat /tmp/$fcpChannel/zipl_prm) " >>/tmp/$fcpChannel/blsfile
    cat /tmp/$fcpChannel/blsfile > $blsfile
    blockdev --flushbufs ${DEST_DEV} > /dev/null
  } #update_zipl_bootloader_fcp{}

  function update_zipl_bootloader_eckd {
    inform "${userID}: Updating zipl"
    mkdir -p /tmp/$userID/boot_partition
    mount_boot_partition /tmp/$userID/boot_partition
    trap 'umount /tmp/$userID/boot_partition; trap - RETURN' RETURN

    #Get ignition config file
    mkdir -p "/tmp/$userID/boot_partition/ignition"
    if [ -f "$ignitionUrl" ]; then
        inform "${userID}: $ignitionUrl is FILE"
        cat $ignitionUrl >/tmp/$userID/boot_partition/ignition/config.ign 2>/dev/null
    else
        inform "${userID}: $ignitionUrl is URL"
        let retry=0
        while true
        do
            curl -Lf $ignitionUrl >/tmp/$userID/boot_partition/ignition/config.ign 2>/dev/null
            RETCODE=$?
            if [ $RETCODE -ne 0 ]
            then
                if [[ $RETCODE -eq 22 && $retry -lt 5 ]]
                then
                    # Network isn't up yet, sleep for a sec and retry
                    sleep 1
                    let retry=$retry+1
                    continue
                else
                    printError "failed fetching ignition config from ${ignitionUrl}: ${RETCODE}"
                    exit 1
                fi
                inform "${userID}: fetching ignition config from ${ignitionUrl}: ${RETCODE}"
            else
                inform "${userID}: Successfully get the ignition config"
                break;
            fi
        done
    fi

    blsfile=$(ls /tmp/$userID/boot_partition/loader/entries/*.conf)
    #Get zipl config file for parameters

    # ipConfig format: <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none[:[<dns1>][:<dns2>]];<mtu>
    # ipsetting format here: <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none[:[<dns1>][:<dns2>]]
    ipsetting=$(echo ${ipConfig%;*})
    # mtu format: <mtu> , example, "1500"
    mtu=$(echo ${ipConfig#*;})
    # nameserver format: nameserver=<nameserver>
    nameserver1=$(echo $ipsetting| awk -F: '{print $8}')
    nameserver2=$(echo $ipsetting| awk -F: '{print $9}')
    # ipsetting format here: <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none:<mtu>
    ipsetting=$(echo ${ipsetting/%none:*/none:}${mtu})

    if [ $nameserver1  ]; then 
        nameserver1='nameserver='${nameserver1}
    fi
    if [ $nameserver2  ]; then 
        nameserver2='nameserver='${nameserver2}
    fi
    
    echo "$(grep options $blsfile | cut -d' ' -f2-) rd.dasd=0.0.$channelID rd.znet=qeth,$nicID,layer2=1,portno=0 ignition.firstboot=1 rd.neednet=1 ip=$ipsetting $nameserver1 $nameserver2" > /tmp/$userID/zipl_prm
    #Run zipl to update bootloader
    zipl --verbose -p /tmp/$userID/zipl_prm -i $(ls /tmp/$userID/boot_partition/ostree/*/*vmlinuz*) -r $(ls /tmp/$userID/boot_partition/ostree/*/*initramfs*) --target /tmp/$userID/boot_partition
    if [[ $? -ne 0 ]]; then
        printError "${userID}: failed updating bootloder"
        exit 1
    fi
    #Update BLS config file for reboot
    sed -i -e 's/ignition.firstboot=1.*//' /tmp/$userID/zipl_prm
    sed -e '/options/d' $blsfile > /tmp/$userID/blsfile
    echo "options $(cat /tmp/$userID/zipl_prm) " >>/tmp/$userID/blsfile
    cat /tmp/$userID/blsfile > $blsfile

    inform "${userID}: Update zipl completed."
  } #update_zipl_bootloader_eckd{}

  function deployFBAptImage {
    : SOURCE: ${BASH_SOURCE}
    : STACK:  ${FUNCNAME[@]}
    # @Description:
    #   Deploy the specified fixed-block disk image.
    # @Code:
    local out
    local rc
    local errorFile

    connectDisk $userID $channelID 'w' -1
    if (( $? )); then
      printError "Failed to connect disk: ${userID}:${channelID}"
      exit 3
    fi
    local alias=$(getDiskAlias $userID $channelID)
    if [[ $(vmcp q v $alias | grep 'BLK ON DASD') ]]; then
      local diskSize="$(dd if=$imageFile bs=1 count=16 skip=20 2>/dev/null |
                        awk '{print $1}')"
      local targetDiskSize=`vmcp q v $alias | awk '{print $6}'`

      if [[ $diskSize -eq 0 ]]; then
          printError "$imageFile does not contain block information.  This appears to be a dummy image file."
          exit 3
      fi

      if [[ ${diskSize} -le ${targetDiskSize} ]]; then
        errorFile=`mktemp -p /var/log/zthin -t unpackStderr.XXXXXXXXXX`
        if (( $? )); then
          printError "Failed to create a temporary file in the /var/log/zthin directory"
          exit 3
        fi
        wipefs --all --force "/dev/disk/by-path/ccw-0.0.${alias}"

        # Zero out the disk header
        out=`dd if=/dev/zero bs=512 count=2 2>&1 of=/dev/disk/by-path/ccw-0.0.${alias}`
        rc=$?
        if (( rc )); then
          out=`echo $out | tr '\n' ' '`
          printError "An error was encountered while zero out the disk header. $out"
          exit 1
        fi
        syncfileutil /dev/disk/by-path/ccw-0.0.${alias}
        disconnectDisk $userID $channelID 0

        # Reconnect to dd the partition
        connectDisk $userID $channelID 'w' -1
        if (( $? )); then
          printError "Failed to connect disk: ${userID}:${channelID}"
          exit 3
        fi
        alias=$(getDiskAlias $userID $channelID)

        if [[ ! -L /dev/disk/by-path/ccw-0.0.${alias}-part1 ]]; then
          printError "Failed to detect the partition on FBA disk: ${userID}:${channelID}"
          exit 3
        fi

        if (( gzipCompression == 0 )); then
          dd if=$imageFile bs=$imageBlockSize skip=$headerBlocks 2>>$errorFile \
            > /dev/disk/by-path/ccw-0.0.${alias}-part1
          rc=$?
          if (( rc != 0 )); then
            out=`cat $errorFile | tr '\n' ' '`
          fi
        else
          declare -a stages=('overall_placeholder' 'dd' 'zcat')
          dd if=$imageFile bs=$imageBlockSize skip=$headerBlocks 2>>$errorFile |
            zcat 2>>$errorFile > /dev/disk/by-path/ccw-0.0.${alias}-part1
          declare -a pipeRC=($PIPESTATUS ${PIPESTATUS[@]})
          rc=${pipeRC[0]}
          if (( pipeRC[0] != 0 )); then
            out=`cat $errorFile | tr '\n' ' '`
            getStageFailures stages[@] pipeRC[@]
            out="at stage(rc):$getStageFailuresOut $out"
          fi
        fi

        rm $errorFile

        syncfileutil /dev/disk/by-path/ccw-0.0.${alias}-part1

        if (( rc != 0 )); then
          printError "Failed deploying disk image $(basename $imageFile) $out"
          exit 3
        fi
      else
        printError "Target disk is too small for specified image, deploy to target disk:$targetDiskSize blocks, image need minimum:$diskSize blocks"
        exit 3
      fi

      # Resize filesystem on target disk's last partition
      if (( diskSize < targetDiskSize )); then
        resizeFBA
        rc=$?
        if (( rc != 0 )); then
          exit 3
        fi
      fi
    else
      printError "Specified image is of a fixed-block volume, but specified\
                  disk is not a fixed-block volume."
      exit 3
    fi

    execZIPL
    rc=$?
    if (( rc != 0 )); then
      exit 3
    fi
  } #deployFBAptImage{}

  function deployFCPImage {
    : SOURCE: ${BASH_SOURCE}
    : STACK:  ${FUNCNAME[@]}
    # @Description:
    #   Deploy the specified fixed-block disk image.
    # @Code:
    local out
    local rc
    local errorFile

    connectFcp ${fcpChannel} ${wwpn} ${lun}
    if (( $? )); then
      printError "Failed to connect disk: ccw-0.0.${fcpChannel}-${wwpn}:${lun}."
      exit 3
    fi

    local targetDiskSize="$(/usr/bin/sg_readcap /dev/disk/by-path/ccw-0.0.${fcpChannel}-zfcp-${wwpn}:${lun} | 
    egrep -i "Device size:" |
      awk '{printf("%0.0f", $3/512)}')"

    local diskSize="$(dd if=$imageFile bs=1 count=16 skip=20 2>/dev/null |
                      awk '{print $1}')"
    if [[ $diskSize -eq 0 ]]; then
      printError "$imageFile does not contain block information.  This appears to be a dummy image file."
      exit 3
    fi
    if [[ $diskSize -le $targetDiskSize ]]; then
      errorFile=`mktemp -p /var/log/zthin -t unpackStderr.XXXXXXXXXX`
      if (( $? )); then
        printError "Failed to create a temporary file in the /var/log/zthin directory"
        exit 3
      fi
      wipefs --all --force "/dev/disk/by-path/ccw-0.0.${fcpChannel}-zfcp-${wwpn}:${lun}"
      if (( gzipCompression == 0 )); then
        dd if=$imageFile bs=$imageBlockSize skip=$headerBlocks 2>>$errorFile \
          > /dev/disk/by-path/ccw-0.0.${fcpChannel}-zfcp-${wwpn}:${lun}
        rc=$?
        if (( rc != 0 )); then
          out=`cat $errorFile | tr '\n' ' '`
        fi
      else
        declare -a stages=('overall_placeholder' 'dd' 'zcat')
        dd if=$imageFile bs=$imageBlockSize skip=$headerBlocks 2>>$errorFile |
          zcat 2>>$errorFile > /dev/disk/by-path/ccw-0.0.${fcpChannel}-zfcp-${wwpn}:${lun}
        declare -a pipeRC=($PIPESTATUS ${PIPESTATUS[@]})
        rc=${pipeRC[0]}
        if (( rc != 0 )); then
          out=`cat $errorFile | tr '\n' ' '`
          getStageFailures stages[@] pipeRC[@]
          out="at stage(rc): $getStageFailuresOut $out"
        fi
      fi

      rm $errorFile

      syncfileutil /dev/disk/by-path/ccw-0.0.${alias}

      if (( rc != 0 )); then
        printError "Failed deploying disk image $(basename $imageFile) $out"
        exit 3
      fi
    else
      printError "Target disk is too small for specified image, deploy to target disk:$targetDiskSize bytes, image need minimum:$diskSize bytes"
      exit 3
    fi
  } #deployFCPImage{}

  function deployRHCOSECKDImage {
    : SOURCE: ${BASH_SOURCE}
    : STACK:  ${FUNCNAME[@]}
    # @Description:
    #   Deploy the specified count-key-data disk image.
    # @Code:
    connectDisk $userID $channelID 'w' 0
    if (( $? )); then
      printError "Failed to connect disk: ${userID}:${channelID}"
      exit 3
    fi

    local alias=$(getDiskAlias $userID $channelID)
    DEST_DEV=$(findLinuxDeviceNode $alias)

    #Wipe any remaining disk labels
    inform "${userID}: Wiping ${DEST_DEV}"
    wipefs --all --force "${DEST_DEV}"
    
    #Check disk type
    local real_disk_type=($(dasdview -x ${DEST_DEV} |grep 'type' |head -1|awk '{print $3}'))
    if [[ $real_disk_type != 'ECKD' ]]; then
        printError "The device type is not ECKD"
        exit 3
    fi   

    #low-level format the ECKD DASD using dasdfmt, if needed
    inform "${userID}: Format ${DEST_DEV} and Copy..."
    if dasdview -x ${DEST_DEV} |grep 'CDL formatted'
    then
        if [ ! $(fdasd -p ${DEST_DEV} | grep blocks\ per\ track | awk '{print $5}') ]; then
            inform "${userID}: Formatting..."
            dasdfmt --blocksize 4096 --disk_layout cdl --mode full -ypv "${DEST_DEV}"
            if [[ $? -ne 0 ]]; then
                printError "failed to format RHCOS DASD device"
                exit 3
            fi
        fi
    else
        #DEST_DEV is not formatted with z/OS compatible disk layout!
        inform "${userID}: Formatting..."
        dasdfmt --blocksize 4096 --disk_layout cdl --mode full -ypv "${DEST_DEV}"
        if [[ $? -ne 0 ]]; then
            printError "failed to format RHCOS DASD device"
            exit 3
        fi
    fi

    #Get blocks for per track of target disk
    inform "${userID}: Getting blocks per track of device ${DEST_DEV}..."
    block_per_tracks=$(fdasd -p ${DEST_DEV} | grep blocks\ per\ track | awk '{print $5}')
    inform "${userID}: Blocks per track of device ${DEST_DEV}: ${block_per_tracks}"
    #the first 2 tracks of the ECKD DASD are reserved
    first_track=2

    # Get boot partition and root partition of target disk
    inform "${userID}: Get boot and root partitions..."
    boot_partition=($(fdisk -b 4096 -l ${imageFile} | grep "${imageFile}p*[0-9]" | head -n 1 | awk '{print $2,$4}'))
    root_partition=($(fdisk -b 4096 -l ${imageFile} | grep "${imageFile}p*[0-9]" | tail -n 1 | head -n 2 | awk '{print $2,$4}'))
    boot_partition+=( $first_track $(( (${boot_partition[1]} + $block_per_tracks - 1) / $block_per_tracks )) )
    root_partition+=($(( ${boot_partition[2]} + ${boot_partition[3]} )) "last")
    mkdir -p /tmp/${userID} && touch /tmp/${userID}/fdasd_conf
    cat > "/tmp/${userID}/fdasd_conf" <<- EOF
[$first_track,$(( ${root_partition[2]} - 1 )),native]
[${root_partition[2]},${root_partition[3]},native]
EOF

    # format the ECKD DASD using fdasd program
    inform "${userID}: Create partitions..."
    fdasd --silent --config /tmp/${userID}/fdasd_conf "${DEST_DEV}"
    RETCODE=$?
    if [[ $RETCODE -ne 0 ]]; then
        local os=""
        local osRelease="/etc/os-release"
        if [[ -e $osRelease ]]; then
            os=`cat $osRelease | grep "^ID=" | sed \
              -e 's/ID=//' \
              -e 's/"//g'`
            version=`cat $osRelease | grep "^VERSION_ID=" | sed \
              -e 's/VERSION_ID=//' \
              -e 's/"//g' \
              -e 's/\.//'`
            os=$os$version
        fi
 
        if [[ $os == rhel7* && $RETCODE -eq 255 ]]; then
            inform "Ignore the error while rereading partition table."
        elif [[ $os == rhel8* && $RETCODE -eq 1 ]]; then
            inform "Ignore the error while rereading partition table."
        else
            printError "failed to set partition for RHCOS DASD device"
            exit 3
        fi
    fi

    # copy the content of each partition
    inform "${userID}: Coping partitions..."
    set -- ${boot_partition[@]} ${root_partition[@]}
    while [ $# -gt 0 ]; do
        dd bs=4096 if="${imageFile}" iflag=fullblock of="${DEST_DEV}" \
            status=progress \
            skip="$1" \
            count="$2" \
            seek="$(( $3 * $block_per_tracks ))"
        if [[ $? -ne 0 ]]; then
            printError "failed to write image to RHCOS DASD device"
            exit 3
        fi
        shift 4
    done

    # Update bootloader records
    update_zipl_bootloader_eckd

    inform "${userID}: Flushing disk buffer..."
    blockdev --flushbufs ${DEST_DEV} > /dev/null

    inform "${userID}: Install complete"

  } #deployRHCOSECKDImage{}

  function deployRHCOSFCPImage {
    : SOURCE: ${BASH_SOURCE}
    : STACK:  ${FUNCNAME[@]}
    # @Description:
    #   According to the requirement, copy the specified SCSI disk image or skip the copy, then update the bootloader records
    # @Code:
    connectFcp $fcpChannel $wwpn $lun
    if (( $? )); then
      printError "Failed to connect disk: ccw-0.0.${fcpChannel}-${wwpn}:${lun}."
      exit 3
    fi

    local DEST_DEV=`ls -al /dev/disk/by-path/ccw-0.0.${fcpChannel}-zfcp-${wwpn}:${lun} | tr '/' ' ' | awk '{print \$NF}'`
    DEST_DEV=/dev/$DEST_DEV

    if [[ $skipcopy != 'YES' ]]; then 
        inform "Copy image..."
        dd if=${imageFile} bs=512 iflag=fullblock of="${DEST_DEV}" status=progress
        if [[ $? -ne 0 ]]; then
            printError "failed to write image to RHCOS SCSI device"
            exit 3
        fi
    fi

    blockdev --flushbufs ${DEST_DEV} > /dev/null

    # Update bootloader records
    update_zipl_bootloader_fcp

    inform "Install complete"

  } #deployRHCOSFCPImage{}

  if [[ $userID ]]; then
    inform "Deploying image to ${userID}'s disk at channel ${channelID}."
  else
    inform "Deploying image to FCP disk at channel ${fcpChannel}."
  fi


  # Technically this only apply for z15+ machine as we can speed up the
  # zip process through zEDC, and no harm if we set for z15-
  export DFLTCC=1
  export DFLTCC_LEVEL_MASK=0x1fe

  if [[ $format = 'FCP' ]]; then
    deployFCPImage
  elif [[ $format = 'FBA' ]]; then
    deployFBAImage
  elif [[ $format = 'FBApt' ]]; then
    deployFBAptImage
  elif [[ $format = 'CKD' ]]; then
    deployCKDImage
  elif [[ $format = 'RHCOS' && $diskType = 'ECKD' ]]; then
    deployRHCOSECKDImage
  elif [[ $format = 'RHCOS' && $diskType = 'SCSI' ]]; then
    deployRHCOSFCPImage
  else
    # It should not be possible to get here after our sanity checks, but in
    # any case...
    printError "Disk image type not recognized."
    exit 3
  fi

  # unpackdiskimage is an indenpendent process, so don't need unset it
  # unset DFLTCC
  # unset DFLTCC_LEVEL_MASK

} #deployDiskImage{}


###############################################################################
### SET TRAP FOR CLEANUP ON EXIT ##############################################
###############################################################################

function cleanup {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Clean up lock files, disk links, and (if we passed the sanity check but
  #   failed after creating one or more z/VM user IDs, those newly-created
  #   user IDs).
  # @Code:
  # Nothing to do for help or version options.
  if [[ $printHelp ]]; then
    return
  fi

  if [[ $successful ]]; then
    if [[ $userID ]]; then
      inform "Deploying image for ${userID} successful."
    else
      inform "Deploying image to FCP disk at channel ${fcpChannel} successful."
    fi
    # Only keep traces of failed disk-image creation attempt unless overriden
    # by a configuration property.
    if [[ -e $logFile ]]; then
      if [[ $saveAllLogs ]]; then
        inform "A detailed trace can be found at: ${logFile}"
      else
        rm -f $logFile
      fi
    fi
  else
    if [[ ! $printHelp ]]; then
      echo -e '\nIMAGE DEPLOYMENT FAILED.'
      [[ $logFile ]] && inform "A detailed trace can be found at: ${logFile}"
    fi
  fi

  # Make sure we've released our connection to the target disk.
  if [[ $userID ]]; then
    disconnectDisk $userID $channelID 0
    if [[ $diskType ]]; then
      rm -rf /tmp/${userID}
    fi
  else
    multipath_enabled=0
    service_output=`systemctl status multipathd | grep "active (running)"`
    if [[ $? -eq 0 ]]; then
      inform "multipathd service is active and running."
      multipath_enabled=1
    fi

    if [[ $multipath_enabled == 1 ]]; then
      fcp_disk_path=/dev/disk/by-path/ccw-0.0.${fcpChannel}-zfcp-${wwpn}:${lun}
      wwid=`/usr/lib/udev/scsi_id --whitelisted ${fcp_disk_path} 2> /dev/null`

      # discover mpath before flush
      multipath
      multipath_output=$(multipath -ll 2>&1)
      inform "multipath output: ${multipath_output}."
      map_name=$(multipath -l $wwid -v 1)

      inform "Cleaning up multipath bindings and wwids: ${map_name} ${wwid}"
      # delete the wwid and refresh multipath map
      multipath -w ${map_name} 1>/dev/null
      multipath -f ${map_name} 1>/dev/null
      if [[ -f /etc/multipath/bindings ]]; then
        sed -i "/${wwid}/d" /etc/multipath/bindings
      fi
      lszfcp_output=$(lszfcp -HD 2>&1)
      inform "lszfcp output: ${lszfcp_output}"
      multipath_output=$(multipath -ll 2>&1)
      inform "multipath output: ${multipath_output}."
      inform "Finish refreshing multipath bindings: ${fcp_disk_path}"
    fi

    # disconnect FCP
    disconnectFcp ${fcpChannel} ${wwpn} ${lun}


    if [[ $diskType ]]; then
      rm -rf /tmp/${fcpChannel}
    fi
  fi

  timestamp=$(date -u --rfc-3339=ns | sed 's/ /-/;s/\.\(...\).*/.\1/')
  if [[ $userID ]]; then
    inform "${userID}: unpackdiskimage end time: ${timestamp}"
  else
    inform "unpackdiskimage to FCP disk at channel ${fcpChannel} end time: ${timestamp}"
  fi
} #cleanup{}

trap 'cleanup' EXIT

trap "echo -e '\nExecution interrupted. Exiting...\n'; exit" SIGINT

###############################################################################
### START EXECUTION ###########################################################
###############################################################################

parseArgs
checkSanity
deployDiskImage
successful='true'

###############################################################################
### END OF SCRIPT #############################################################
###############################################################################

