#!/bin/bash
#  Copyright Contributors to the Feilong Project.
#  SPDX-License-Identifier: Apache-2.0

###############################################################################
# Copyright 2020,2022 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#                            #
###############################################################################
# COMPONENT: refresh_bootmap                                                  #
#                                                                             #
# Refresh bootmap info of the specific device, the device can be FCP, eckd,   #
# etc. Currently, the script support FCP device only.                         #
###############################################################################

source /opt/zthin/lib/zthinshellutils

###############################################################################
### FUNCTIONS #################################################################
###############################################################################

function printCMDDescription {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Prints a short description of this command.
  # @Overrides:
  #   printCMDDescription{} in "zthinshellutils".
  # @Code:
  echo -n "Refresh bootmap info of the specific device. "
  echo    "Currently, the script only support FCP device."
} #printCMDDescription{}

function printCMDUsage {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Prints a short description of this command.
  # @Overrides:
  #   printCMDUsage{} in "zthinshellutils".
  # @Code:
  echo -n "Usage: $CMDNAME OPTIONS"
  [[ $namedArgListing ]] && echo -n " ${namedArgListing}" |
                            sed 's/\[--/ \\\n       '"${CMDNAME//?/ }"' [--/g'
  if [[ $positionalArgListing ]]; then
    echo " ${positionalArgListing}" |
      sed 's/ / \\\n       '"${CMDNAME//?/ }"' /g'
  else
    echo ''
  fi
  echo "${optionHelp}"
} #printCMDUsage

function cleanup_umountdir_and_disconnect_fcp {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  
  if [[ -n ${deviceMountPoint} ]]; then
    if [[ -n ${zipl_conf} && -n ${BLS_dir} && -n ${zipl_postfix} ]]; then
      # remove the temporary copy of zipl.conf
      if [[ -f ${deviceMountPoint}/${zipl_conf}${zipl_postfix} ]]; then
        rm -fr ${deviceMountPoint}/${zipl_conf}${zipl_postfix}
        # "rm -fr xxx" return code is always 0
        if [[ ! -e ${deviceMountPoint}/${zipl_conf}${zipl_postfix} ]]; then
          inform "Removed ${deviceMountPoint}/${zipl_conf}${zipl_postfix}"
        fi
      fi
      # remove the temporary copy of BLS_dir
      if [[ -d ${deviceMountPoint}/${BLS_dir}${zipl_postfix} ]]; then
        rm -fr ${deviceMountPoint}/${BLS_dir}${zipl_postfix}
        # "rm -fr xxx" return code is always 0
        if [[ ! -e ${deviceMountPoint}/${BLS_dir}${zipl_postfix} ]]; then
          inform "Removed ${deviceMountPoint}/${BLS_dir}${zipl_postfix}"
        fi
      fi
    fi

    # Umount dir and cleanup all FCP devices.
    inform "Begin to cleanup umount dir."
    out=`umount ${deviceMountPoint} 2>&1`
    rc=$?
    if [[ $rc -ne 0 ]]; then
      # if the umount failed, we need to continue the cleanup process to clean multipath/lun/FCP from compute node
      printError "Failed to Umount devNode $devNode from $deviceMountPoint. rc: $rc. out: $out."
    else
      inform "Successed to umount devNode $devNode from $deviceMountPoint"
    fi
    # remove the temporary mount point dir
    if [[ -d ${deviceMountPoint} ]]; then
      # If umount step failed, but the device is still mounted, then the `rm -rf` command will fail like:
      # rm: cannot remove 'mount_point': Device or resource busy
      rm -rf ${deviceMountPoint}
      if [[ ! -e ${deviceMountPoint} ]]; then
        inform "Removed ${deviceMountPoint}"
      fi
    fi
    inform "Finish cleanuping umount dir."
  else
    printError "Failed to umount devNode $devNode due to deviceMountPoint not defined."
  fi

  cleanup_fcpdevices
} # cleanup_umountdir_and_disconnect_fcp

function find_and_deactivate_lvm_vgs {
    # @Description:

    local map_name=$1
    inform "Start to find volume groups on this device: $map_name."
    vgs_output=($(vgs -o devices,vg_name --noheadings --separator ,))
    rc=$?
    if [[ $rc -ne 0 ]]; then
        printError "Failed to do vg scan on $map_name. exit code: $rd. Skip deactivating vg."
        return
    fi
    # get all the volume groups on the map_name
    vgs_str=""
    for entry in "${vgs_output[@]}"
    do
        if [[ $entry =~ /dev/mapper/${map_name}[0-9]* ]]; then
            vg=$(echo $entry | awk -F',' '{print $2}')
            if [[ $(echo $vgs_str | grep -w "$vg") ]]; then
                inform "find again the volume group $vg."
            else
                inform "found new volume group: $vg on $map_name."
	            if [[ "$vgs" == "" ]]; then
	                vgs_str=$vg
	            else
	                vgs_str+=" $vg"
	            fi
            fi
        fi
    done
    if [[ "$vgs_str" == "" ]]; then
        inform "No volume group found on $map_name."
        return
    else
        inform "The volume groups detected on $map_name include: $vgs_str."
    fi
    vgs_list=($vgs_str)
    for vg in "${vgs_list[@]}"
    do
        inform "Deactivating volume group: $vg."
        output=$(vgchange -an $vg)
        rc=$?
        if [[ $rc -ne 0 ]]; then
            printError "Failed to deactivate volume group: $vg. exit_code: $rc. output: $output."
        else
            inform "volume group: $vg is successfully deactivated."
        fi
    done
    inform "Finish deactivating volume groups on $map_name."
}

function cleanup_fcpdevices {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  inform "Begin to refresh multipath bindings."
  # Don't need to judge if the multipath_enabled exist,
  # if it doesn't exist, the $multipath_enabled will be None.
  if [[ ( $multipath_enabled == 1 ) && ( "$wwid" != "" ) ]]; then
    # inform "Cleaning up multipath bindings and wwids."
    lsblk_output=$(lsblk 2>&1)
    inform "lsblk output: ${lsblk_output}."
    multipath_output=$(multipath -ll 2>&1)
    inform "multipath output: ${multipath_output}."
    # delete the wwid and refresh multipath map
    map_name=$(multipath -l $wwid -v 1)
    rc=$?
    if [[ ( $rc -eq 0 ) && ( "$map_name" != "" ) ]]; then
        inform "multipath wwid: $wwid, map name: $map_name."
        find_and_deactivate_lvm_vgs $map_name
        #wwid_out=`multipath -w $map_name 2>&1`
        #rc1=$?
        # add retry according to multipath -h:
        # -R num: number of times to retry removes of in-use devices 
        flush_out=`multipath -R 3 -f $map_name 2>&1`
        rc2=$?
        if [[ ( $rc2 -ne 0 ) ]]; then
            inform "Failed to flush $map_name. rc: $rc2, output: $flush_out."
        else
            inform "multipath $map_name flushed successfully."
        fi
        # cleanup multipath leftover if any
        if [[ ( $rc2 -ne 0 ) ]]; then
            inform 'start: clean multipath leftover'
            cmd_out=$(dmsetup info)
            inform "dmsetup info output: ${cmd_out}."
            if [[ `which lsof &> /dev/null; echo $?` -eq 0 ]]; then
                lsof_mpath=$(lsof /dev/mapper/$map_name 2>&1)
                inform "lsof of device ${map_name}: ${lsof_mpath}."
            fi
            # check whether any path exists for current multipath device
            # path_num examples:
            #   0: if no path
            #   1: if 1 path exists
            #   2: if 2 paths exist
            path_num=$(multipathd show map $map_name json | grep -c '"dev"')
            # delete paths if any
            if [[ ( $path_num -ne 0 ) ]]; then
                # leftover_paths examples:
                #   sda    : if 1 path exists
                #   sda sdb: if 2 paths exist
                leftover_paths=$(multipathd show map $map_name json | awk '/\"dev\"/ {split($3,a,"\""); print a[2]}')
                inform "leftover_paths output: ${leftover_paths}."
                for p in ${leftover_paths[@]}
                do
                    echo offline > /sys/block/$p/device/state
                    echo 1 > /sys/block/$p/device/delete
                    inform "path $p deleted"
                done
            fi
            # delete multipath device itself
            # first, try multipath -f. 
            cmd_out=$(multipath -f $map_name 2>&1)
            # if multipath device already gone after deleting paths above,
            # then rc is 0
            rc=$?
            inform "multipath -f output: ${cmd_out}."
            if [[ ( $rc -ne 0 ) ]]; then
                # second, try dmsetup suspend/remove
                cmd_out=$(dmsetup suspend $map_name 2>&1)
                inform "dmsetup suspend output: ${cmd_out}."
                cmd_out=$(dmsetup remove $map_name -f 2>&1)
                rc=$?
                inform "dmsetup remove output: ${cmd_out}."
                if [[ ( $rc -ne 0 ) ]]; then
                    # third, try multipath -f again
                    cmd_out=$(multipath -f $map_name 2>&1)
                    inform "multipath -f output: ${cmd_out}."
                fi
            fi
            inform 'end: clean multipath leftover'
        fi
    else
        inform "Failed to get the multipath map name or map name is null for wwid: $wwid. rc is $rc."
    fi
    #if [[ -f /etc/multipath/bindings ]]; then
        #sed -i "/$wwid/d" /etc/multipath/bindings
    #fi
    lszfcp_output=$(lszfcp -HD 2>&1)
    inform "lszfcp output: ${lszfcp_output}"
    multipath_output=$(multipath -ll 2>&1)
    inform "multipath output: ${multipath_output}."
    cmd_out=$(dmsetup info)
    inform "dmsetup info output: ${cmd_out}." 
    if [[ ( `which lsof &> /dev/null; echo $?` -eq 0 ) && ( "$map_name" != "" ) && ( -e /dev/mapper/${map_name} ) ]]; then
      # log whether there's any process still occupying the mpath device
      lsof_mpath=$(lsof /dev/mapper/$map_name 2>&1)
      inform "lsof of device ${map_name}: ${lsof_mpath}."
    fi
  fi
  inform "Finish refreshing multipath bindings."

  # Disconnect all FCPs
  inform "Begin to remove LUN."
  # loop to do unit_remove to each (fcp, wwpn) in valid_paths_dict and check lun existence
  # as the unit_add is done with the combinations from valid_paths_dict, so does unit_remove
  for fcp in ${!valid_paths_dict[*]}
  do
      wwpns=(${valid_paths_dict[$fcp]})
      inform "Removing LUN: $lun from FCP: $fcp and WWPNs: ${wwpns[*]}."
      for wwpn in "${wwpns[@]}"
      do
          # unit_remove for the path
          removeLun $fcp 0x${wwpn} ${lun}
          rc=$?
          if [[ -d /sys/bus/ccw/devices/0.0.${fcp}/0x${wwpn}/${lun} ]]; then
              printError "Failed to remove disk on fcp: $fcp and wwpn: $wwpn, rc is $rc."
          fi
      done
  done
  inform "Finish removing LUN."

  inform "Begin to offline and undedicate FCP devices"
  # For offline and undedicate FCP, it should be done for all the inputed FCP devices
  for fcp in "${INPUT_FCPS[@]}"
  do
      # dedicate and online FCP device
      inform "Setting FCP device $fcp offline."
      offlineUndedicateFcp $fcp 
  done
  inform "Finish disconnecting FCP devices."
  
  # In some case, though the `multipath -f` reported the multipath device is flushed successfully, multipathd will continue to add new path
  # into the mpath device making the mpath device showing up again and the LVs on it then automatically be activated.
  # So, we add the check of mpath existence again here and do the deactivate of LVs and flush mpath device, to avoid leftover issue.
  inform "Checking mpath device's existence again."
  devmapper_content=$(ls /dev/mapper/)
  inform "/dev/mapper/ folder content: $devmapper_content."
  multipath_output=$(multipath -ll 2>&1)
  inform "multipath output: ${multipath_output}."
  if [[ "$map_name" != "" ]]; then
      inform "Checking $map_name existence again."
      # Step 1: check whether the volume contains lvm. If yes, then remove the LVs.
      if [[ "$vgs_str" != "" ]]; then
          # re-use the previously found vg name because under the leftover scenario, the vg name cann't be detected any more
          vg_name=$(echo $vgs_str | sed 's/[ \t]*//g')
          if [[ $vg_name != "" ]]; then
              inform "checking volume group: $vg_name."
              # find all LVs on this volume group
              devices=$(dmsetup info | grep "Name:" | awk -F: '{print $2}')
              for dev in $devices
              do
                  stripped_dev=$(echo $dev | sed 's/[ \t]*//g')
                  if [[ "${stripped_dev}" =~ ^${vg_name}-.* ]]; then
                      inform "find LV: ${stripped_dev}."
                      suspend_out=$(dmsetup suspend ${stripped_dev} 2>&1)
                      inform "dmsetup suspend output: ${suspend_out}."
                      cmd_out=$(dmsetup remove ${stripped_dev} 2>&1)
                      rc=$?
                      if [[ $rc -eq 0 ]]; then
                          inform "successfully removed LV: ${stripped_dev}."
                      else
                          inform "dmsetup remove failed. RC: $rc, output: ${cmd_out}."
                      fi
                  fi
              done
          fi
      fi
      # Step 2: try flush multipath again.
      inform "Try to flush $map_name again."
      cmd_out=$(multipath -f $map_name 2>&1)
      rc=$?
      if [[ $rc -eq 0 ]]; then
          inform "Successfully flushed $map_name."
      else
          inform "Failed to flush $map_name. RC: $rc, output: ${cmd_out}."
      fi
      # Step 3: try flush multipath with "dmsetup suspend & remove".
      inform "Try to flush $map_name with dmsetup cmds."
      # For most cases, the device doesn't exist when enter here so the
      # "dmsetup info" command would return "No such device or address".
      dm_info=$(dmsetup info $map_name 2>&1)
      rc=$?
      if [[ $rc -eq 0 ]]; then
          inform "Before dmsetup suspend, the dmsetup info: ${dm_info}."
          suspend_out=$(dmsetup suspend $map_name 2>&1)
          suspend_rc=$?
          inform "dmsetup suspend rc: ${suspend_rc}, output: ${suspend_out}."
          dm_info=$(dmsetup info $map_name 2>&1)
          inform "After dmsetup suspend, the dmsetup info: ${dm_info}."
          remove_out=$(dmsetup remove -f $map_name 2>&1)
          remove_rc=$?
          inform "dmsetup remove rc: ${remove_rc}, output: ${remove_out}."
          dm_info=$(dmsetup info $map_name 2>&1)
          info_rc=$?
          if [[ $info_rc -eq 0 ]]; then
              inform "Warning: $map_name still exist after dmsetup remove, info: ${dm_info}."
          else
              inform "$map_name does not exist in dmsetup info. RC: $info_rc, output: ${dm_info}."
          fi 
      else
          # most cases would go here
          inform "$map_name does not exist in dmsetup info. RC: $rc, output: ${dm_info}."
      fi
  fi

  inform "Log information before EXIT."
  lszfcp_output=$(lszfcp -HD 2>&1)
  inform "lszfcp output: ${lszfcp_output}"
  multipath_output=$(multipath -ll 2>&1)
  inform "multipath output: ${multipath_output}."
  dmsetup_info=$(dmsetup info 2>&1)
  inform "dmsetup info: ${dmsetup_info}."
  mapper_folder=$(ls -l /dev/mapper/ 2>&1)
  inform "/dev/mapper/ folder: ${mapper_folder}."
  dm_device=$(ls -l /dev/dm-* 2>&1)
  inform "/dev/dm-* devices: ${dm_device}."
  if [[ ( `which lsof &> /dev/null; echo $?` -eq 0 ) && ( "$map_name" != "" ) && ( -e /dev/mapper/${map_name} ) ]]; then
      # log whether there's any process still occupying the mpath device
      lsof_mpath=$(lsof /dev/mapper/$map_name 2>&1)
      inform "lsof of device ${map_name}: ${lsof_mpath}."
  fi
} #cleanup_fcpdevices

function printCMDExamples {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Prints a short description of this command.
  # @Overrides:
  #   printCMDDescription{} in "zthinshellutils".
  # @Code:
  echo "Example:
  ./refresh_bootmap.sh --fcpchannel="5d71" --wwpn="5005076802100C1B" --lun="0000000000000000" --wwid="600507640083826de00000000000605b" --minfcp="1"
  ./refresh_bootmap.sh --fcpchannel="5d71,5d72" --wwpn="5005076802100C1B,5005076802200C1B,5005076802300C1B,5005076802400C1B,5005076802400C1A,5005076802300C1A,5005076802200C1A,5005076802100C1A" --lun="0000000000000000" --wwid="600507640083826de00000000000605b" --minfcp="2""
} #printCMDDescription

function parseArgs {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Parses and checks command-line arguments.
  # @Code:
  # Non-local variables in this function are intentionally non-local.
  isOption -h --help "         Print this help message."   && printHelp='true'
  isOption -v --verbose "      Print verbose output."      && verbose='-v'
  isOption -x --debug "        Print debugging output."    && debug='-x'
  isOption -t --type "         Device type. The default device typs is fcp and fcp is the only currently supported type."
  isOption -f --fcpchannel "   FCP channel IDs. Support multi fcpchannel, split by ','."
  isOption -w --wwpn "         World Wide Port Name IDs. Support multi wwpn, split by ',' Example: 5005076802400c1b"
  isOption -l --lun "          Logical Unit Number ID. Example: 0000000000000000"
  isOption -d --wwid "         WWID of the volume, Example: 600507640083826de00000000000605b"
  isOption -m --minfcp "      The minimum acceptable FCP count should be defined to the target vm."
  isOption -i --ignitionurl "  (RHCOS only) The URL or local path that points to RHCOS ignition file"
  isOption -n --nicid "        (RHCOS only) The nic ID. Example: 0.0.1000,0.0.1001,0.0.1002"
  isOption -p --ipconfig "     (RHCOS only) The network configuration info. In format of <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none[:[<dns1>][:<dns2>]];<mtu>"

  if [[ $printHelp ]]; then
    printHelp
    exit 0
  fi

  for i in "$@"
  do
  case $i in
      -t|--type=*)
      device_type="${i#*=}"
      shift # past argument=value
      ;;
      -f|--fcpchannel=*)
      fcpchannel="${i#*=}"
      shift # past argument=value
      ;;
      -w|--wwpn=*)
      wwpn="${i#*=}"
      shift # past argument=value
      ;;
      -l|--lun=*)
      lun="${i#*=}"
      shift # past argument=value
      ;;
      -d|--wwid=*)
      wwid="${i#*=}"
      shift # past argument=value
      ;;
      -m|--minfcp=*)
      minfcp="${i#*=}"
      shift # past argument=value
      ;;
      -i|--ignitionurl=*)
      ignitionurl="${i#*=}"
      shift # past argument=value
      ;;
      -n|--nicid=*)
      nicid="${i#*=}"
      shift # past argument=value
      ;;
      -p|--ipconfig=*)
      ipconfig="${i#*=}"
      shift # past argument=value
      ;;
      --default)
      DEFAULT=YES
      shift # past argument with no value
      ;;
      *)
      printError "Exit MSG: Unknow option: $i." 
      exit 3
      ;;
  esac
  done

  # Set default device type to FCP if not specific.
  device_type=${device_type:-fcp}
  device_type=$(echo ${device_type} | tr '[:upper:]' '[:lower:]')
} #parseArgs

function checkSanity {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Performs basic checks to ensure that a successful deploy can reasonably
  #   be expected.
  if [[ $device_type = 'fcp' ]]; then
    if [[ ! $lun || ! $fcpchannel ]];then
        printError "Exit MSG: Please specify lun and fcpchannel."
        exit 4
    fi
    # Intentionally non-local variable.
    format='FCP'
  elif [[ $device_type = 'fba' ]]; then
    # We don't support fba now.
    :
  else
    printError "Exit MSG: Unknown device type."
    exit 5
  fi
  if [[ ignitionurl || nicid || ipconfig ]]; then
    # ignitionurl, nicid and ipconfig are required for RHCOS BFV
    if [[ ! ignitionurl || ! nicid || ! ipconfig ]]; then
        printError "Exit MSG: Please specify ignitionurl, nicid and ipconfig."
        exit 6
    fi
  fi
} #checkSanity

function printLogs {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   print logs essential to debug issues before exiting with errors
  # lszfcp -HD
  lszfcp_output=$(lszfcp -HD 2>&1)
  inform "lszfcp output: ${lszfcp_output}."
  # lsluns
  lsluns_output=$(lsluns 2>&1)
  inform "lsluns output: ${lsluns_output}."
  # ls /dev/disk/by-path
  disk_by_path_folder=$(ls /dev/disk/by-path/)
  inform "/dev/disk/by-path/ folder content: ${disk_by_path_folder}."
  # ls /dev/disk/by-id
  disk_by_id_folder=$(ls /dev/disk/by-id/)
  inform "/dev/disk/by-id/ folder content: ${disk_by_id_folder}."
  # multipath -ll
  multipath_output=$(multipath -ll 2>&1)
  inform "multipath output: ${multipath_output}."
  # lsblk
  lsblk_output=$(lsblk 2>&1)
  inform "lsblk output: ${lsblk_output}."
} #printLogs

function checkMultipath {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Check whether multipath is enabled by checking the multipathd service status
  #   Set the multipath_enabled variable according to the result.
  multipath_enabled=0
  service_output=`systemctl status multipathd | grep "active (running)"`
  if [[ $? -eq 0 ]]; then
      inform "multipathd service is active and running."
      multipath_enabled=1
  fi
} #checkMultipath

function truncaterdzfcp {
	: SOURCE: ${BASH_SOURCE}
	: STACK:  ${FUNCNAME[@]}
	# @Description:
	#   select 8 paths from all the valid paths to get the rd.zfcp kernel command line
	#   select combinations based on the rule: 
	#   each FCP should be covered by at least one path in rd.zfcp so that all the FCPs
	#   will be automatically online when vm is started
    rd_zfcp=""
    wwpn_idx=0
    selected_paths=0
    valid_fcps=$(echo ${!valid_paths_dict[*]})
	while [[ $selected_paths -lt 8 ]]
	do
		inform "checking with wwpn_index: $wwpn_idx."
		# for each fcp, get the wwpn corresponding to wwpn_idx
		for fcp in ${valid_fcps[@]}
		do
			wwpns=(${valid_paths_dict[$fcp]})
			if [[ $wwpn_idx -lt ${#wwpns[@]} ]]; then
				wwpn=${wwpns[$wwpn_idx]}
				rd_zfcp+="rd.zfcp=0.0.$fcp,0x$wwpn,$lun "
				((selected_paths=$selected_paths+1))
			fi
			# break when 8 paths found
			if [[ $selected_paths -eq 8 ]]; then
				break
			fi
		done
		((wwpn_idx=$wwpn_idx+1))
	done
}

function probePartition {
    : SOURCE: ${BASH_SOURCE}
    : STACK:  ${FUNCNAME[@]}
    # @Description:
    #   Run partprobe to try to find the partition
    # @Parameters:
    #   None
    # @Returns:
    #   None
    # ls /dev/disk/by-path

    disk_by_path_folder=$(ls /dev/disk/by-path/)
    inform "Before partprobe, /dev/disk/by-path/ folder content: ${disk_by_path_folder}."
    # ls /dev/disk/by-id
    disk_by_id_folder=$(ls /dev/disk/by-id/)
    inform "Before partprobe, /dev/disk/by-id/ folder content: ${disk_by_id_folder}."
    # ls /dev/mapper/
    disk_mapper_folder=$(ls /dev/mapper/)
    inform "Before partprobe, /dev/mapper/ folder content: ${disk_mapper_folder}."

    # get the devPath under /dev/mapper
    if [[ "$wwid" != "" ]]; then
        map_name=$(multipath -l $wwid -v 1)
        rc=$?
        # probe partition
        if [[ ( $rc -eq 0 ) && ( "$map_name" != "" ) ]]; then
            devPath="/dev/mapper/$map_name"
            probe_partition=$(/usr/sbin/partprobe $devPath 2> /dev/null)
            rc=$?
            inform "Manually run command partprobe on $devPath of wwid $wwid with return code: $rc"
        else
            inform "Failed to get map_name before partprobe, call partprobe without parameters."
            probe_partition=$(/usr/sbin/partprobe 2> /dev/null)
            rc=$?
            inform "Manually run command partprobe without parameters return code: $rc"
        fi
    fi

    # ls /dev/disk/by-path
    disk_by_path_folder=$(ls /dev/disk/by-path/)
    inform "After partprobe, /dev/disk/by-path/ folder content: ${disk_by_path_folder}."
    # ls /dev/disk/by-id
    disk_by_id_folder=$(ls /dev/disk/by-id/)
    inform "After partprobe, /dev/disk/by-id/ folder content: ${disk_by_id_folder}."
    # ls /dev/mapper/
    disk_mapper_folder=$(ls /dev/mapper/)
    inform "After partprobe, /dev/mapper/ folder content: ${disk_mapper_folder}."
}

function mount_boot_partition_rhcos {
    # The location of the mountpoint
    mountpoint=$1

    if [ ! -d $mountpoint ]; then
        printError "Exit MSG: mountpoint $mountpoint must be a directory."
        exit 17
    fi

    # Get the boot partition
    set -o pipefail
    let retry=0
    while true
    do
        eval $(lsblk "${DEST_DEV}" --output LABEL,NAME --pairs | grep 'LABEL="boot' | tr ' ' '\n' | tail -n 1)
        if [[ ${NAME} =~ "mpath" ]]; then
            mount "/dev/mapper/${NAME}" "$mountpoint"
        else
            mount "/dev/${NAME}" "$mountpoint"
        fi
        RETCODE=$?
        if [[ $RETCODE -ne 0 ]]; then
            if [[ $retry -lt 30 ]]; then
                # retry and sleep to allow udevd to populate the disk
                sleep 1
                let retry=$retry+1
                continue
            fi
            printError "Exit MSG: failed mounting boot partition"
            exit 17
        fi
        break;
    done
    set +o pipefail
} #mount_boot_partition_rhcos{}

function update_zipl_bootloader_rhcos {
    inform "Updating zipl"

    if [[ $valid_paths_count -gt 8 ]]; then
      inform "valid paths count $valid_paths_count is greater than 8, do truncation to get the rd.zfcp parameter."
      truncaterdzfcp
    fi
    inform "Final rd.zfcp value is: $rd_zfcp."

    # Create a mount dir.
    rhcosTempDir=$(/usr/bin/mktemp -d /tmp/XXXXX)
    rc=$?
    if [[ $rc -ne 0 ]]; then
      printError "Exit MSG: Create mount dir fails, rc: $rc."
      exit 7
    fi
    deviceMountPoint=$rhcosTempDir/boot_partition
    mkdir -p $deviceMountPoint

    # Register EXIT signal for umount dir and disconnect fcp devices
    trap 'cleanup_umountdir_and_disconnect_fcp' EXIT

    mount_boot_partition_rhcos $deviceMountPoint

    #Get ignition config file
    mkdir -p "$deviceMountPoint/ignition"
    if [ -f "$ignitionurl" ]; then
        inform "$ignitionurl is FILE"
        cat $ignitionurl > $deviceMountPoint/ignition/config.ign 2>/dev/null
    else
        inform "$ignitionurl is URL"
        let retry=0
        while true
        do
            curl -Lf $ignitionurl >$deviceMountPoint/ignition/config.ign 2>/dev/null
            RETCODE=$?
            if [ $RETCODE -ne 0 ]
            then
                if [[ $RETCODE -eq 22 && $retry -lt 5 ]]
                then
                    # Network isn't up yet, sleep for a sec and retry
                    sleep 1
                    let retry=$retry+1
                    continue
                else
                    printError "Exit MSG: failed fetching ignition config from ${ignitionurl}: ${RETCODE}"
                    exit 17
                fi
                inform "fetching ignition config from ${ignitionurl}: ${RETCODE}"
            else
                inform "Successfully get the ignition config"
                break;
            fi
        done
    fi

    blsfile=$(ls $deviceMountPoint/loader/entries/*.conf)
    #Get zipl config file for parameters

    # ipconfig format: <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none[:[<dns1>][:<dns2>]];<mtu>
    # ipsetting format here: <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none[:[<dns1>][:<dns2>]]
    ipsetting=$(echo ${ipconfig%;*})
    # mtu format: <mtu> , example, "1500"
    mtu=$(echo ${ipconfig#*;})
    # nameserver format: nameserver=<nameserver>
    nameserver1=$(echo $ipsetting| awk -F: '{print $8}')
    nameserver2=$(echo $ipsetting| awk -F: '{print $9}')
    # ipsetting format here: <client-IP>:[<peer>]:<gateway-IP>:<netmask>:<client_hostname>:<interface>:none:<mtu>
    ipsetting=$(echo ${ipsetting/%none:*/none:}${mtu})
    if [ $nameserver1  ]; then 
        nameserver1='nameserver='${nameserver1}
    fi
    if [ $nameserver2  ]; then 
        nameserver2='nameserver='${nameserver2}
    fi

    echo "$(grep options $blsfile | cut -d' ' -f2-) zfcp.allow_lun_scan=0 rw $rd_zfcp rd.znet=qeth,$nicid,layer2=1,portno=0 ignition.firstboot=1 rd.neednet=1 ip=$ipsetting $nameserver1 $nameserver2" > $rhcosTempDir/zipl_prm
    ziplstr=`cat $rhcosTempDir/zipl_prm`
    inform "zipl param string: $ziplstr"

    #Run zipl to update bootloader
    zipl --verbose -p $rhcosTempDir/zipl_prm -i $(ls $deviceMountPoint/ostree/*/*vmlinuz*) -r $(ls $deviceMountPoint/ostree/*/*initramfs*) --target $deviceMountPoint
    if [[ $? -ne 0 ]]; then
        printError "Exit MSG: failed updating bootloder"
        exit 17
    fi
    #Update BLS config file for reboot
    sed -i -e 's/ignition.firstboot=1.*//' $rhcosTempDir/zipl_prm
    sed -e '/options/d' $blsfile > $rhcosTempDir/blsfile
    echo "options $(cat $rhcosTempDir/zipl_prm) " >> $rhcosTempDir/blsfile
    cat $rhcosTempDir/blsfile > $blsfile
    blockdev --flushbufs ${DEST_DEV} > /dev/null
} #update_zipl_bootloader_rhcos{}

function update_rdzfcp_to_fcp_only {
	: SOURCE: ${BASH_SOURCE}
	: STACK:  ${FUNCNAME[@]}
	# @Description:
  # remove wwpn and lun from rd.zfcp
  # before function: rd_zfcp="rd.zfcp=0.0.$fcp,0x$wwpn,$lun "
  # after function: rd_zfcp="rd.zfcp=0.0.$fcp "
    rd_zfcp=""
    for fcp in "${INPUT_FCPS[@]}"
    do
      rd_zfcp+="rd.zfcp=0.0.$fcp "
    done
} #updaterdzfcp_with_fcp_only

function refreshZIPL {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Retrieves initial information for the IPL from disk
  # @Parameters:
  #   devNode: the path to access the disk
  # @Returns:
  #   None
  local devNode=$1
  zipl_conf='etc/zipl.conf'
  zipl_postfix='_zipl_postfix'
  BLS_dir='boot/loader/entries'
  inform "Begin to refreshZIPL."

  if [[ $ignitionurl ]]; then
      # refresh bootmap for RHCOS
      DEST_DEV="/dev/disk/by-path/ccw-0.0.${first_fcp}-zfcp-0x${first_wwpn}:${lun}"
      inform "refreshing bootmap for RHCOS"
      update_zipl_bootloader_rhcos
      inform "RESULT PATHS: $valid_paths_str"
      exit 0
  fi

  if [[ ! -e "$devNode" ]]; then
    # first, run partprobe to refresh partition table if multipath is enabled
    if [[ $multipath_enabled == 1 ]]; then
        inform "multipath is enabled, so probe partition on dev of wwid: $wwid"
        probePartition
    fi
    # retry, while waiting various durations
    inform "devNode file ${devNode} not exist yet. Retrying..."
    sleepTimes=".001 .01 .1 .5 1 2 3 5 8 15"

    for seconds in $sleepTimes; do
      sleep $seconds
      inform "Retrying with interval as ${seconds} seconds"
      if [[ -e "$devNode" ]]; then
        # successful - leave loop
        break
      fi
    done
    # print error log if retry fails
    if [[ ! -e "$devNode" ]]; then
      printError "Exit MSG: devNode ${devNode} doesn't exist."
      printLogs
      exit 6
    fi
  fi

  # Create a mount dir.
  deviceMountPoint=$(/usr/bin/mktemp -d /tmp/XXXXX)
  rc=$?
  if [[ $rc -ne 0 ]]; then
    printError "Exit MSG: Create mount dir fails, rc: $rc."
    exit 7
  fi

  # Register EXIT sigual for umount dir and disconnect fcp devices
  trap 'cleanup_umountdir_and_disconnect_fcp' EXIT

  # Try to mount fcp device.
  inform "devNode is: $devNode point: ${deviceMountPoint}"
  mount $devNode ${deviceMountPoint}
  rc=$?
  if [[ $rc -ne 0 ]]; then
    printError "Exit MSG: Mount devNode fails. rc: $rc."
    exit 8
  fi

  # Remove dirty data in /etc/fstab left by the previous vm deploy
  # remove lines containing "/mnt/ephemeral" or " swap swap "
  # TODO: need to update the implementation method when later the ephemeral
  # mount point maybe changed to user configurable, to not hardcode to /mnt/ephemeralxxx
  inform "Updating ${deviceMountPoint}/etc/fstab"
  sed -i '/\/mnt\/ephemeral/d' ${deviceMountPoint}/etc/fstab
  sed -i '/ swap swap /d' ${deviceMountPoint}/etc/fstab

  # Remove dirty data in /etc/zfcp.conf left in image
  if [[ -r "${deviceMountPoint}/etc/zfcp.conf" ]]; then
      inform "Purge contents in ${deviceMountPoint}/etc/zfcp.conf"
      echo "" > ${deviceMountPoint}/etc/zfcp.conf
  fi
  # Remove stale /etc/multipath/bindings and /etc/multipath/wwids
  # These files may contain the old info of the captured base vm,
  # which disrupts the volume bindings of the new vm deployed from above captured image, resulting
  #  a. mpath name inconsistent between 'df' and 'multipath'
  #  b. in some scenairo, boot-volume extend partition fail due to 'a'
  # By removing them, they can be recreated later with correct volume bindings
  # when multipathd start at vm boot time.
  inform "Move stale multipath bindings and wwids if any"
  if [[ -r "${deviceMountPoint}/etc/multipath/bindings" ]]; then
    mv ${deviceMountPoint}/etc/multipath/bindings ${deviceMountPoint}/etc/multipath/bindings.bak
  fi
  if [[ -r "${deviceMountPoint}/etc/multipath/wwids" ]]; then
    mv ${deviceMountPoint}/etc/multipath/wwids ${deviceMountPoint}/etc/multipath/wwids.bak
  fi

  # Get target os version
  local osRelease="${deviceMountPoint}/etc/os-release"
  local slesRelease="${deviceMountPoint}/etc/SuSE-release"
  local rhelRelease="${deviceMountPoint}/etc/redhat-release"

  if [[ -e $osRelease ]]; then
    os=`cat $osRelease | grep "^ID=" | sed \
      -e 's/ID=//' \
      -e 's/"//g'`
    version=`cat $osRelease | grep "^VERSION_ID=" | sed \
      -e 's/VERSION_ID=//' \
      -e 's/"//g' \
      -e 's/\.//'`
    os=$os$version

  #The /etc/redhat-release file will be deprecated in rhel7 and later release
  elif [[ -e $rhelRelease ]]; then
    os='rhel'
    version=`cat $rhelRelease | grep -i "Red Hat Enterprise Linux Server" | sed \
      -e 's/[A-Za-z\/\.\(\)]//g' \
      -e 's/^ *//g' \
      -e 's/ *$//g' \
      -e 's/\s.*$//'`
    os=$os$version
  fi
  # Prepare the new rd.zfcp parameter value 
  # Due to the kernel command line length limitation of RHEL, we will limit the rd.zfcp
  # items count to 8. so if the valid paths count is greater than 8, we need to do truncation.
  # otherwise we can use the rd_zfcp string we get while finding the valid paths.
  if [[ ($valid_paths_count -gt 8) && ($os == rhel7*) ]]; then
  	inform "valid paths count $valid_paths_count is greater than 8, do truncation to get the rd.zfcp parameter."
  	truncaterdzfcp
  fi
  if [[ ($os == rhel8*) || ($os == rhel9*) ]]; then
    update_rdzfcp_to_fcp_only
  fi
  inform "Final rd.zfcp value is: $rd_zfcp."

  # Exec zipl command to prepare device for initial program load
  if [[ ($os == rhel7*) || ($os == rhel8*) || ($os == rhel9*) ]]; then
    inform "Refresh $os zipl."
    if [[ ! -e ${deviceMountPoint}/${zipl_conf} ]]; then
      printError "Exit MSG: Failed to execute zipl on $os due to ${deviceMountPoint}/${zipl_conf} not exist."
      exit 13
    else
      # Get root path with wwid, instead of using the UUID
      if [[ -z "$wwid" ]]; then
          wwid=`/usr/lib/udev/scsi_id --whitelisted $diskPath 2> /dev/null`
          inform "The volume wwid is $wwid."
      fi
      rootPath="root=\/dev\/disk\/by-id\/dm-uuid-part1-mpath-$wwid "
      # Delete all items start with "rd.zfcp="
      sed -ri 's/rd.zfcp=\S+\s*[[:space:]]//g' ${deviceMountPoint}/${zipl_conf}
      # Remove quote
      sed -i 's/\"$//g' ${deviceMountPoint}/${zipl_conf}
      # Remove trailing space
      sed -i 's/[ \t]*$//g' ${deviceMountPoint}/${zipl_conf}
      # Update the root file system target path
      sed -ri "s/root=\S+\s*[[:space:]]/$rootPath /g" ${deviceMountPoint}/${zipl_conf}
      # Append rd.zfcp= string to "parameters=" line.
      sed -i "/^[[:space:]]parameters=/ s/$/ $rd_zfcp/" ${deviceMountPoint}/${zipl_conf}
      # Append quote to "parameters=" line
      sed -i "/^[[:space:]]parameters=/ s/$/\"/" ${deviceMountPoint}/${zipl_conf}

      # Create a copy of zipl_conf to be used by zipl command later
      out=`/usr/bin/cp -a ${deviceMountPoint}/${zipl_conf} ${deviceMountPoint}/${zipl_conf}${zipl_postfix} 2>&1`
      if [[ $? -ne 0 ]]; then
        printError "Exit MSG: Failed to create a copy of zipl_conf due to ${out}."
        exit 14
      fi
      # Add ${deviceMountPoint} as prefix of "target=|image=|randisk=" to the copy of zipl_conf
      sed -i "s|target[[:space:]]*=[[:space:]]*|target=${deviceMountPoint}|" ${deviceMountPoint}/${zipl_conf}${zipl_postfix}
      sed -i "s|image[[:space:]]*=[[:space:]]*|image=${deviceMountPoint}|" ${deviceMountPoint}/${zipl_conf}${zipl_postfix}
      sed -i "s|ramdisk[[:space:]]*=[[:space:]]*|ramdisk=${deviceMountPoint}|" ${deviceMountPoint}/${zipl_conf}${zipl_postfix}

      # Refresh bootmap without chroot
      if [[ $os == rhel7* ]]; then
        # zipl for RHEL7 NOT support BLS(BootLoaderSpec) file
        cmd_zipl="${deviceMountPoint}/usr/sbin/zipl -V -c ${deviceMountPoint}/${zipl_conf}${zipl_postfix} 2>&1"
        out_zipl=`eval ${cmd_zipl}`
        rc_zipl=$?
      else
        # zipl for RHEL8 support BLS(BootLoaderSpec) files.
        # Files that contain BLS snippets can have any name, but must have the file extension .conf.
        # /boot/loader/entries/ is the default directory containing BLS files.
        BLS_file_count=`ls -1 ${deviceMountPoint}/${BLS_dir}/*.conf 2>/dev/null | wc -l`
        if [[ ${BLS_file_count} -eq 0 ]]; then
          inform "No BLS files found in ${deviceMountPoint}/${BLS_dir}"
        else
          # Delete all items start with "rd.zfcp="
          sed -ri 's/rd.zfcp=\S+\s*[[:space:]]//g' ${deviceMountPoint}/${BLS_dir}/*.conf
          # Remove trailing space
          sed -i 's/[ \t]*$//g' ${deviceMountPoint}/${BLS_dir}/*.conf
          # Update the root file system target path
          sed -ri "s/root=\S+\s*[[:space:]]/$rootPath /g" ${deviceMountPoint}/${BLS_dir}/*.conf
          # Append rd.zfcp= string to "options root=" line.
          sed -i "/^options root=/ s/$/ $rd_zfcp/" ${deviceMountPoint}/${BLS_dir}/*.conf

          # Create a copy of BLS_dir to be used by zipl command later
          out=`mkdir ${deviceMountPoint}/${BLS_dir}${zipl_postfix} 2>&1`
          if [[ ! -d ${deviceMountPoint}/${BLS_dir}${zipl_postfix} ]]; then
            printError "Exit MSG: Failed to create a copy of BLS_dir due to ${out}"
            exit 15
          fi
          # Copy BLS files to the copy of BLS_dir
          out=`/usr/bin/cp -a ${deviceMountPoint}/${BLS_dir}/*.conf ${deviceMountPoint}/${BLS_dir}${zipl_postfix} 2>&1`
          if [[ $? -ne 0 ]]; then
            printError "Exit MSG: Failed to copy BLS files to ${BLS_dir}${zipl_postfix} due to ${out}."
            exit 16
          fi
          # Add ${deviceMountPoint} as prefix of "linux | initrd" to BLS files of the copy of BLS_dir
          sed -i "s|linux[[:space:]]\+/|linux ${deviceMountPoint}/|" ${deviceMountPoint}/${BLS_dir}${zipl_postfix}/*.conf
          sed -i "s|initrd[[:space:]]\+/|initrd ${deviceMountPoint}/|" ${deviceMountPoint}/${BLS_dir}${zipl_postfix}/*.conf
        fi
        # Refresh bootmap without chroot
        # -b <BLS DIRECTORY>: To parse BootLoaderSpec config files.
        #                     If none is supplied, the /boot/loader/entries directory is used.
        if [[ $os == rhel9* ]]; then
          # Because our compute node did not support RHEL9 now,
          # while the zipl required dependencies in RHEL9 image are different with zipl in RHEL7 or RHEL8.
          # So when booting RHEL9 image, use zipl on compute node to avoid errors
          cmd_zipl="/usr/sbin/zipl -V -c ${deviceMountPoint}/${zipl_conf}${zipl_postfix} -b ${deviceMountPoint}/${BLS_dir}${zipl_postfix} 2>&1"
        else
          # Use zipl in image when booting RHEL7 or RHEL8 images
          cmd_zipl="${deviceMountPoint}/usr/sbin/zipl -V -c ${deviceMountPoint}/${zipl_conf}${zipl_postfix} -b ${deviceMountPoint}/${BLS_dir}${zipl_postfix} 2>&1"
        fi
        out_zipl=`eval ${cmd_zipl}`
        rc_zipl=$?
      fi
    fi
  elif [[ $os == "" ]]; then
    inform "This is not the root disk, zipl will not be executed on it"
  else
    inform "The os version is: $os, this is not a supported linux distro"
  fi

  # Log zipl command
  inform "zipl command: $cmd_zipl"
  # Log zipl.conf
  out=`cat ${deviceMountPoint}/${zipl_conf}`
  inform "zipl.conf content: $out"

  # Check zipl result
  if [[ $rc_zipl -ne 0 ]]; then
    printError "Exit MSG: Failed to execute zipl on $os due to $out_zipl."
    return 1
  else
    inform "Successed to execute zipl on $os. Output: $out_zipl"
  fi
  
  # disable lvmetad service for RHEL 7
  # RHEL 8 does not have lvmetad
  if [[ ($os == rhel7*) && (-f ${deviceMountPoint}/etc/lvm/lvm.conf) ]]; then
      sed -i 's/use_lvmetad = 1/use_lvmetad = 0/g' ${deviceMountPoint}/etc/lvm/lvm.conf
  fi

  # Sync to ensure cached date writen back to target disk
  blockdev --flushbufs $diskPath > /dev/null

  # Return valid paths
  inform "RESULT PATHS: $valid_paths_str"

  return
} #refreshZIPL

function refreshFCPBootmap {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Refresh bootmap info of FCP device.
  # @Code:

  local out
  local rc
  local errorFile

  # all the input values don't have the prefix 0x
  fcpchannels=$(echo ${fcpchannel} | tr '[:upper:]' '[:lower:]')
  lun=$(echo ${lun} | tr '[:upper:]' '[:lower:]')
  wwpn=$(echo ${wwpn} | tr '[:upper:]' '[:lower:]')
  wwid=$(echo ${wwid} | tr '[:upper:]' '[:lower:]')

  # Add 0x to lun
  lun=0x$lun
  # The disk path in Linux layer will have a '3' as prefix as shown in following example:
  # /dev/disk/by-id/dm-uuid-part1-mpath-3600507640083826de00000000000605b 
  # so, append a '3' as a prefix to the input wwid
  if [[ "$wwid" != "" ]]; then
      wwid="3"$wwid
  fi

  # Split fcpchannels and wwpn by ","
  IFS=',' read -r -a INPUT_FCPS <<< "$fcpchannels"
  IFS=',' read -r -a INPUT_WWPNS <<< "$wwpn"
  
  # print the parameters value
  inform "RefreshFCPBootmap begins. FCP: ${INPUT_FCPS[*]}, WWPNs: ${INPUT_WWPNS[*]}, LUN: $lun, WWID: $wwid, minfcp: $minfcp."

  # Check the count of FCP devices, if greater than 8, report error and exit.
  fcps_len=${#INPUT_FCPS[@]}
  if [ $fcps_len -gt 8 ]
  then
      printError "Exit MSG: The FCP channel count $fcps_len is greater than 8."
      exit 12
  fi

  enable_zfcp_mod=`lsmod | grep zfcp`
  if [ -z "$enable_zfcp_mod" ];then
      modprobe zfcp
      # disable allow_lun_scan
      echo N > /sys/module/zfcp/parameters/allow_lun_scan
  fi

  local zthinUserID=$(vmcp q userid | awk '{printf $1}')
  inform "zthinUserID is $zthinUserID"

  # Check whether multipath is enabled by checking the multipathd service
  checkMultipath

  # Register EXIT sigual for cleanup FCP devices.
  trap 'cleanup_fcpdevices' EXIT

  # Dedicate and online each FCP device. Record the number of available FCP devices
  onlinefcpscount=0
  onlinefailed=""
  onlinesuccess=""
  for fcp in "${INPUT_FCPS[@]}"
  do
    # dedicate and online FCP device
    dedicateOnlineFcp $fcp
    rc=$?
    if [[ $rc -ne 0 ]]; then
        printError "Failed to dedicate or online FCP: $fcp, rc is $rc."
        onlinefailed+="$fcp "
    else
        ((onlinefcpscount=$onlinefcpscount+1))
        onlinesuccess+="$fcp "
    fi
  done

  # Print info of each FCP
  for fcp in "${INPUT_FCPS[@]}"
  do
      # print the output of `systool -c fc_host` for debug
      host=$(lszfcp | grep -i $fcp | awk -F' ' '{print $2}')
      if [[ -n "$host" ]]; then
          port_state=$(systool -c fc_host -v $host | grep port_state | awk -F'"' '{print $2}')
          inform "Port state of fcp: $fcp is: $port_state."
          host_info=$(systool -c fc_host -v $host)
          inform "Port detail info of fcp: $fcp is: $host_info."
      else
          inform "Failed to find host of fcp: $fcp from the lszfcp output, cann't get port state."
      fi
  done

  # if the number of online fcp is less than the min FCP number, exit
  if [[ $onlinefcpscount -lt $minfcp ]]; then
      printError "Exit MSG: Failed to online FCP devices( $onlinefailed) and the number of online FCP devices( $onlinesuccess) is less than the required minimum FCP path count: $minfcp."
      exit 1
  else
      inform "The number of online FCP devices: $onlinefcpscount satisfies the required minimum FCP path count: $minfcp."
  fi

  declare -A valid_paths_dict
  valid_paths_count=0
  rd_zfcp=""
  # check whether each combination of input fcp and input wwpn is in the lszfcp_output
  # the valid_paths_dict will contain all the valid combinations of inputed fcps and wwpns in the format:
  # (["FCP1"]="W1 W2" ["FCP2"]="W3 W4")
  for fcp in "${INPUT_FCPS[@]}"
  do
      # Retry each FCP device to make sure there are matched WWPNs for it
      retry_num=10
      while [[ $retry_num -gt 0 ]]
      do
          # use the `lszfcp -P` cmd to get the valid combinations of (fcp,wwpn)
          # sample output:
          #[root@112-cmp-ydy ~]# lszfcp -P
          # 0.0.1c04/0x5005076306105388 rport-0:0-0
          # 0.0.1c04/0x5005076306035388 rport-0:0-1
          # 0.0.1c04/0x500507680b22bac7 rport-0:0-10
          # 0.0.1c04/0x500507680d760027 rport-0:0-2
          # sample lszfcp_output:
          # 0.0.1c04 0x5005076306105388
          # 0.0.1c04 0x5005076306035388
          # 0.0.1c04 0x500507680b22bac7
          # 0.0.1c04 0x500507680d760027
          lszfcp_output=`lszfcp -P | awk '{print $1}' | awk -F'/' '{print $1,$2}'`
          inform "Output of 'lszfcp -P': $lszfcp_output."
          # Get WWPNs under /sys/bus/ccw/drivers/zfcp/.
          wwpns_shown_in_sys=`ls /sys/bus/ccw/drivers/zfcp/0.0.$fcp/ | grep "0x"`
          inform "Target WWPNs shown under /sys/bus/ccw/drivers/zfcp/0.0.$fcp/: $wwpns_shown_in_sys"
          # Try to find match between system WWPNs(from lszfcp output or under /sys) and input WWPNs
          found_match=0

          # match accessable target wwpns of this fcp
          for wwpn in "${INPUT_WWPNS[@]}"
          do
              fcp_wwpn_str="0.0.${fcp} 0x${wwpn}"
              if [[ $lszfcp_output =~ $fcp_wwpn_str || $wwpns_shown_in_sys =~ $wwpn ]]; then
                  found_match=1
                  # add this combination into valid paths
                  inform "Found one valid path: ($fcp,0x$wwpn)."
                  if [[ -z ${valid_paths_dict[$fcp]} ]]; then
                      valid_paths_dict+=([$fcp]="$wwpn")
                  else
                      old_value=${valid_paths_dict[$fcp]}
                      new_value=${old_value}" "$wwpn
                      valid_paths_dict[$fcp]=$new_value
                  fi
                  # increase valid_paths_count
                  ((valid_paths_count=$valid_paths_count+1))
                  # add this (fcp, wwpn) combinations into the rd_zfcp string which will be used to
                  # set the kernel command line in zipl.conf to refresh zipl
                  rd_zfcp+="rd.zfcp=0.0.$fcp,0x$wwpn,$lun "
              fi
          done

          # If no matched wwpn found, need retry
          if [[ $found_match -eq 0 ]]; then
              sleep 1
              retry_num=$((retry_num-1))
              inform "Retrying to get the target WWPNs for FCP device $fcp, $retry_num times left..."
          else
              inform "Found target WWPNs ${valid_paths_dict[$fcp]} for FCP device $fcp."
              break
          fi
      done
  done

  # check whether there is valid combination of FCP and target wwpns found from either `lszfcp -P` or wwpns_shown_in_sys
  if [[ $valid_paths_count -le 1 ]]; then
      printError "Exit MSG: At least 2 paths are required for the server to boot, but ${valid_paths_count} valid path found between FCP devices: ${INPUT_FCPS[*]} and wwpns: ${INPUT_WWPNS[*]}."
      exit 18
  fi
  # Usable FCP: Only the FCPs that we can find a valid target WWPN via the `lszfcp -P` can be defined
  # to VM.
  # Check whether the count of usable FCPs are bigger than the minimum FCP count
  #
  usable_fcps=(${!valid_paths_dict[*]})
  if [[ ${#usable_fcps[@]} -lt $minfcp ]]; then
      printError "Exit MSG: Allocated FCPs include: ${INPUT_FCPS[*]}, but the number of usable FCPs(${usable_fcps[*]}) is less than required minimum FCP count: $minfcp."
      exit 9
  fi

  # 
  # check and record all the path that show up after unit_add (combinations of FCP and WWPN)
  # the available_paths_dict will be in the format: (["FCP1"]="W1 W2" ["FCP2"]="W3 W4")
  declare -A available_paths_dict
  # the first_fcp and first_wwpn will be used in single path scenario (multipathd service not running)
  first_fcp=""
  first_wwpn=""
  disk_path=""
  
  # loop to do unit_add to each (fcp, wwpn) in valid_paths_dict and check lun existence
  for fcp in ${!valid_paths_dict[*]}
  do
      wwpns=(${valid_paths_dict[$fcp]})
      inform "valid target ports found on FCP: $fcp are: ${wwpns[*]}."
      for wwpn in "${wwpns[@]}"
      do
          # 1. do connectdisk which will do unit_add
          addLun $fcp 0x${wwpn} ${lun}
          rc=$?
          if [[ $rc -ne 0 ]]; then
              printError "Failed to add disk on fcp: $fcp and wwpn: $wwpn, rc is $rc."
          else
              path="/dev/disk/by-path/ccw-0.0.${fcp}-zfcp-0x${wwpn}:${lun}"
              if [[ -z $diskPath ]]; then
                  # set diskPath to the first found valid path
                  # this diskPath will only be used to access volume when multipathd service
                  # is not started. Otherwise it will normally be used to find the multipath disk.
                  diskPath=$path
                  first_fcp=$fcp
                  first_wwpn=$wwpn
              fi
              inform "Successfully added disk path: $path."
              # add the paths to the dict
              if [[ -z ${available_paths_dict["$fcp"]} ]]; then
                  available_paths_dict+=([$fcp]="$wwpn")
              else
                  old_value=${available_paths_dict["$fcp"]}
                  new_value=${old_value}" "$wwpn
                  available_paths_dict[$fcp]=$new_value
              fi
              # the /dev/disk/by-path/ccw-0.0.${fcp}-zfcp-0x${wwpn}:${lun) existence is checked in addLUN
          fi
      done
  done

  # check whether there is valid paths found
  if [[ -z $diskPath ]]; then
    printError "Exit MSG: No avaialble disk found between FCP devices: ${INPUT_FCPS[*]} and wwpns: ${INPUT_WWPNS[*]}"
    printLogs
    exit 10
  fi

  # get the valid paths string to return to upper layer
  # format: "FCP1:W1 W2,FCP2:W3 W4"
  # print the valid path count and details
  inform "$valid_paths_count valid paths found."
  valid_paths_str=""
  for fcp in $(echo ${!valid_paths_dict[*]})
  do
	wwpns=${valid_paths_dict[$fcp]}
	if [[ -z $valid_paths_str ]]; then
	    valid_paths_str=$fcp":"$wwpns
    else
        valid_paths_str+=","$fcp":"$wwpns
    fi
  done
  # sample:
  # valid paths string is: 1a1b:500507680b21bac6 500507680b21bac7 500507680b22bac6 500507680b22bac7,
  # 1b1b:500507680b21bac6 500507680b21bac7 500507680b22bac6 500507680b22bac7
  inform "valid paths string is: $valid_paths_str."

  if [[ $multipath_enabled == 1 ]]; then
      if [[ -z "$wwid" ]]; then
          wwid=`/usr/lib/udev/scsi_id --whitelisted $diskPath 2> /dev/null`
          inform "The volume wwid is $wwid."
      fi
      devNode="/dev/disk/by-id/dm-uuid-part1-mpath-$wwid"
      inform "Multipath is enabled, using multipath devNode: $devNode."
  else
      # use the first fcp and wwpn found in valid paths
      devNode="/dev/disk/by-path/ccw-0.0.${first_fcp}-zfcp-0x${first_wwpn}:${lun}-part1"
      # For blockdev command use.
      inform "Multipath is not enabled, using single path devNode: $devNode."
  fi

  refreshZIPL $devNode
} #refreshFCPBootmap

function refreshBootMap {
  : SOURCE: ${BASH_SOURCE}
  : STACK:  ${FUNCNAME[@]}
  # @Description:
  #   Refresh the bootmap of disk image.
  # @Code:

  if [[ $format = 'FCP' ]]; then
    refreshFCPBootmap
  elif [[ $format = 'FBA' ]]; then
    :
  else
    printError "Exit MSG: Device type not recognised."
    exit 11
  fi 
} #refreshBootMap


###############################################################################
### START EXECUTION ###########################################################
###############################################################################

parseArgs $@
checkSanity
refreshBootMap

###############################################################################
### END OF SCRIPT #############################################################
###############################################################################
