#! /bin/bash

MYNAME=$(basename "$0")
MYDIR=$(dirname "$0")
MYDIR=$(readlink -m "$MYDIR")
VERSION="0.0.6"
VERBOSE=""
LOGFILE=""

if [ -f "$MYDIR/load_config.sh" ]; then
    source $MYDIR/load_config.sh
else
    echo "File $MYDIR/load_config.sh was not found." >&2
    exit 5
fi

OPTION_DEVICE=""
OPTION_PRINT_FILES=""
OPTION_USER=$PROJECT_OWNER
LONG_OPTION=""

if [ -f "$MYDIR/utilityfunctions.sh" ]; then
    source $MYDIR/utilityfunctions.sh
else
    echo "File '$MYDIR/utilityfunctions.sh' was not found." >&2
    exit 5
fi

SUM_SERVERS=0
SUM_OFF_LINE_SERVERS=0
SUM_TESTING_SERVERS=0
SUM_PROTECTED_SERVERS=0
SUM_IDLE_SERVERS=0
SUM_ON_SERVERS=0

SUM_CORES=0
SUM_MEMORY=0
SUM_DISK=0
SUM_CONTAINERS=0

SUM_CORES_ONLINE=0
SUM_MEMORY_ONLINE=0
SUM_DISK_ONLINE=0

OPTION_STATE_FILTER=""

#
# Prints the software version and exits.
#
function printVersionAndExit()
{
    echo "$MYNAME Version $VERSION on $(hostname)" >&2
}

function printHelpAndExit()
{
cat <<EOF
Usage:
  $MYNAME [OPTION]... [HOSTNAME]...

  $MYNAME - Lists and controls servers.

  GENERAL OPTIONS
  -h, --help           Print this help and exit.
  -v, --version        Print version information and exit.
  --verbose            Print more messages.
  --log-file=FILE      Store all the messages in the given file too.
  
  MAIN OPTIONS
  --list               List the power distribution units.
  --ping               Check if the server is up.
  --shut-down          Shut down the server.
  --wake               Start the server.
  
  LIST SPECIFIERS
  --print-containers   Print the permanent containers on the server(s).
  --print-files        Print the config files of the server(s). 

  FORMAT SPECIFIERS
  --long               Print detailed/long list.
  --print-json         Print the output in JSon format.
  --print-report       Print a summary at the end.
  --only-report        Print only the report, not the list.
  
  --user=USERNAME      Run the program in the name of the user.
  --wait               Wait for the server.
  --all                Turn on or off all the servers.

  FILTER OPTIONS
  --disabled           Process only the disabled servers.
  --idle               Process only the idle servers.
  --offline            Process only the servers that are off-line.
  --online             Process only the servers that are on-line.
  --protected          Process only the servers that are protected.
  --testing            Process only the servers that are testing.

EXAMPLE:
  pip-server-control --list --long host01 host02
  pip-server-control --wake --wait host01 host02
  pip-server-control --shut-down host01 host02

EOF
    exit 0
}

ARGS=$(\
    getopt \
        -o hvs:c:l \
        -l "help,verbose,version,log-file:,long,print-report,print-json,\
only-report,list,wake,shut-down,ping,\
print-files,print-containers,user:,wait,all,\
disabled,idle,offline,online,protected,testing" \
        -- "$@")

if [ $? -ne 0 ]; then
    exit 6
fi

eval set -- "$ARGS"
while true; do
    case "$1" in
        -h|--help)
            shift
            printHelpAndExit
            ;;

        --verbose)
            shift
            VERBOSE="true"
            VERBOSE_OPTION="--verbose"
            ;;

        -v|--version)
            shift
            VERSION_OPTION="--version"
            ;;

        --log-file)
            shift
            LOGFILE=$(readlink -m "$1")
            shift
            ;;

        --list)
            shift
            LIST_OPTION="true"
            ;;

        --wake)
            shift
            WAKE_OPTION="true"
            ;;
        
        --shut-down)
            shift
            SHUT_DOWN_OPTION="true"
            ;;

        --ping)
            shift
            OPTION_PING="true"
            ;;

        --long)
            shift
            LONG_OPTION="true"
            ;;

        --print-report)
            shift
            OPTION_PRINT_REPORT="yes"
            ;;

        --only-report)
            shift
            OPTION_ONLY_REPORT="true"
            ;;

        --print-json)
            shift
            OPTION_PRINT_JSON="yes"
            ;;

        --user)
            shift
            OPTION_USER="$1"
            shift
            ;;

        --wait)
            shift
            WAIT_OPTION="true"
            ;;
 
        --all)
            shift
            ALL_OPTION="true"
            ;;

        --print-files)
            shift
            OPTION_PRINT_FILES="true"
            ;;

        --print-containers)
            shift
            OPTION_PRINT_CONTAINERS="true"
            ;;

        --idle)
            shift
            OPTION_STATE_FILTER="IDLE"
            ;;
        
        --testing)
            shift
            OPTION_STATE_FILTER="TESTING"
            ;;

        --offline)
            shift
            OPTION_STATE_FILTER="OFF-LINE"
            ;;
        
        --online)
            shift
            OPTION_STATE_FILTER="ON-LINE"
            ;;

        --disabled)
            shift
            OPTION_STATE_FILTER="DISABLED"
            ;;
        
        --protected)
            shift
            OPTION_STATE_FILTER="PROTECTED"
            ;;

        --)
            shift
            break
            ;;

        *)
            break
            ;;
    esac
done

ARGUMENTS=$*

#
# Returns true if the given server was requested providing the --server command
# line option or the --server option was not used at all.
# FIXME: Move this to include file.
#
function is_server_requested()
{
    local server="$1"
    local this_server

    if [ -z "$ARGUMENTS" ]; then
        return 0
    fi

    for this_server in $(echo "$ARGUMENTS" | tr ',' ' '); do
        if [ "$this_server" = "$server" ]; then
            return 0
        fi
    done

    return 1
}

#
# $1: the server name
#
function is_server_running_ssh()
{
    local serverName="$1"
    local isOK

    isOk=$(ssh -o ConnectTimeout=1 "$serverName" 2>/dev/null -- echo OK)
    if [ "$isOk" = "OK" ]; then
        return 0
    fi

    return 1
}

#
# $2: the server name
#
# Waits until the server is accepting SSH connections. There is also a timeout
# implemented here.
#
function wait_for_server_ssh()
{
    local serverName="$1"
    local nTry=0

    while true; do
        if is_server_running_ssh "$serverName"; then
            echo "Server '$serverName' is started."
            return 0
        fi

        # 60 x 10 = 10 minutes
        if [ "$nTry" -gt 60 ]; then
            printError "Server '$serverName' did not came alive."
            return 1
        fi

        sleep 10
        let nTry+=1
    done

    return 0
}

#
# $1: the server name
#
# This function finds the MAC address for a given host. The MAC addresses should
# be stored in '~/.pip/servers.mac' like this:
#
# server1;00:1a:a0:03:b3:b5
# server2;00:1d:09:15:17:8a
# storage01;00:1d:09:24:59:33
#
# Very primitive format, I did not want to spend much time developing it. 
#
function mac_for_server()
{
    local serverName="$1"
    local line
    local field1
    local field2

    for line in $(cat ~/.pip/servers.mac); do
        line=$(echo "$line" | tr ';' ' ' | tr ',' ' ')
        field1=$(echo "$line" | cut -d' ' -f1)
        field2=$(echo "$line" | cut -d' ' -f2)

        if [ "$field1" = "$serverName" ]; then
            echo "$field2"
            return 0
        fi
    done
}

#
# $1: the name of the server
#
function wake_server()
{
    local hostName="$1"
    local macAddress
    local pdu
    local config_file
    local output
    local retcode

    if [ -z "$hostName" ]; then
        return 1
    fi

    config_file="/etc/s9s-cmon-test/${hostName}.host"
    if [ ! -f "$config_file" ]; then
        echo "There is no .host file in /etc/s9s-cmon-test/ directory."
        exit -1
    fi

    if [ -f "$config_file" ]; then
        printVerbose "Loading config from '$config_file'."
        CONF_HOST_NAME=""
        CONF_HOST_URL=""
        CONF_HOST_CATEGORY=""
        CONF_HOST_MAC=""
        CONF_HOST_PROTECTED=""
        CONF_HOST_VIRTUAL=""
        CONF_HOST_PDU=""
        CONF_HOST_ILOM=""
        
        source "$config_file"
    fi

    printVerbose "Controller is starting up ${hostname}."

    if [ -n "$CONF_HOST_PDU" ]; then
        printVerbose "Powering on server $CONF_HOST_NAME through the power distribution unit."
        for pdu in $(echo "$CONF_HOST_PDU" | tr ',' ' '); do
            printVerbose "Switching on PDU $pdu."
            pip-pdu-control --on $pdu
        done

        sleep 6
    fi

    if [ -n "$CONF_HOST_ILOM" ]; then
        printVerbose "Will use ILOM of ${CONF_HOST_NAME}."
        printVerbose "Will use integrated lights out manager on ${CONF_HOST_NAME}."

        # ILOM needs time to start...
        sleep 20
        printVerbose "ILOM start on ${CONF_HOST_NAME} proceeds in 40 seconds"
        
        sleep 20
        printVerbose "ILOM start on ${CONF_HOST_NAME} proceeds in 20 seconds"

        sleep 20
        printVerbose "Calling ILOM to start ${CONF_HOST_NAME}."
        printVerbose "Using ILOM to start ${CONF_HOST_NAME}."
        file="/tmp/ilom.expect"

        sudo rm -f "$file"

        cat >$file <<EOF
#!/usr/bin/expect -f

set timeout 120
spawn ssh -o StrictHostKeyChecking=no root@192.168.3.100
expect "*?assword:*"
send -- "changeme\r"
sleep 2
send -- "start -script /SYS\r"
sleep 1
send -- "exit\r"
expect eof
EOF
        chmod +x "$file"
        
        output=$(timeout 15 $file)
        retcode=$?

        if [ "$retcode" -ne 0 ]; then
            printVerbose "ILOM request failed with error code $retcode, attempting to compensate."
            printWarning "output: $output"
            printWarning "The $file exited with error."
            sleep 30
            timeout 15 $file

            if [ $? -ne 0 ]; then
                printVerbose "The second ILOM request is failed, server is off-line."
            else
                printVerbose "The second ILOM request succeeded."
            fi
        else
            printVerbose "ILOM request suceeded, server is booting."
        fi

        rm -f $file
    else
        macAddress=$(mac_for_server "$hostName")
        if [ -z "$macAddress" ]; then
            printError "MAC address for $hostName was not found."
            printVerbose "MAC address for $hostName was not found."
            return 1
        fi

        printVerbose "Waking server ${hostName} on ${macAddress}."
        printVerbose "Sending wake on lan message to host ${hostName}."
        wakeonlan "$macAddress" >/dev/null 2>/dev/null
    fi

    return 0
}

function shut_down_server()
{
    local hostname="$1"

    if [ -z "$hostname" ]; then
        return 1
    fi

    printVerbose "Shutting down ${hostname}."
    echo "Trying to shut down host ${hostname}."
    if [ -n "$OPTION_USER" ]; then
        ssh -o UserKnownHostsFile=/dev/null \
            -o StrictHostKeyChecking=no \
            -o LogLevel=quiet \
            -i /home/${OPTION_USER}/.ssh/id_rsa \
            "${OPTION_USER}@$hostname" -- sudo poweroff
    else
        ssh -o UserKnownHostsFile=/dev/null \
            -o StrictHostKeyChecking=no \
            -o LogLevel=quiet \
            "$hostname" -- sudo poweroff
    fi
}

#
# pip-server-control --ping --print-json host01
# {"hostname": "host01", "status": "on"}
#
function ping_server()
{
    local hostname="$1"
    local state
    local message

    if [ -z "$hostname" ]; then
        return 1
    fi

    if ping -c 1 -W 1 $hostname >/dev/null 2>/dev/null; then
        state="on"
    else
        state="off"
    fi
        
    if [ -n "$OPTION_PRINT_JSON" ]; then
        message="{\"hostname\": \"$hostname\", \"status\": \"$state\"}"
    else
        message="$state"
    fi

    echo "$message"
}

function print_header()
{
    local extra_title

    if [ -n "$OPTION_ONLY_REPORT" ]; then
        return 0
    fi

    echo -en $TERM_BOLD
    
    if [ -n "$OPTION_PRINT_FILES" ]; then
        extra_title="CONFIG_FILE"
    elif [ -n "$OPTION_PRINT_CONTAINERS" ]; then
        extra_title="PERMANENT_CONTAINERS"
    else
        extra_title="DESCRIPTION"
    fi

    cat <<EOF
SERVER     GROUP    CORES  MEM    DISK CN   LOAD STATUS    $extra_title
--------------------------------------------------------------------------------
EOF

    echo -en $TERM_NORMAL
}

function print_summary()
{
    local disk_terabytes
    local disk_terabytes_online

    if [ -z "$OPTION_PRINT_REPORT" ]; then
        return 0
    fi

    let disk_terabytes=SUM_DISK
    let disk_terabytes/=1000
    
    let disk_terabytes_online=SUM_DISK_ONLINE
    let disk_terabytes_online/=1000

    if [ -z "$OPTION_PRINT_JSON" -a -n "$LONG_OPTION" ]; then
        printf "\n"
        printf "Summary:\n"
        printf "         Total: %'6d server(s)\n"   "$SUM_SERVERS" 
        printf "       Testing: %'6d server(s)\n"   "$SUM_TESTING_SERVERS" 
        printf "          Idle: %'6d server(s)\n"   "$SUM_IDLE_SERVERS" 
        printf "      Off-line: %'6d server(s)\n"   "$SUM_OFF_LINE_SERVERS" 
        printf "     Protected: %'6d server(s)\n"   "$SUM_PROTECTED_SERVERS" 

#        printf "    Containers: %'6d instance(s)\n" "$SUM_CONTAINERS"
        printf "\n"
        printf "Total\n"
        printf "     Processor: %'6d thread(s)\n"  "$SUM_CORES" 
        printf "  Total memory: %6d GBytes\n"      "$SUM_MEMORY"
        printf "   Total Disks: %6d TBytes\n"      "$disk_terabytes"
        printf "\n"
        printf "Online\n"
        printf "     Processor: %'6d thread(s)\n"  "$SUM_CORES_ONLINE" 
        printf "  Total memory: %6d GBytes\n"      "$SUM_MEMORY_ONLINE"
        printf "   Total Disks: %6d TBytes\n" "$disk_terabytes_online"
        printf "\n"
    elif [ -z "$OPTION_PRINT_JSON" ]; then
        printf "%3dsrv %4did %4dtt\n"  \
            "$SUM_ON_SERVERS" "$SUM_IDLE_SERVERS" "$SUM_TESTING_SERVERS"

        printf "%3dthr %4dGB %4dTB\n" \
            "$SUM_CORES_ONLINE" "$SUM_MEMORY_ONLINE" "$disk_terabytes_online"

    else
        cat <<EOF
{
  "servers": $SUM_SERVERS,
  "testing": $SUM_TESTING_SERVERS,
  "idle": $SUM_IDLE_SERVERS,
  "offLine": $SUM_OFF_LINE_SERVERS,
  "protected": $SUM_PROTECTED_SERVERS,
  "containers": $SUM_CONTAINERS,
  "processorThreads": $SUM_CORES,
  "memoryGbytes": $SUM_MEMORY,
  "diskTerabytes": $disk_terabytes
}
EOF
    fi
}

HOST_STAT_ROOT="$PROJECT_DATA_ROOT/servers"
function download_current_data()
{
    local root_dir=$HOST_STAT_ROOT

    for config in /etc/s9s-cmon-test/*.host; do
        if [ ! -f "$config" ]; then 
            continue
        fi
        
        CONF_HOST_NAME=""
        CONF_HOST_URL=""
        CONF_HOST_CATEGORY=""
        CONF_HOST_STANDARD=""
        CONF_HOST_MAC=""
        CONF_HOST_PROTECTED=""
        CONF_HOST_DATA_ROOT=""
        CONF_HOST_USER=""

        source "$config"

        if [ -n "$category" -a "$CONF_HOST_CATEGORY" != "$category" ]; then
            continue
        fi

        printVerbose "Collecting hostinfo on $CONF_HOST_NAME"

        #ssh $CONF_HOST_NAME -- "sudo pip-host-control"

        printVerbose "Downloading current data from $CONF_HOST_NAME:$CONF_HOST_DATA_ROOT/servers"
        if [ ! -d "$root_dir" ]; then
            printVerbose "Creating directory '$root_dir'."
            mkdir -p "$root_dir"
            if [ ! -d "$root_dir" ]; then
                printError "Unable to create '$root_dir'."
                return 1
            fi
        fi

        printVerbose "Entering directory '$root_dir'..."
        pushd $root_dir >/dev/null 2>/dev/null
        if [ $? -ne 0 ]; then
            printError "Unable to jump into '$root_dir'."
            return 1
        fi

        printVerbose "Downloading $CONF_HOST_NAME:$CONF_HOST_DATA_ROOT/servers/${CONF_HOST_NAME}.hostinfo..."
        scp -p $CONF_HOST_NAME:$CONF_HOST_DATA_ROOT/servers/${CONF_HOST_NAME}.hostinfo . >/dev/null 2>/dev/null
        chown $PROJECT_OWNER:$PROJECT_OWNER ${CONF_HOST_NAME}.hostinfo >/dev/null 2>/dev/null

        popd >/dev/null 2>/dev/null
    done
}

#
# Prints the age of the file in seconds, e.g. 1 means the file was modified 1
# second ago.
#
# $1: the file name
#
function fileAgeInSeconds()
{
    echo $((`date +%s` - `stat -L --format %Y $1` ))
}

#
# This is where we print the detailed list.
#
function print_host_list_long()
{
    local config_file
    local info_file
    local category
    local name

    printVerbose "Printing detailed list..."
    print_header

    for config_file in /etc/s9s-cmon-test/*.host; do
        if [ ! -f "$config_file" ]; then
            continue
        fi

        #echo "device_name=\$(basename $config_file .host)"
        device_name=$(basename "$config_file" .host)
        #echo "info_file=$HOST_STAT_ROOT/${device_name}.hostinfo"
        info_file="$HOST_STAT_ROOT/${device_name}.hostinfo"
        printVerbose "info_file: $info_file"

        if ! is_server_requested "$device_name"; then
            continue
        fi

        CONF_HOST_NAME=""
        CONF_HOST_URL=""
        CONF_HOST_CATEGORY=""
        CONF_HOST_MAC=""
        CONF_HOST_PROTECTED=""
        CONF_HOST_VIRTUAL=""
        CONF_HOST_PDU=""
        CONF_HOST_CONTAINERS=""

        HOSTNAME=""
        NUMBER_OF_CORES="0"
        MEMORY_GIGABYTES="0"
        DISK_GIGABYTES="0"
        NUMBER_OF_CONTAINERS="0"
        AVERAGE_LOAD=""
        HOST_STATUS=""
        STATUS_TEXT=""

        source "$config_file"
        if [ -f "$info_file" ]; then
            source "$info_file"
        fi
    
        if [ -n "$CONF_HOST_VIRTUAL" ]; then
            continue
        fi

        category="$CONF_HOST_CATEGORY"
        if [ -z "$category" ]; then
            category="-"
        fi

        [ -z "$HOST_STATUS" ] && HOST_STATUS="OFF-LINE"
        [ -n "$CONF_HOST_PROTECTED" -a "$HOST_STATUS" != "OFF-LINE" ] && HOST_STATUS="PROTECTED"

        if [ -f "$info_file" ]; then
            fileAge="$(fileAgeInSeconds "$info_file")"
            let fileAge/=60
            if [ "$fileAge" -gt 5 ]; then
                #printVerbose "File '$info_file' older that 5 minutes."
                if [ "$HOST_STATUS" != "DISABLED" ]; then
                    HOST_STATUS="OFF-LINE"
                    STATUS_TEXT="-"
                fi
            fi
        else
            if [ "$HOST_STATUS" != "DISABLED" ]; then
                HOST_STATUS="OFF-LINE"
                STATUS_TEXT="-"
            fi
        fi

        if [ -z "$HOSTNAME" ]; then
                HOST_STATUS="MISSING"
        fi

        if [ "$HOST_STATUS" = "OFF-LINE" ]; then
            ping -c 1 $HOSTNAME > /dev/null 2> /dev/null && \
                HOST_STATUS="IDLE"
        fi
       
        if [ "$OPTION_STATE_FILTER" ]; then 
            if [ "$OPTION_STATE_FILTER" = "ON-LINE" ]; then
                if [ "$HOST_STATUS" = "OFF-LINE" ]; then
                    continue
                fi
            elif [ "$OPTION_STATE_FILTER" != "$HOST_STATUS" ]; then
                continue
            fi
        fi

        let SUM_CORES+=NUMBER_OF_CORES
        let SUM_MEMORY+=MEMORY_GIGABYTES
        let SUM_DISK+=DISK_GIGABYTES
        let SUM_CONTAINERS+=NUMBER_OF_CONTAINERS

        [ -z "$AVERAGE_LOAD" ] && AVERAGE_LOAD="-"

        case "$HOST_STATUS" in
            OFF-LINE)
                let SUM_OFF_LINE_SERVERS+=1
                let SUM_SERVERS+=1
                ;;

            HALTED)
                let SUM_OFF_LINE_SERVERS+=1
                let SUM_SERVERS+=1
                ;;

            TESTING)
                let SUM_TESTING_SERVERS+=1
                let SUM_SERVERS+=1
                let SUM_CORES_ONLINE+=NUMBER_OF_CORES
                let SUM_MEMORY_ONLINE+=MEMORY_GIGABYTES
                let SUM_DISK_ONLINE+=DISK_GIGABYTES
                let SUM_ON_SERVERS+=1
                ;;
           
            PROTECTED)
                let SUM_PROTECTED_SERVERS+=1
                let SUM_SERVERS+=1
                let SUM_CORES_ONLINE+=NUMBER_OF_CORES
                let SUM_MEMORY_ONLINE+=MEMORY_GIGABYTES
                let SUM_DISK_ONLINE+=DISK_GIGABYTES
                let SUM_ON_SERVERS+=1
                ;;

            IDLE)
                let SUM_IDLE_SERVERS+=1
                let SUM_SERVERS+=1
                let SUM_CORES_ONLINE+=NUMBER_OF_CORES
                let SUM_MEMORY_ONLINE+=MEMORY_GIGABYTES
                let SUM_DISK_ONLINE+=DISK_GIGABYTES
                let SUM_ON_SERVERS+=1
                ;;

            *)
                let SUM_SERVERS+=1
                ;;
        esac

        # If only the report needed we are not printing anything.
        if [ -n "$OPTION_ONLY_REPORT" ]; then
            continue
        fi

        #
        # Printing the detailed list here.
        #
        printf "$DEVICE_COLOR%-10s$TERM_NORMAL "  "$device_name"
        printf "$OWNER_COLOR%-10s$TERM_NORMAL "   "$category"
        #printf "$IP_COLOR%-13s$TERM_NORMAL "      "$CONF_HOST_URL"
        
        printf "%2dc "  "$NUMBER_OF_CORES"
        printf "%3dG "  "$MEMORY_GIGABYTES"
        printf "%'6dG " "$DISK_GIGABYTES"
        printf "%3d "   "$NUMBER_OF_CONTAINERS"
        printf "%6s "   "$AVERAGE_LOAD"

        if [ "$HOST_STATUS" = "IDLE" ]; then
            printf "$OK_COLOR%-9s$TERM_NORMAL " "$HOST_STATUS"
        elif [ "$HOST_STATUS" = "TESTING" ]; then
            printf "$OK_COLOR%-9s$TERM_NORMAL " "$HOST_STATUS"
        elif [ "$HOST_STATUS" = "OFF-LINE" ]; then
            printf "$WARN_COLOR%-9s$TERM_NORMAL " "$HOST_STATUS"
        elif [ "$HOST_STATUS" = "ON-LINE" ]; then
            printf "$OK_COLOR%-9s$TERM_NORMAL " "$HOST_STATUS"
        elif [ "$HOST_STATUS" = "HALTED" ]; then
            printf "$WARN_COLOR%-9s$TERM_NORMAL " "$HOST_STATUS"
        elif [ "$HOST_STATUS" = "PROTECTED" ]; then
            printf "$XTERM_COLOR_RED%-9s$TERM_NORMAL " "$HOST_STATUS"
        else
            printf "%-9s " "$HOST_STATUS"
        fi

        if [ -n "$OPTION_PRINT_FILES" ]; then
            tmp=$(echo "$config_file" | sed -e "s#$HOME/#~/#g")
            printf "$FILE_COLOR%s$TERM_NORMAL"     "$tmp"
        elif [ -n "$OPTION_PRINT_CONTAINERS" ]; then
            printf "%s" "$CONF_HOST_CONTAINERS"
        else
            printf "$COMMENT_COLOR%s$TERM_NORMAL"  "$STATUS_TEXT"
        fi

        printf "\n"
    done
}

#
# This is where we print the brief list of hosts.
#
function print_host_list_brief()
{
    local config_file
    local info_file
    local category
    local name

    printVerbose "Printing brief list."
    for config_file in /etc/s9s-cmon-test/*.host; do
        if [ ! -f "$config_file" ]; then
            continue
        fi

        device_name=$(basename "$config_file" .host)
        info_file="$HOST_STAT_ROOT/${device_name}.hostinfo"
        
        if ! is_server_requested "$device_name"; then
            continue
        fi

        CONF_HOST_CATEGORY=""
        CONF_HOST_MAC=""
        CONF_HOST_NAME=""
        CONF_HOST_PDU=""
        CONF_HOST_PROTECTED=""
        CONF_HOST_URL=""
        CONF_HOST_VIRTUAL=""
        CONF_HOST_CONTAINERS=""
        CONF_HOST_DATA_ROOT=""
        CONF_HOST_USER=""

        HOSTNAME=""
        NUMBER_OF_CORES=""
        MEMORY_GIGABYTES=""
        DISK_GIGABYTES=""
        NUMBER_OF_CONTAINERS=""
        AVERAGE_LOAD=""
        HOST_STATUS=""
        STATUS_TEXT=""

        printVerbose "Loading config file '$config_file'."
        source "$config_file"
        if [ -f "$info_file" ]; then
            printVerbose "Loading info file '$info_file'."
            source "$info_file"
        else
            printVerbose "No info file found at '$info_file'."
        fi
    
        let SUM_SERVERS+=1
        
        if [ -n "$CONF_HOST_VIRTUAL" ]; then
            continue
        fi

        category="$CONF_HOST_CATEGORY"
        if [ -z "$category" ]; then
            category="-"
        fi
    
        [ -z "$HOST_STATUS" ] && HOST_STATUS="OFF-LINE"
        [ -n "$CONF_HOST_PROTECTED" ] && HOST_STATUS="PROTECTED"

        if [ -f "$info_file" ]; then
            fileAge="$(fileAgeInSeconds "$info_file")"
            let fileAge/=60
            if [ "$fileAge" -gt 5 ]; then
                if [ "$HOST_STATUS" != "DISABLED" ]; then
                    HOST_STATUS="OFF-LINE"
                    STATUS_TEXT="-"
                fi
            fi
        else
            if [ "$HOST_STATUS" != "DISABLED" ]; then
                HOST_STATUS="OFF-LINE"
                STATUS_TEXT="-"
            fi
        fi

        if [ "$HOST_STATUS" = "OFF-LINE" ]; then
            ping -c 1 $HOSTNAME > /dev/null 2> /dev/null && \
                HOST_STATUS="IDLE"
        fi

        if [ "$OPTION_STATE_FILTER" ]; then 
            if [ "$OPTION_STATE_FILTER" = "ON-LINE" ]; then
                if [ "$HOST_STATUS" = "OFF-LINE" ]; then
                    continue
                fi
            elif [ "$OPTION_STATE_FILTER" != "$HOST_STATUS" ]; then
                continue
            fi
        fi
        
        let SUM_CORES+=NUMBER_OF_CORES
        let SUM_MEMORY+=MEMORY_GIGABYTES

        [ -z "$AVERAGE_LOAD" ] && AVERAGE_LOAD="-"
       
        if [ -n "$OPTION_PRINT_FILES" ]; then
            tmp=$(echo "$config_file" | sed -e "s#$HOME/#~/#g")
            printf "%s\n"     "$tmp"
        elif [ -n "$OPTION_PRINT_CONTAINERS" ]; then
            for container in $(echo "$CONF_HOST_CONTAINERS" | tr ',' ' '); do
                printf "$container\n"
            done
        else             
            printf "%s\n"  "$device_name"
        fi
    done
}

function wake_all()
{
    printVerbose "Starting all servers."
        for server in \
            core2 core3 storage01 \
            host01 host02 host03 host04 host05 \
            host06 host07 host08 host09 host10 host11
        do
            pip-server-control --wake $server
            sleep .5
        done

        pip-pdu-control --on dell1955_pdu01,dell1955_pdu02
        sleep 1
        pip-pdu-control --on dell1955_pdu01,dell1955_pdu02

    printVerbose "Starting all servers finished."
}

if [ -n "$LIST_OPTION" ]; then
    printVerbose "Listing servers..."
    download_current_data
    
    if [ -n "$LONG_OPTION" -o -n "$OPTION_ONLY_REPORT" ]; then
        print_host_list_long
        print_summary
    else
        if [ -t 1 ]; then
            print_host_list_brief | column -s' '
        else
            print_host_list_brief
        fi
    fi

    exit 0
elif [ -n "$ALL_OPTION" ]; then
    if [ "$WAKE_OPTION" ]; then
        wake_all
    elif [ "$SHUT_DOWN_OPTION" ]; then
        printVerbose "Shutting down all servers."
        for server in \
            core2 core3 \
            host01 host02 host03 host04 host05 \
            host06 host07 host08 host09 host10 host11 \
            blade01 blade02 blade03 blade04 blade05 blade06 \
            blade07 blade08 blade09 blade10
        do
            pip-server-control --shut-down $server
        done
        printVerbose "Shutting down all servers finished."
    fi
elif [ -n "$SHUT_DOWN_OPTION" ]; then
    for hostname in $(echo "$ARGUMENTS" | tr ',' ' '); do 
        shut_down_server "$hostname"
    done
elif [ -n "$OPTION_PING" ]; then
    for hostname in $(echo "$ARGUMENTS" | tr ',' ' '); do 
        ping_server "$hostname"
    done 
elif [ -n "$WAKE_OPTION" ]; then
    #
    # Wake up the servers.
    #
    nthserver=0
    for hostname in $(echo "$ARGUMENTS" | tr ',' ' '); do 
        # Not proper: we should know when the last actual wake happened.
        if [ "$nthserver" -gt 0 ]; then
            printVerbose "Waiting a while before waking the next server."
            sleep 4
        fi

        wake_server "$hostname"
        let nthserver+=1
    done

    retcode=0
    if [ -n "$WAIT_OPTION" ]; then
        for hostname in $(echo "$ARGUMENTS" | tr ',' ' '); do
            wait_for_server_ssh "$hostname"
            if [ $? -gt 0 ]; then
                retcode=1
            fi
        done
    fi
fi


