#! /bin/bash

MYNAME=$(basename $0)
MYDIR=$(dirname $0)
MYDIR=$(readlink -m "$MYDIR")
PATH=${MYDIR}:${PATH}
VERSION="0.0.1"

OPTION_CONTINUOUS=""
OPTION_MANAGER="false"

START_TIME=$(date)
LAST_TEST_STARTED="-"
LAST_TEST_NAME="-"
LAST_HOST_NAME="-"
LAST_USER_NAME="-"
SUM_STARTED_TESTS=0

if [ -f "$MYDIR/load_config.sh" ]; then
    source $MYDIR/load_config.sh
else
    echo "File $MYDIR/load_config.sh was not found." >&2
    exit 5
fi


#source "./utilityfunctions.sh"
#[ -f "./htmlfunctions.sh" ] && source "./htmlfunctions.sh"

#
# Prints the help text and exits.
#
function printHelpAndExit()
{
cat <<EOF
Usage:
  $MYNAME [OPTION]... 

  $MYNAME - Runs tests as they appear in the file-system.

  -h, --help         Print this help and exit.
  -v, --version      Print version information and exit.
  --verbose          Print more messages.
  --log-file=FILE    Store all the messages in the given file too.
  --manager          Don start pip-cmon-test, just scp schedules to worker hosts
  --continuous       Continue watching if no schedules found.

EOF
    exit 0
}

ARGS=$(\
    getopt \
        -o hv \
        -l "help,verbose,version,log-file:,manager,continuous" \
        -- "$@")

if [ $? -ne 0 ]; then
    exit 6
fi

eval set -- "$ARGS"
while true; do
    case "$1" in
        -h|--help)
            shift
            printHelpAndExit
            ;;

        --verbose)
            shift
            VERBOSE_OPTION="--verbose"
            ;;

        -v|--version)
            shift
            printVersionAndExit
            ;;

        --manager)
            shift
            OPTION_MANAGER="true"
            ;;

        --continuous)
            shift
            OPTION_CONTINUOUS="true"
            ;;

        --)
            shift
            break
            ;;

        *)
            break
            ;;
    esac
done

function changedir()
{
    local dir=$1
    echo "cd $dir"
    cd $dir > /dev/null 2> /dev/null
    if [ $? -ne 0 ]; then
        echo "Failed to change directory to $dir."
        return 1
    fi
    return 0
}

#
#
# Worker server related functions
#
#

function pull_local_testorigin_source()
{
    local server=$(hostname)

    if [ "${PROJECT_AUTO_PULL_TESTORIGIN}" = "true" ]; then

        echo ""

        echo "Pulling in ${server}:${PROJECT_CC_TESTORIGIN_DIR}"

        changedir "${PROJECT_CC_TESTORIGIN_DIR}"
        git reset --hard HEAD >/dev/null 2>/dev/null
        git pull >/dev/null 2>/dev/null

        echo "Pulling in ${server}:${PROJECT_S9S_TESTORIGIN_DIR}"

        changedir "${PROJECT_S9S_TESTORIGIN_DIR}"
        git reset --hard HEAD >/dev/null 2>/dev/null
        git pull >/dev/null 2>/dev/null
    fi
}

function cleanup_containers()
{
    if [ "${PROJECT_KEEP_NODE_CONTAINERS}" != "true" ]; then
        changedir "${PROJECT_S9S_TESTORIGIN_DIR}"
        ft_container_list=$(pip-container-list | grep "^ft_")
        pip-container-destroy ${ft_container_list}
        ft_container_list=$(pip-container-list | grep "^node")
        pip-container-destroy ${ft_container_list}
        ft_container_list=$(pip-container-list | grep "^ftgalera")
        pip-container-destroy ${ft_container_list}
    fi
}

function run_test()
{
    local config_file="$1"
    local server="$(hostname)"
    local testname=""
    local state=""
    local user=""
    local project=""
    local retcode
    local counter

    echo " --------------------------------------"
    echo "Current date: $(date "+%Y-%m-%d %H:%M")"
    echo "Loading '$config_file'..."
    source "$config_file"

    # 
    # Getting the list of available servers.
    #
    echo "Getting idle test servers"

    let counter=0
    if [ -z "$user" ]; then
        user="$USER"
    fi

    logbasename="$PROJECT_TEST_REPORT_DIR/${testname}-${gitbranch}-${githash}"
    pipcmontestlog="${logbasename}.pip-cmon-test.log"

    LAST_USER_NAME="$user"

    echo "           server: $server"
    echo "         testname: $testname"
    echo "            state: $state"
    echo "             user: $user"
    echo "          project: $project"
    echo "       git branch: $gitbranch"
    echo "       git commit: $githash"
    echo "pip-cmon-test.log: $pipcmontestlog"
    echo ""

    LAST_TEST_STARTED=$(date)
    LAST_TEST_NAME="$testname"
    LAST_HOST_NAME="$server"

    let SUM_STARTED_TESTS+=1

    echo "nohup pip-cmon-test "\
        "$VERBOSE_OPTION "\
        "--test=$testname "\
        "--git-branch=$gitbranch"\
        "--git-commit=$githash" \> ${pipcmontestlog}

    nohup pip-cmon-test \
        $VERBOSE_OPTION \
        --test="$testname" \
        --git-branch="$gitbranch" \
        --git-commit="$githash" > ${pipcmontestlog} 2> ${pipcmontestlog}

    mv "$PROJECT_LOGDIR/pip-cmon-test.log" "${logbasename}.slog"

    cleanup_containers

    rm -f "$config_file"

    # Some stats

    echo ""
    printf "        Start time: %s\n" "$START_TIME"
    printf "      Current time: %s\n" "$(date)"
    printf " Last test started: %s\n" "$LAST_TEST_STARTED"
    printf "\n"
    printf "    Last test name: %s\n" "$LAST_TEST_NAME"
    printf "         Last user: %s\n" "$LAST_USER_NAME"
    printf "    Last host name: %s\n" "$LAST_HOST_NAME"
    printf "           Started: %'d test(s)\n" "$SUM_STARTED_TESTS"
    printf "\n\n\n"
}

#
#
# Manager server related functions
#
#

function refresh_daemon_on_worker()
{
    local idle_server=$1

    #
    # Pull testorigin repos on the worker
    #
    # FIXME BUGGY, PROJECT_S9S_TESTORIGIN_DIR IS A LOCAL CONFIG VALUE
    #

    echo "Pulling in ${idle_server}:${PROJECT_S9S_TESTORIGIN_DIR}"

    ssh ${CONF_HOST_USER}@${idle_server} -- \
    git -C "${PROJECT_S9S_TESTORIGIN_DIR}" reset --hard HEAD
        #>/dev/null 2>/dev/null

    ssh ${CONF_HOST_USER}@${idle_server} -- \
    git -C "${PROJECT_S9S_TESTORIGIN_DIR}" pull
        #>/dev/null 2>/dev/null

    #
    # Clear the logs on the worker
    #

    echo "ssh ${CONF_HOST_USER}@${idle_server} -- "\
        "sudo rm ${CONF_HOST_DATA_ROOT}/log/*"

    ssh ${CONF_HOST_USER}@${idle_server} -- \
        sudo rm "${CONF_HOST_DATA_ROOT}/log/*"

    echo "ssh ${CONF_HOST_USER}@${idle_server} -- "\
        "sudo rm /var/log/pip-test-daemon.log"

    ssh ${CONF_HOST_USER}@${idle_server} -- \
        sudo rm /var/log/pip-test-daemon.log

    #
    # Restart pip-test-daemon on the worker
    #

    echo "ssh ${CONF_HOST_USER}@${idle_server} -- "\
        "sudo systemctl restart pip-test-daemon "

    ssh ${CONF_HOST_USER}@${idle_server} -- \
        sudo systemctl restart pip-test-daemon
}

function collect_host_status()
{
    local remote=$1
    local user=$2
    local worker_data_root=$3

    #echo "scp $user@$remote:${worker_data_root}/servers/* "\
    #                            "${PROJECT_HOST_STAT_DIR}/"
    scp "$user"@"$remote":"${worker_data_root}/servers/${remote}.*" \
            "${PROJECT_HOST_STAT_DIR}/" > /dev/null 2> /dev/null
}

function collect_test_results()
{
    local remote=$1
    local user=$2
    local worker_data_root=$3

    # Skip localhost. It would result in deleting local test results
    if [ "${idle_server}" = "$(hostname)" ]; then
        continue
    fi

    #echo "scp $user@$remote:${worker_data_root}/test-results/* "\
    #                                "${PROJECT_TEST_REPORT_DIR}/"
    scp "$user"@"$remote":"${worker_data_root}/test-results/*" \
            "${PROJECT_TEST_REPORT_DIR}/" > /dev/null 2> /dev/null

    #echo "ssh ${user}@${remote} rm ${worker_data_root}/test-results/*"
    ssh "${user}"@"${remote}" rm "${worker_data_root}/test-results/*" \
            > /dev/null 2> /dev/null
}

function collect_test_results_from_idle_workers()
{
    #echo "Checking for idle workers to download results from"

    for idle_server in $(pip-server-control --list --idle); do
        #echo "Found idle worker: ${idle_server}"

        if [ ! -e /etc/s9s-cmon-test/${idle_server}.host ]; then
            echo "ERROR: Host file for idle host ${idle_server} not found."
            continue
        fi

        source /etc/s9s-cmon-test/${idle_server}.host
        if [ "${CONF_HOST_USER}" = "" ]; then
            CONF_HOST_USER=${USER}
        fi
        if [ "${CONF_HOST_DATA_ROOT}" = "" ]; then
            echo "ERROR: No test data root is defined for host ${idle_host}"
            return 1
        fi

        #echo "Lets collect results from ${idle_server}"
        collect_test_results \
            "${idle_server}" \
            "${CONF_HOST_USER}" \
            "${CONF_HOST_DATA_ROOT}"
    done
}

function delegate_test()
{
    local config_file="$1"
    local idle_server=""
    local testname=""
    local state=""
    local user=""
    local project=""
    local retcode

    echo " --------------------------------------"
    echo "Loading '$config_file'..."
    source "$config_file"

    if [ -z "$user" ]; then
        user="$USER"
    fi

    echo "pip-server-control --list --idle"
    idle_server=$(pip-server-control --list --idle | head -n 1)

    if [ "$idle_server" = "" ]; then
        echo "No idle test server available"
        return 6
    fi

    #logbasename="$PROJECT_TEST_REPORT_DIR/${testname}-${gitbranch}-${githash}"

    echo "      idle_server: $idle_server"
    echo "         testname: $testname"
    echo "            state: $state"
    echo "             user: $user"
    echo "          project: $project"
    echo "       git branch: $gitbranch"
    echo "       git commit: $githash"
    echo ""

    #
    # Load the host config file of the found idle server
    #

    # Lets source host config file for the found idle host
    CONF_HOST_CATEGORY=""
    CONF_HOST_MAC=""
    CONF_HOST_NAME=""
    CONF_HOST_PDU=""
    CONF_HOST_PROTECTED=""
    CONF_HOST_URL=""
    CONF_HOST_VIRTUAL=""
    CONF_HOST_CONTAINERS=""
    CONF_HOST_DATA_ROOT=""
    CONF_HOST_USER=""
    CONF_HOST_AUTO_UPDATE_WORKER=""
    if [ ! -e /etc/s9s-cmon-test/${idle_server}.host ]; then
        echo "ERROR: Host file for idle host ${idle_server} not found."
        return 1
    fi
    source /etc/s9s-cmon-test/${idle_server}.host
    if [ "${CONF_HOST_USER}" = "" ]; then
        CONF_HOST_USER=${USER}
    fi
    if [ "${CONF_HOST_DATA_ROOT}" = "" ]; then
        echo "ERROR: No test data root is defined for host ${idle_host}"
        return 1
    fi

    #
    # Collect any previous test results from ${idle_server}
    #

    collect_test_results \
        "${idle_server}" \
        "${CONF_HOST_USER}" \
        "${CONF_HOST_DATA_ROOT}"

    if [ "${CONF_HOST_AUTO_UPDATE_WORKER}" = "true" ]; then
        refresh_daemon_on_worker ${idle_server}
    fi

    #
    # Delegate the ${config_file} schedule to ${idle_server}
    #

    echo scp "${config_file}" ${CONF_HOST_USER}@${idle_server}:${CONF_HOST_DATA_ROOT}/scheduled-tests

    scp "${config_file}" ${CONF_HOST_USER}@${idle_server}:${CONF_HOST_DATA_ROOT}/scheduled-tests

    # Immediatelly set testing state on worker,
    # so we wont delegate it a second scheduled test

    echo "ssh ${CONF_HOST_USER}@${idle_server} "\
        "pip-host-control "\
        "--testing "\
        "--status=Scheduled for $(basename ${config_file})"

    ssh ${CONF_HOST_USER}@${idle_server} \
        pip-host-control \
        --testing \
        --status="Scheduled for $(basename ${config_file})"

    collect_host_status \
        "${idle_server}" \
        "${CONF_HOST_USER}" \
        "${CONF_HOST_DATA_ROOT}"

    #
    # Delete test schedule so we wont
    # delegate it to another worker
    #

    echo "rm -f $config_file"
    rm -f "$config_file"
}




#
#
# Common functionc for worker and manager mode
#
#


# Lists the schedule files in reverse timestamp
# order (oldest files first).
function schedule_files()
{
    local root_dir="$PROJECT_DATA_ROOT"

    if [ -z "$root_dir" ]; then
        root_dir="/var/lib/test-ui"
    fi

    if [ ! -d "$root_dir" ]; then
        printError "The project root dir '$root_dir' does not exist."
        return 1
    fi

    ls -tr $root_dir/scheduled-tests/*.conf 2>/dev/null
}

# The main function that does some polling
# by checking the scheduled tests and
# returns once in every round.
function process_scheduled_tests()
{
    local config_file
    local config_files=$(schedule_files)

    # Even if there is no new test to start, we should check
    # if any worker host has results files to download
    if [ "$OPTION_MANAGER" = "true" ]; then
        collect_test_results_from_idle_workers
    fi

    if [ -z "$config_files" ]; then
        #echo "No tests scheduled."
        return 0
    fi

    pull_local_testorigin_source

    cleanup_containers

    for config_file in $(schedule_files); do
        echo "SCHEDULE file: $config_file"
        if [ ! -f "$config_file" ]; then
            continue
        elif [ ! -x "$config_file" ]; then
            # Files that are not executable are being
            # created and not yet ready.
            continue
        fi

        if [ "$OPTION_MANAGER" = "true" ]; then
            delegate_test "$config_file"
            if [ $? -ne 0 ]; then
                break
            fi
        else
            run_test "$config_file"
        fi
    done
}

if [ "$OPTION_CONTINUOUS" ]; then
    while true; do
        process_scheduled_tests
        sleep 10
    done
else 
    process_scheduled_tests
fi

