#! /bin/bash

MYNAME=$(basename $0)
MYDIR=$(dirname $0)
MYDIR=$(readlink -m $MYDIR)
#VERBOSE=""
VERBOSE="true"
VERBOSE_OPTION="--verbose"
LOGFILE="$HOME/$MYNAME.log"
EXITCODE=0
COMMAND_LINE_OPTIONS="$0 $*"

GIT_REPO_NAME=""
GIT_HASH=""
GIT_BRANCH=""
GIT_COMMIT_DATE=""
START_EPOCH_TIME="$(date +%s)"

NO_GIT_PULL=""
OPTION_NO_COMPILE=""
OPTION_KEEP_NODES=""

OPTION_TEST_NAME=""
OPTION_GIT_BRANCH=""
OPTION_GIT_COMMIT=""

declare -A REMOTE_SERVER_STATE

if [ -f "$MYDIR/load_config.sh" ]; then
    source $MYDIR/load_config.sh
else
    echo "File $MYDIR/load_config.sh was not found." >&2
    exit 5
fi

LOGFILE="$PROJECT_LOGDIR/$MYNAME.log"
echo "" >$LOGFILE

source $MYDIR/utilityfunctions.sh

if [ "${PROJECT_AUTO_PULL_WORKER}" != "true" ]; then
    NO_GIT_PULL="true"
fi

function printHelpAndExit()
{
cat <<EOF

Usage:
  $MYNAME [OPTION]...

  $MYNAME - Runs Cmon tests on multiple servers.

  -h, --help              Print this help and exit.
  -v, --version           Print version information and exit.
  --verbose               Print more messages.
  --log-file=FILE         Store all the messages in the given file too.

  --pid                   Print the PID of the test script if it is running.

  --no-git-pull           Do not execute 'git pull' before the test.
  --no-compile            Do not compile the source, use as it is.
  --keep-nodes            Pass --keep-nodes to tests not to destroy containers.
  --git-branch=BRANCH     The git branch to test.
  --git-commit=HASH       The git commit to test.

  --s9s                   Test the s9s project.
  --test=NAME             The name of the test to run.

EXAMPLE:
  $MYNAME --server=blade01

EOF

  exit 0
}

ARGS=$(\
    getopt \
        -o hvj: \
        -l "help,verbose,version,log-file:,server:,\
pid,\
test:,unit-tests,functional-tests,os:,s9s,\
no-git-pull,no-compile,keep-nodes,wait,git-branch:,git-commit:" \
        -- "$@")

if [ $? -ne 0 ]; then
    exit 6
fi

eval set -- "$ARGS"
while true; do
    case "$1" in
        -h|--help)
            shift
            printHelpAndExit
            ;;

        --verbose)
            shift
            VERBOSE="true"
            VERBOSE_OPTION="--verbose"
            ;;

        -v|--version)
            shift
            printVersionAndExit
            ;;

        --log-file)
            shift
            LOGFILE=$(readlink -m "$1")
            shift
            ;;

        --keep-nodes)
            shift
            OPTION_KEEP_NODES="--keep-nodes"
            ;;

        --test)
            shift
            OPTION_TEST_NAME="$1"
            shift
            ;;

        --no-git-pull)
            shift
            NO_GIT_PULL="true"
            ;;

        --no-compile)
            shift
            OPTION_NO_COMPILE="true"
            ;;

        --wait)
            shift
            WAIT_OPTION="--wait"
            ;;

        --git-branch)
            OPTION_GIT_BRANCH="$2"
            shift 2
            ;;

        --git-commit)
            OPTION_GIT_COMMIT="$2"
            shift 2
            ;;

        --)
            shift
            break
            ;;

        *)
            printVerbose "Unhandled option $1"
            printError "Unhandled option $1"
            exit 1
            ;;
    esac
done

#
# This is how we log from this script. Every run truncates the log file.
#
printVerbose "Starting test runner script..."
printVerbose "             TEST HOST: $(hostname)"
printVerbose "                MYNAME: $MYNAME"
printVerbose "                 MYDIR: $MYDIR"
printVerbose "  COMMAND_LINE_OPTIONS: $COMMAND_LINE_OPTIONS"
printVerbose "       TEST_REPORT_DIR: $PROJECT_TEST_REPORT_DIR"
printVerbose "     OPTION_GIT_BRANCH: $OPTION_GIT_BRANCH"
printVerbose "     OPTION_GIT_COMMIT: $OPTION_GIT_COMMIT"
printVerbose "                  date: $(date "+%Y-%m-%d %H:%M:%S")"

function prompt()
{
    echo "${XTERM_COLOR_GREEN}$USER@$HOSTNAME:$(pwd)\$${TERM_NORMAL}"
}


# Execute arguments as shell command in silent mode (stdout to null)
function run_silent()
{
    local cmd="$1"
    shift

    printVerbose "$(prompt) $cmd $*"
    $cmd $* 2>&1 >/dev/null | tee -a $LOGFILE
    retcode=$?
    if [ "$cmd" == "git" -o "$cmd" == "rm" ]; then
        if [ $retcode -gt 1 ]; then
            printVerbose "Exit code: $retcode"
        fi
    elif [ $retcode -ne 0 ]; then
        printVerbose "Exit code: $retcode"
    fi
    return $retcode
}

# Execute arguments as shell command
function run()
{
    local cmd="$1"
    shift

    printVerbose "$(prompt) $cmd $*"
    $cmd $* 2>&1 | tee -a $LOGFILE
    retcode=$?
    if [ "$cmd" == "git" -o "$cmd" == "rm" ]; then
        if [ $retcode -gt 1 ]; then
            printVerbose "Exit code: $retcode"
        fi
    elif [ $retcode -ne 0 ]; then
        printVerbose "Exit code: $retcode"
    fi
    return $retcode
}

function changedir()
{
    printVerbose "$(prompt) cd $*"
    err=$(cd $* > /dev/null 2>&1)
    retcode=$?
    if [ $retcode -ne 0 ]; then
        printVerbose "$err"
        printVerbose "Exit code: $retcode"
    else
        cd $*
        retcode=$?
    fi
    return $retcode
}

#
# $1: test name
#
function result_file_name_base()
{
    echo "${OPTION_TEST_NAME}-${GIT_REPO_NAME}-${GIT_BRANCH}-${GIT_HASH}"
}

#
# Copies the log file of this script to the test report directory
#
function save_to_results()
{
    local localfile="$1"
    local savedfile="${PROJECT_TEST_REPORT_DIR}/$2"

    if [ -f "$localfile" ]; then
        #printVerbose "cp $localfile $savedfile"
        cp "$localfile" "$savedfile"
    else
        touch "$savedfile"
    fi
}

#
# Moves the log file of this script to the test report directory
#
function move_to_results()
{
    local source="$1"
    local dest="$2"

    save_to_results "$source" "$dest"

    #printVerbose "rm -f $source"
    rm -f "$source"
}

#
# Copies the log file of this script to the test report directory
#
function save_slog_file()
{
    save_to_results "$LOGFILE" "$(result_file_name_base).slog"
}

#
# This expects to be in the source directory.
#
function pull_source()
{
    local current_branch
    local requested_branch
    local requested_commit

    if [ "$NO_GIT_PULL" ]; then
        return 0
    fi

    while [ -n "$1" ]; do
        case "$1" in
            --requested-branch)
                requested_branch="$2"
                shift 2
                ;;

            --requested-commit)
                requested_commit="$2"
                shift 2
                ;;

            *)
                break
                ;;
        esac
    done

    #
    # Checking the branch.
    #
    current_branch=$(run git rev-parse --abbrev-ref HEAD)
    printVerbose "Current branch is '$current_branch'."

    if [ -z "$requested_branch" ]; then
        printVerbose "Requested branch is not set, using 'master'."
        requested_branch="master"
    else
        printVerbose "Requested branch is '$requested_branch'."
    fi

    run_silent git fetch --all

    # always reset any local changes first, then go to master
    run_silent git reset --hard

    run_silent git checkout master

    if [ "$requested_branch" != "master" ]; then
        # first delete any local branch with the same name
        # (as the remote branch might be overwritten from stratch)
        run_silent git branch -d $requested_branch

        # and lets check out the branch now
        # (it will be re-constructed from remote)
        run_silent git checkout $requested_branch
    fi

    run_silent git pull

    if [ ! -z "$requested_commit" ]; then
        run_silent git checkout $requested_commit
    fi

    # Files are left everywhere... :(
    if [ -d "./tests/configs/tls" ]; then
        run_silent rm -rvf "./tests/configs/tls"
    fi
}

function detect_repo_commit()
{
    GIT_HASH=$(git log -n 1 --pretty=format:"%H")
    GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
    if [ "$GIT_BRANCH" = "HEAD" ]; then
        GIT_BRANCH="$OPTION_GIT_BRANCH"
    fi
    GIT_COMMIT_DATE=$(git show -s --format=%ci $GIT_HASH)
}

function compile_clustercontrol()
{
    local usecache="$1"

    changedir "${PROJECT_CC_WORKER_DIR}"

    if [ "$OPTION_NO_COMPILE" ]; then
        return 0
    fi

    generate_test_result_file "PREPARING"

    # Check the CC binary cache

    if [ "$usecache" = "usecache" ]; then
        printVerbose "GIT_HASH: $GIT_HASH"
        if [ "$GIT_HASH" != "" ]; then
            repodir="$HOME/${GIT_REPO_NAME}_bin_repo/$GIT_BRANCH-$GIT_HASH"
            ccdir="${PROJECT_CC_WORKER_DIR}"

            if [ -d "$repodir" ]; then
                printVerbose "Using previously compiled binaries from $repodir"
                run_silent cp $repodir/ft_full $ccdir/tests/ft_full/
                run_silent cp $repodir/cmon $ccdir/src/
                return 0
            fi
        fi
    fi

    # Cache miss, lets compile

    pip-host-control --testing --status="Compiling CC for $OPTION_TEST_NAME"

    # clean previous object, make and other compilation files

    run_silent make clean
    run_silent make distclean

    # autogen

    run_silent ./autogen.sh --with-no-optimization $prefix_option
    retcode=$?
    if [ $retcode -ne 0 ]; then
        return "$retcode"
    fi

    # clean previous object, make and other compilation files
    # This is in case the previous Makefiles were incomplete and
    # real clean can work only after running autogen.sh

    run_silent make clean
    run_silent make distclean

    # Autogen again to have the new Makefiles

    run_silent ./autogen.sh --with-no-optimization $prefix_option
    retcode=$?
    if [ $retcode -ne 0 ]; then
        return "$retcode"
    fi

    # Lets make the new build

    run_silent make -j10
    retcode=$?
    if [ "$retcode" -ne 0 ]; then
        return $retcode
    fi

    # make tests

    changedir "${PROJECT_CC_WORKER_DIR}/tests"
    run_silent make -j10
    retcode=$?
    changedir "${PROJECT_CC_WORKER_DIR}"
    if [ "$retcode" -ne 0 ]; then
        return $retcode
    fi

    # cache created binaries (cmon, ft_full)

    if [ "$usecache" = "usecache" ]; then
        repodir="$HOME/${GIT_REPO_NAME}_bin_repo/$GIT_BRANCH-$GIT_HASH"
        ccdir="${PROJECT_CC_WORKER_DIR}"

        run_silent mkdir -p "$repodir"
        printVerbose "Saving created binaries to $repodir"
        run_silent cp $ccdir/tests/ft_full/ft_full $repodir/
        run_silent cp $ccdir/src/cmon $repodir/
    fi

    return 0
}

function install_clustercontrol()
{
    changedir "${PROJECT_CC_WORKER_DIR}"

    # make install

    run_silent sudo make install
    retcode=$?; if [ "$retcode" -gt 1 ]; then return $retcode; fi

    run ${PROJECT_CC_TESTORIGIN_DIR}/tests/scripts/create_cmon_db.sh
    retcode=$?; if [ "$retcode" -ne 0 ]; then return $retcode; fi

    CC_IP=$(host $(hostname) | cut -d' ' -f 4)

    printVerbose "Creating /etc/cmon.cnf file"
    cat <<EOF | sudo tee /etc/cmon.cnf >> $LOGFILE 
#
# clustercontrol-controller configuration file
# Copyright 2016 severalnines.com
#

## CMON database config - mysql_password is for the 'cmon' user
mysql_port=3306
mysql_hostname=127.0.0.1
mysql_password=pwd

## hostname is the hostname of the current host
hostname=${CC_IP}

## The default logfile
logfile=/var/log/cmon.log

## For possible access restriction
# rpc_key = DEADBEEF01234567ABCDEF
#
controller_id=6d7aa76c-ba79-44a1-863a-1262049a4dbe

rpc_key=0a7c043fb43a13fcb953e9689ff177eb4ee6aadd
EOF

    return 0
}

function compile_s9s_tools()
{
    local usecache="$1"

    changedir "${PROJECT_S9S_WORKER_DIR}"

    if [ "$OPTION_NO_COMPILE" ]; then
        return 0
    fi

    pip-host-control --testing --status="Compiling s9s for $OPTION_TEST_NAME"

    # Check the s9s binary cache

    if [ "$usecache" = "usecache" ]; then
        S9S_GIT_HASH=$(git log -n 1 --pretty=format:"%H")
        printVerbose "S9S_GIT_HASH: $S9S_GIT_HASH"
        S9S_GIT_BRANCH=master
        if [ "$S9S_GIT_HASH" != "" ]; then
            repodir="$HOME/${GIT_REPO_NAME}_bin_repo/$S9S_GIT_BRANCH-$S9S_GIT_HASH"
            s9sdir="${PROJECT_S9S_WORKER_DIR}"
            printVerbose "repodir: $repodir"

            if [ -d "$repodir" ]; then
                printVerbose "Using previously compiled s9s"
                run_silent cp $repodir/s9s $s9sdir/s9s/
                run_silent cp $repodir/libs9s.so $s9sdir/libs9s/
                run_silent /usr/bin/install -c $repodir/libs9s.so "/usr/lib"
                run_silent /usr/bin/install -c $repodir/s9s "/usr/bin"
                return 0
            fi
        fi
    fi

    # clean old object, Makefile and other produced files

    run_silent make clean
    run_silent make distclean

    # autogen

    run_silent ./autogen.sh $prefix_option
    retcode=$?
    if [ $retcode -ne 0 ]; then
        return "$retcode"
    fi

    # clean old object, Makefile and other produced files
    # This is in case the Makefiles were broken end the first make clean
    # were not complete.

    run_silent make clean
    run_silent make distclean

    # autogen again to have the new Makefiles

    run_silent ./autogen.sh $prefix_option
    retcode=$?
    if [ $retcode -ne 0 ]; then
        return "$retcode"
    fi

    # make

    run_silent make -j10
    retcode=$?
    if [ "$retcode" -ne 0 ]; then
        return $retcode
    fi

    # make tests

    changedir "${PROJECT_S9S_WORKER_DIR}/tests"
    run_silent make -j10
    retcode=$?
    if [ "$retcode" -ne 0 ]; then
        changedir "${PROJECT_S9S_WORKER_DIR}"
        return $retcode
    fi
    changedir "${PROJECT_S9S_WORKER_DIR}"

    # cache s9s

    if [ "$usecache" = "usecache" ]; then
        repodir="$HOME/${GIT_REPO_NAME}_bin_repo/$S9S_GIT_BRANCH-$S9S_GIT_HASH"
        s9sdir="${PROJECT_S9S_WORKER_DIR}"

        run_silent mkdir -p "$repodir"
        printVerbose "Saving s9s to cache"
        run_silent cp $s9sdir/s9s/s9s $repodir/
        run_silent cp $s9sdir/libs9s/libs9s.so $repodir/
    fi

    return "$retcode"
}

function install_s9s_tools()
{
    changedir "${PROJECT_S9S_WORKER_DIR}"

    # make install

    run_silent sudo make install
    retcode=$?; if [ "$retcode" -gt 1 ]; then return $retcode; fi

    return 0
}

#
# $1: test name
#
# The report is a HTML file where we put the report about the test run.
#
function report_file_name()
{
    echo "$(result_file_name_base).html"
}

function result_file_name()
{
    echo "$(result_file_name_base).result"
}

function log_bin_file_name()
{
    echo "$(result_file_name_base).cmon"
}

function log_core_file_name()
{
    echo "$(result_file_name_base).core"
}

function log_file_name()
{
    echo "$(result_file_name_base).log"
}

function daemon_log_file_name()
{
    echo "$(result_file_name_base).dlog"
}

function error_report_file_name()
{
    echo "$(result_file_name_base).error_report.tar.gz"
}

function ft_test_ended()
{
    local exit_code="$1"
    local backend_pid="$2"
    local core_file

    #printVerbose "<h2>Backtrace</h2>"

    printVerbose "The test ended."
    printVerbose "  exit_code: $exit_code"
    printVerbose "         wd: '$PWD'"

    pip-host-control --testing --status="Finishing $OPTION_TEST_NAME"

    #for core_file in \
    #    *core* \
    #    */*core* \
    #    ${PROJECT_CC_WORKER_DIR}/*core* \
    #    ${PROJECT_CC_WORKER_DIR}/tests/*core* \
    for core_file in \
        /tmp/*.core;
    do
        if [ ! -f "$core_file" ]; then
            continue
        fi

        printVerbose "Core file found: $core_file"
    done

    if [ ! -f "$core_file" -a "$exit_code" -eq 0 ]; then
        return 0
    fi

    if [ -z "$(which gdb)" ]; then
        printVerbose "Installing gdb."
        run_silent sudo apt-get -y --force-yes install gdb
    fi

    if [ -e "$core_file" ]; then
        printVerbose "Saving backtrace after crash"
        run sudo gdb \
            -batch \
            -ex "thread apply all back" \
            -ex "quit" \
            cmon "$core_file"

        run_silent sudo mkdir -p "$HOME/tmp"

        #sudo mv "$core_file" "/tmp/$(log_core_file_name)"
        #sudo chown $USER:$USER "/tmp/$(log_core_file_name)"
        #move_to_results "/tmp/$(log_core_file_name)" "$(log_core_file_name)"
        move_to_results "$core_file" "$(log_core_file_name)"

        move_to_results "/var/log/cmon.log" "$(log_bin_file_name)"

    elif [ -n "$backend_pid" -a "$exit_code" -ne 0 ]; then
        printVerbose "Saving backtrace"
        run sudo gdb \
            -p $backend_pid -batch \
            -ex "thread apply all back" \
            -ex "quit"

    else
        printVerbose "No backtrace is necessary. No crash, no error on exit."
    fi
}

#
# $2: the state, "FAILURE", "RUNNING" or "SUCCESS"
#
function collect_test_log_files()
{
    local test_state
    local pid_to_watch
    local counter

    while true; do
        case "$1" in
            --pid)
                shift
                pid_to_watch="$1"
                shift
                ;;

            *)
                break
        esac
    done

    test_state="$1"


    #
    # If we have a PID we do the uploading until that ends. If not, we just
    # upload once.
    #
    counter=0

    while true; do
        local report_file=$(report_file_name)
        local tmp_file="/var/tmp/$report_file"

        cat <<EOF >>$tmp_file
        <html>
            <h1>Test results for '$1'</h1>
            <p>
            This test created no report. Try to check the programs output
            for details. The web interface shows the output when clicked on
            the test's name.
            </p>
        </html>
EOF

        move_to_results "$tmp_file" "$report_file"

        generate_test_result_file "$test_state"

        if [ -f /var/log/cmon.log ]; then
            save_to_results /var/log/cmon.log "$(daemon_log_file_name)"
        fi

        if [ -z "$pid_to_watch" ]; then
            break
        fi

        if ! ps --pid $pid_to_watch; then
            break
        fi

        # We are not gonna run indefinitely, that is a huge disaster.
        let counter+=10
        if [ $counter -gt 5400 ]; then
            printVerbose "One and a half hour elapsed, the watcher will abort now."
            break
        fi

        sleep 10
    done
}

function elapsedTimeString()
{
    local start_date="$1"
    local end_date="$(date +%s)"
    local T=$((end_date-start_date))

    printf "%02d:%02d:%02d" \
        "$((T/3600%24))" "$((T/60%60))" "$((T%60))"
}

function generate_test_result_file()
{
    local git_has_local_changes="no"
    local elapsed_time
    local test_state="$1"

    git_has_local_changes="no"

    if ! git diff-index --quiet HEAD --; then
        git_has_local_changes="yes"
    fi

    elapsed_time="$(date +%s)"
    let elapsed_time-="$START_EPOCH_TIME"

    cat > "${PROJECT_TEST_REPORT_DIR}/$(result_file_name)" <<EOF
test_server="$(hostname)"
testname="$OPTION_TEST_NAME"
gitreponame="$GIT_REPO_NAME"
githash="$GIT_HASH"
gitbranch="$GIT_BRANCH"
gitcommitdate="$GIT_COMMIT_DATE"
githaslocalchanges="$git_has_local_changes"
state="$test_state"
report_file="$(report_file_name)"
user="$USER"
elapsed_time="$elapsed_time"
elapsed_time_string="$(elapsedTimeString $START_EPOCH_TIME)"
test_start_epoch="$START_EPOCH_TIME"
EOF
}

function enable_core_file_generation()
{
    printVerbose "Enabling core file creation."
    echo "core" | sudo tee /proc/sys/kernel/core_pattern >> /dev/null 2>> $LOGFILE
    retcode=$?
    run sudo cat /proc/sys/kernel/core_pattern
    if [ $retcode -ne 0 ]; then
        printVerbose "Failed: echo 'core' | sudo tee /proc/sys/kernel/core_pattern"
    fi

    for core_file in \
        /tmp/*.core \
        *core* \
        */*core* \
        ${PROJECT_CC_WORKER_DIR}/*core* \
        ${PROJECT_CC_WORKER_DIR}/tests/*core*;
    do
        if [ ! -f "$core_file" ]; then
            continue
        fi

        rm -f "$core_file"
    done

    run ulimit -c unlimited

    sudo sysctl -p /etc/sysctl.conf
    printVerbose "Core pattern: $(sudo cat /proc/sys/kernel/core_pattern)"
    printVerbose "User limits :"
    run ulimit -a
    printVerbose "User limits with sudo:"
    run sudo ulimit -c unlimited
    run sudo ulimit -a
}

function stop_or_kill_cmon()
{
    local backend_pid=$1

    printVerbose "Stopping the cmon backend."

    if [ "$backend_pid" != "" ]; then
        sudo kill $backend_pid
        sleep 3
    fi

    backend_pid=$(pidof cmon)
    if [ "$backend_pid" != "" ]; then
        sudo kill -9 $backend_pid
        sleep 3
    fi

    backend_pid=$(pidof cmon)
    if [ "$backend_pid" != "" ]; then
        sudo pkill cmon
        sleep 3
    fi

    backend_pid=$(pidof cmon)
    if [ "$backend_pid" != "" ]; then
        sudo pkill -9 cmon
        sleep 3
    fi
}

function cleanup_test_host()
{
    run_silent sudo rm -f /var/log/cmon.log
    run_silent sudo rm -f /tmp/cmon_1.log
    run_silent sudo rm -f /tmp/cmon_*.log
    run_silent sudo rm -f /tmp/BACKUP-*
    run_silent sudo rm -fr /var/log/cmon
    run_silent sudo rm -f /var/log/cmon*
    run_silent sudo rm -fr /var/lib/cmon
    run_silent sudo rm -f /var/lib/cmon*
    run_silent sudo rm -f /etc/cmon.d/*.cnf
    run_silent sudo rm -f /root/.s9s/s9s.conf
    run_silent sudo mv -v /etc/s9s.conf /etc/s9s.conf.bak
    #.$(date +%Y%m%d%H%M)
    run_silent sudo mv -v /home/${USER}/.s9s/s9s.conf /home/${USER}/.s9s/s9s.conf.bak
    #.$(date +%Y%m%d%H%M)

    # FIXME Why rm -rvf /etc/cmon.d/*.cnf is not working
    # and why below is working?:
    for file in $(sudo ls /etc/cmon.d/ -1); do
        sudo rm -fr "/etc/cmon.d/$file";
    done
}

function run_s9s_ft_test()
{
    local return_code
    local logfile

    logfile="${PROJECT_TEST_REPORT_DIR}/$(log_file_name)"

    printVerbose "Preparing to run $OPTION_TEST_NAME."
    save_slog_file

    export DOWNLOAD_KEYSERVER="hkp://keyserver.ubuntu.com"
    changedir "${PROJECT_CC_WORKER_DIR}"

    if [ ! -x "/usr/sbin/cmon" ]; then
        printVerbose "/usr/sbin/cmon binary is not found or not executeable."
        save_slog_file
        return 1
    fi

    #
    # Cleaning up before.
    #

    backend_pid=$(pidof cmon)
    if [ "" != "$backend_pid" ]; then
        printVerbose "Cmon is already running."
        printVerbose "Backend PID is ${backend_pid}."
        printVerbose "Now we will kill and start it again."
        stop_or_kill_cmon "$backend_pid"
    fi

    cleanup_test_host

    #
    # Start cmon
    #

    printVerbose "Running 'cmon'..."

    run sudo /usr/sbin/cmon &
    backend_pid=$!
    # Sleep a lot, so we make sure that cmon can initialize itself
    sleep 15
    #backend_pid=$(pidof cmon)
    printVerbose "Backend PID is ${backend_pid}."

    #
    # Running the test script itself.
    #

    changedir "${PROJECT_S9S_TESTORIGIN_DIR}/tests"

    if [ ! -f "$OPTION_TEST_NAME" ]; then
        printVerbose "Program $OPTION_TEST_NAME was not found in $PWD"
        return 1
    fi

    printVerbose "Running '$OPTION_TEST_NAME'..."
    save_slog_file
    printVerbose "timeout --kill-after=92m 90m ./$OPTION_TEST_NAME \
        --print-commands --reset-config --server=$(hostname) \
        \>\>$logfile 2\>\>$logfile"

    pip-host-control --testing --status="Running $OPTION_TEST_NAME"

    changedir "${PROJECT_S9S_TESTORIGIN_DIR}/tests"

    timeout --kill-after=92m 90m ./$OPTION_TEST_NAME \
        --print-commands \
        --reset-config \
        --server=$(hostname) \
        $OPTION_KEEP_NODES \
        >>"$logfile" 2>>"$logfile" &

    test_pid=$!
    printVerbose "Test script PID is ${test_pid}."

    #
    # Starting a watcher that keeps collecting the output...
    #

    #collect_test_log_files --pid "$test_pid" "RUNNING"
    collect_test_log_files "RUNNING"

    #watcher_pid=$!
    #printVerbose "Watcher PID is ${watcher_pid}."
    save_slog_file

    #
    # Waiting for the test to be ended.
    #

    printVerbose "Waiting until the test script ends..."
    wait "$test_pid" 2>> $LOGFILE
    return_code=$?

    printVerbose "Finished ${OPTION_TEST_NAME} ($return_code)."

    collect_test_log_files "RUNNING"

    #printVerbose "Waiting for test watcher to end..."
    #wait "$watcher_pid" 2>> $LOGFILE

    changedir "${PROJECT_CC_WORKER_DIR}/tests"

    ft_test_ended \
        "$return_code" \
        "$backend_pid"

    #
    # Lets collect error report
    #

    CID=$(s9s cluster --list --long | head -n 2 | tail -n 1 | awk '{print $1}')
    if [ "$CID" = "" -o "$CID" = "0" ]; then
        CID="1"
    fi
    error_report_file_dirty=$(sudo s9s_error_reporter -i ${CID} -d "$HOME" | grep "Please attach" | awk '{print $3}')
    error_report_file=$(echo "${error_report_file_dirty}" | ansi2txt)
    if [ "${error_report_file}" != "" ]; then
        run_silent sudo chown ${PROJECT_OWNER}:${PROJECT_OWNER} ${error_report_file}
        err_report_file="$(error_report_file_name)"
        run_silent mv -f "${error_report_file}" "${PROJECT_TEST_REPORT_DIR}/${err_report_file}"
        printVerbose "Generated error report ${err_report_file}"
    else
        printVerbose "Failed to generate error report or to find report file."
    fi

    #
    # Cleaning up after.
    #

    stop_or_kill_cmon "$backend_pid"

    if [ $return_code -eq 0 ]; then
        collect_test_log_files "SUCCESS"
    else
        collect_test_log_files "FAILURE"
    fi

    printVerbose "Finished running '$OPTION_TEST_NAME' ($return_code)."
    save_slog_file
}

#
# DEPRECATED, not maintained, buggy
#
function run_cc_ft_test()
{
    local return_code
    local logfile

    logfile="${PROJECT_TEST_REPORT_DIR}/$(log_file_name)"

    cleanup_test_host

    #
    # Notifying the user about a test is starting.
    #
    printVerbose "Preparing to run $OPTION_TEST_NAME."
    save_slog_file

    #
    # Running the actual test.
    #
    changedir "${PROJECT_CC_WORKER_DIR}/tests"

    if [ -f "Vagrantfile" ]; then
        vagrant destroy -f
        rm -f "Vagrantfile"
    fi

    rm -f test_ssh_key.pub*

    printVerbose "Running ${OPTION_TEST_NAME}."

    if [ ! -f "$OPTION_TEST_NAME/$OPTION_TEST_NAME" ]; then
        printVerbose "Program $OPTION_TEST_NAME was not found."

        return 1
    fi

    pip-host-control --testing --status="Running $OPTION_TEST_NAME"
    printVerbose "Running $OPTION_TEST_NAME."
    ./$OPTION_TEST_NAME/$OPTION_TEST_NAME \
        --template=headless \
        >>"$logfile" 2>>"$logfile" &

    test_pid=$!

    #collect_test_log_files --pid "$test_pid" "RUNNING"
    collect_test_log_files "RUNNING"

    #watcher_pid=$!
    wait "$test_pid"
    return_code=$?

    printVerbose "Finished ${OPTION_TEST_NAME} ($return_code)."

    ft_test_ended \
        "$return_code" \
        ""

    if [ $return_code -eq 0 ]; then
        collect_test_log_files "SUCCESS"
    else
        collect_test_log_files "FAILURE"
    fi

    printVerbose "Finished running '$OPTION_TEST_NAME' ($return_code)."
    save_slog_file
}

#
# Running s9s unit tests by pip-cmon-test was mark by Laszlo as:
# "This is under development."
#
function run_s9s_ut_test()
{
    local return_code
    local logfile

    logfile="${PROJECT_TEST_REPORT_DIR}/$(log_file_name)"

    #
    # Notifying the user about a test is starting.
    #
    printVerbose "Preparing to run $OPTION_TEST_NAME."
    save_slog_file

    #
    # Running the actual test.
    #
    changedir "${PROJECT_S9S_WORKER_DIR}/tests"

    collect_test_log_files "RUNNING"

    #
    # Running the test.
    #
    printVerbose "Running $OPTION_TEST_NAME."
    printVerbose "$OPTION_TEST_NAME/$OPTION_TEST_NAME >>$logfile 2>>$logfile"

    if [ ! -f "$OPTION_TEST_NAME/$OPTION_TEST_NAME" ]; then
        printVerbose "File $OPTION_TEST_NAME/$OPTION_TEST_NAME not found in ${PWD}."

        collect_test_log_files "FAILURE"

        return 1
    fi

    printVerbose "Running $OPTION_TEST_NAME."

    pip-host-control --testing --status="Running $OPTION_TEST_NAME"

    echo "<pre>" >>$logfile 2>>$logfile
    $OPTION_TEST_NAME/$OPTION_TEST_NAME --color --verbose >>$logfile 2>>$logfile
    return_code=$?
    echo "</pre>" >>$logfile 2>>$logfile

    printVerbose "Finished ${OPTION_TEST_NAME} ($return_code)."
    if [ $return_code -eq 0 ]; then
        collect_test_log_files "SUCCESS"
    else
        collect_test_log_files "FAILURE"
    fi

    printVerbose "Finished running '$OPTION_TEST_NAME' ($return_code)."
    save_slog_file
}

function install_packages_on_test_server()
{
    pip-host-control --testing --status="Started host for $OPTION_TEST_NAME"

    if [ -z "$(psql --version)" ]; then
        printVerbose "Installing postgresql..."
        sudo apt-get -y --force-yes install postgresql
    fi

    if [ -z "$(bc --version)" ]; then
        printVerbose "Installing bc..."
        sudo apt-get -y --force-yes install bc
    fi

    if [ ! -f "/usr/include/hiredis/hiredis.h" ]; then
        printVerbose "Installing hiredis..."
        sudo apt-get -y --force-yes install libhiredis-dev
    fi
}

function main_case()
{
    case "$OPTION_TEST_NAME" in
        ft_*.sh)
            GIT_REPO_NAME="cmon"
            changedir "${PROJECT_CC_WORKER_DIR}"
            pull_source \
                --requested-branch  "$OPTION_GIT_BRANCH" \
                --requested-commit  "$OPTION_GIT_COMMIT"
            detect_repo_commit
            compile_clustercontrol "usecache"
            retcode=$?; if [ "$retcode" -ne 0 ]; then return $retcode; fi
            install_clustercontrol
            retcode=$?; if [ "$retcode" -ne 0 ]; then return $retcode; fi
            changedir "${PROJECT_S9S_WORKER_DIR}"
            pull_source --requested-branch  master
            compile_s9s_tools "usecache"
            retcode=$?; if [ "$retcode" -ne 0 ]; then return $retcode; fi
            install_s9s_tools
            retcode=$?; if [ "$retcode" -ne 0 ]; then return $retcode; fi
            run_s9s_ft_test
            retcode=$?; if [ "$retcode" -ne 0 ]; then return $retcode; fi
            ;;
        ft_*)
            run_cc_ft_test
            ;;
        ut_*)
            GIT_REPO_NAME="s9stools"
            changedir "${PROJECT_S9S_WORKER_DIR}"
            pull_source --requested-branch  master
            detect_repo_commit
            compile_s9s_tools "dontusecache"
            retcode=$?; if [ "$retcode" -ne 0 ]; then return $retcode; fi
            run_s9s_ut_test
            retcode=$?; if [ "$retcode" -ne 0 ]; then return $retcode; fi
            ;;
        *)
            printVerbose "Unhandled test name '${OPTION_TEST_NAME}'."
    esac

    return 0
}

#
function main()
{
    local retcode=0

    printVerbose "Starting main()."
    pip-host-control --testing --status="Prepare host for $OPTION_TEST_NAME"

    install_packages_on_test_server

    #
    # Finding the source root.
    #
    if [ ! -d "${PROJECT_CC_WORKER_DIR}" ]; then
        printError "Directory '${PROJECT_CC_WORKER_DIR}' does not exist."
        return 6
    fi

    run_silent rm -f "${PROJECT_CC_WORKER_DIR}/*.log"
    run_silent rm -f "${PROJECT_CC_WORKER_DIR}/tests/*.log"

    enable_core_file_generation

    main_case

    if [ "$retcode" -ne 0 ]; then
        generate_test_result_file "FAILURE"
        save_slog_file
        return "$retcode"
    fi

    printVerbose "Finished with tests, idle."
}

#checkOtherInstances
#printVerbose "checkOtherInstances : $?"
pip-host-control --testing --status="Started with $OPTION_TEST_NAME"
main
RET=$?
pip-host-control --idle --status="Idle"
exit $RET

