#!/bin/sh
#
#  Copyright (c) 2011 Platform Computing
#  Copyright (c) 2007 Platform Computing
#
# This script is a wrapper for openmpi mpirun
# it generates the machine file based on the hosts
# given to it by openlava.
#

usage() {
        cat <<USEEOF
USAGE:  $0
        This command is a wrapper for mpirun (openmpi).  It can
        only be run within Lava using bsub e.g.
                bsub -n # "$0 -np # {my mpi command and args}"

        The wrapper will automatically generate the
        machinefile used by mpirun.

        NOTE:  The list of hosts cannot exceed 4KBytes.
USEEOF
}

if [ x"${LSB_JOBFILENAME}" = x -o x"${LSB_HOSTS}" = x ]; then
    usage
    exit -1
fi

MYARGS=$*
WORKDIR=`dirname ${LSB_JOBFILENAME}`
MACHFILE=${WORKDIR}/mpi_machines
ARGLIST=${WORKDIR}/mpi_args

# Check if mpirun is in the PATH
T=`which mpirun`
if [ $? -ne 0 ]; then
    echo "Error:  mpirun is not in your PATH."
    exit -2
fi

echo "${MYARGS}" > ${ARGLIST}
T=`grep -- -machinefile ${ARGLIST} |wc -l`
if [ $T -gt 0 ]; then
    echo "Error:  Do not provide the machinefile for mpirun."
    echo "        It is generated automatically for you."
    exit -3
fi

# Make the open-mpi machine file
echo "${LSB_HOSTS}" > ${MACHFILE}.lst
tr '\/ ' '\r\n' < ${MACHFILE}.lst > ${MACHFILE}

MPIRUN=`which --skip-alias mpirun`
${MPIRUN} -x LD_LIBRARY_PATH -machinefile ${MACHFILE} ${MYARGS}

exit $?
