#!/bin/bash
#
# cs_show_supportconfig
#
# (c) 2011-2014 SUSE Linux GmbH, Germany. Author: L.Pinne.
# GNU Public License. No warranty.
#
# Version: 0.01 2014-03-25
#
# For SLES11, for Supportconfig Script Version: 2.25-237
#

EXE=$0
ERR="/dev/null"
TMP="/tmp/cltl.$RANDOM"

CFG="/etc/ClusterTools2/cs_show_supportconfig"
test -s $CFG && source $CFG
test -z "${TEMP}" && TMP="$TEMP}"

# TODO make DIR= work
DIR=${2:-$PWD}
FUN=${3:-"chk_overview"}
OWD=$PWD


function echo_msgsep(){
	echo "============================================================ ${1:0:14} ==="
}


function split_file(){
	F=$1
	mkdir -p ${TMP}/$F
	cp $F ${TMP}/${F}/
	cd ${TMP}/$F
#	csplit $F /^#==\[ [a-zA-Z] ^C \]==*#$/ {*}
	csplit $F /^#==..Co.*===========================#$/ {*} >>$ERR
	cd $OLDPWD
}


function chk_hardware(){

	echo_msgsep $FUNCNAME

	echo "machine:"
	FIL="hardware.txt"
	grep -e "system.hardware.*=" -e "system.*.machine.=..s390" $FIL | grep -v "system.*video"
	grep "system.firmware.*=" $FIL
	grep -e "Manufacturer:.\"[A-Za-z]" -e "Product:.\"" -e "Product.id:.\"" -e "Serial:.\"" $FIL | sort -u
	grep -e "str3:.\"../../....\"" -e "str5:.\"[A-Za-z]" $FIL |\
		 grep -v "Unknown"
	grep -A4 "^# Virtuali.ation" $FIL
	echo
	
	echo "processor:"
	FIL="hardware.txt"
	grep -A16 "^# /usr/bin/lscpu" $FIL
	grep -e "^model.name.*:" $FIL | sort -u
	grep -A5 "^vendor_id.*:.IBM/S390" $FIL
	FIL="boot.txt"
	grep -e "^\[.*Detected.*MHz.processor" -e "^\[.*Brought up.*CPUs" $FIL
	echo

	echo "memory:"
	FIL="boot.txt"
	grep "System RAM: " $FIL | sort -u

	FIL="basic-health-check.txt"
	grep -A 4 "# /usr/bin/free -k" $FIL | tail -3 
	echo

# TODO more specific patterns
#	FIL="hardware.txt"
#	for f in "iLO" "IPMI" "DRAC" "Emulex" "Brocade" "QLogic" "Broadcom" "MPT"; do
#		grep -e ${f} -e "pci.product.=" $FIL | sort -u
#	done

	echo "boards:"
	FIL="hardware.txt"
	grep "pci.product.=" $FIL |\
		 grep -v "USB.*HCI.*ontroller" | grep -v "PCI.*Port" |\
		 grep -v "I/O Hub" | grep -v "Integrated.Memory.Controller" | sort -u
	grep "scsi.vendor.=" $FIL | sort -u
	echo

	echo "disk:"
	FIL="fs-diskio.txt"
	grep -A1 "^Model:" $FIL | grep -v "^--$"
	awk '$0~/^Disk..dev.dasd/ {print $2,$3,$4}' $FIL
	echo

	echo "storage:"
	FIL="mpio.txt"
	grep "Vendor:.*Model:.*Rev:" $FIL | sort -u
	echo
	FIL="hardware.txt"
	for f in "supports.DPO.and.FUA" "support.DPO.or.FUA"; do
		echo -n "${f} : "
		grep -c ${f} $FIL
	done
	FIL="messages.txt"
	for f in "disabling.barriers"; do
		echo -n "${f} : "
		grep -c ${f} $FIL
	done
	echo

	echo "network_card:"
	FIL="network.txt"
	test -z "${NIC_PTRN}" &&\
	NIC_PTRN="
Model:
Device.File:
"
	# TODO better loop
	for f in $NIC_PTRN; do
		grep --binary-file=text -A20 -e "^..: PCI.*Ethernet controller$" -e "HiperSocket.controller$" $FIL |\
		grep $f
	done | sort -u | sed s/Device.File:/"NIC Device File:"/g
	echo	
}


function chk_modules(){

	echo_msgsep $FUNCNAME

	# tainted [ 4086.446776]
	grep -e "Modules.linked.in:.*(..)" -e "Modules.linked.in:.*(.)" boot.txt |\
		cut -b16- | sort -u 
	echo

	FIL="boot.txt"
	grep "^\[.*AppArmor:.*initialized" $FIL
	echo

	# TODO add Cisco HBA
	# TODO test HP eth  nx_nic
	# TODO watchdog ppc64 alim7101_wdt.ko?, s390x vmwatchdog.ko?
	test -z "${HARDW_MOD}" &&\
	HARDW_MOD="
ixgbe.ko
netxen_nic.ko
nx_nic.ko
bfa.ko
lpfc.ko
qlge.ko
qla2xxx.ko
bnx2.ko
bnx.ko
e1000.ko
tg3.ko
igb.ko
be2net.ko
be2iscsi.ko
vmxnet3.ko
vmw_balloon.ko
vmw_pvscsi.ko
cciss.ko
mptbase.ko
dm_multipath.ko
dm_mirror.ko
bonding.ko
hpwdt.ko
hangcheck-timer.ko
ibmasr.ko
iTCO_wdt.ko
cpu5wdt.ko
advantechwdt.ko
acquirewdt.ko
softdog.ko
vmwatchdog.ko
alim7101_wdt.ko
"
	FIL="boot.txt"
	for m in INITRD_MODULES MODULES_LOADED_ON_BOOT; do
		grep "^${m}=" $FIL
	done
	echo
	for m in $HARDW_MOD; do
		grep ${m} $FIL | sort -u
		n=$(basename ${m} .ko)
		grep "modprobe ${n}" $FIL
	done
	echo
# TODO see below - use split files to records instead of -A9, to find "supported:"
	FIL="modules.txt"
	for m in $HARDW_MOD; do
		grep -A9 "^filename:.*${m}" $FIL |\
		 awk '$1=="filename:" || $1=="license:" || $1=="supported:" {print $0}' 
	done
	echo
	for m in $HARDW_MOD; do
		n=$(basename ${m} .ko)
		grep "options ${n}" $FIL
	done
	echo
	FIL="modules.txt"
	split_file $FIL
	f=$(grep -l "^# /sbin/lsmod$" ${TMP}/${FIL}/xx*)
	test -z "$f" ||\
		grep -vh "^#==..Co.*==#" $f | sort -u | grep -v "#"
	rm -rf ${TMP}/${FIL}
	echo	
	echo -n "process on watchdog : "
        awk '$9=="/dev/watchdog" {print $1,$9}' open-files.txt
	echo
	echo
}


function chk_services(){

	echo_msgsep $FUNCNAME

	FIL="boot.txt"
	grep "exits with status" $FIL | grep -v "exits with status 0$"
 	grep "Skipped.features:" $FIL | sort -u
	echo
	grep "INIT: Entering runlevel:" $FIL | sort -u
	echo

	FIL="chkconfig.txt"
	for p in "3:on" "3:off"; do
		echo -n "${p} : "
		grep -c "${p}" $FIL
	done
	echo

# TODO precheck_for_sap
	test -z "${SVC_YES}" &&\
	SVC_YES="
boot.clock
boot.devicemapper
boot.lvm
boot.multipath
boot.sapconf
boot.sysstat
boot.sysctl
multipathd
ntp
"
	test -z "${SVC_OFF}" &&\
	SVC_OFF="
boot.dmraid
boot.md
alsasound
hp-asrd
ipmi
mdadmd
oracle
openais
powerd
slpd
smartd
sapinit
uuidd
"
	echo "service should be on:"
	for s in $SVC_YES; do
		grep "${s}:..on" $FIL
		grep "${s}:..off" $FIL
		grep "^${s}.*0:.*3:on" $FIL | grep -v "boot\."
		grep "^${s}.*0.*3:off" $FIL | grep -v "boot\."
	done
	echo
	echo "service should be off:"
	for s in $SVC_OFF; do
		grep "${s}:..off" $FIL
		grep "${s}:..on" $FIL
		grep "^${s}.*0.*3:off" $FIL | grep -v "boot\."
		grep "^${s}.*0.*3:on" $FIL | grep -v "boot\."
	done
	echo
}


function chk_veritas(){

	echo_msgsep $FUNCNAME

	echo "Veritas Storage Foundation and Veritas Cluster Services"
	echo
	grep -e "Modules.linked.in:.*(..)" -e "Modules.linked.in:.*(.)" boot.txt |\
		cut -b16- | sort -u
	echo

	test -z "${VRT_SVC_YES}" &&\
	VRT_SVC_YES="
boot.vxfs
qlremote
amf
isisd
gab
llt
mcelog
vcs
vras-vradmind.sh
vxatd
vxdbdctrl
vxdcli
vxfen
vxnm-vxnetd
vxodm
vxpbx_exchanged
vxrsyncd.sh
vxvm-boot
vxvm-reconfig
vxvm-recover
sfmh-discovery
xprtld
eccmad
"
	FIL="chkconfig.txt"
	for s in $VRT_SVC_YES; do
		grep "${s}:..on" $FIL
		grep "${s}:..off" $FIL
		grep "^${s}.*0:.*3:on" $FIL | grep -v "boot\."
		grep "^${s}.*0.*3:off" $FIL | grep -v "boot\."
	done
	echo
#
#VRT_MOD="
#vxfen
#gab
#vxodm
#llt
#dmpaa
#vxspec
#vxio
#vxdmp
#vxportal
#fdd
#vxfs
#"
#
	test -z "${VRT_ERR_PTRN}" &&\
	VRT_ERR_PTRN="
"GAB.*failed"
"Symantec.Technical.Support"
"error.*VRT"
"VCS.CRITICAL"
"vxvm.*volume.*not"
"
	LOG="boot.txt messages.txt"
	for e in ${VRT_ERR_PTRN}; do
		echo -n "${e} : "
		for f in ${LOG}; do
			cat $f
		done | grep -ic ${e} 
	done
	echo
	grep "^#./etc/vx" etc.txt
	grep "^#./etc/opt/VRT" etc.txt
	echo

	FIL="fs-diskio.txt"
	echo -n "VxDMP: "
	grep -c "VxDMP.*/devices/virtual/block/VxDMP" $FIL
	echo -n "VxVM:  "
	grep -c "VxVM.*/devices/virtual/block/VxVM" $FIL
	echo
}


function chk_network(){

	echo_msgsep $FUNCNAME
	
	test -z "${NETW_PTRN}" &&\
	NETW_PTRN="
"Setting.up.hostname"
"Setting.up.loopback.interface"
"eth.*IP.address:.*/"
"bond.*enslaved.interface:"
"bond.*IP.address:.*/"
"
	FIL="boot.txt"
	echo "boot setup:"
	for p in $NETW_PTRN; do
		grep $p $FIL
	done | sort -u
	echo "last setup:"
	FIL="network.txt"
	for p in $NETW_PTRN; do
		grep --binary-file=text $p $FIL
	done | sort -u
	echo
	grep --binary-file=text "eth.*\ device:" $FIL
	echo

	FIL="network.txt"
	( grep --binary-file=text -A3 "^# /sbin/ethtool bond" $FIL
	  grep --binary-file=text -A18 -e "^# /sbin/ethtool hsi" \
		-e "^#.*/sbin/ethtool sit" \
		-e "^#.*/sbin/ethtool eth" $FIL ) |\
		 grep -v "^#==\[ " | grep -v "^# /sbin/ethtool" | grep -v "^--$"
	echo
	
	# TODO summary num. NICs, num. "not connected"
	( grep --binary-file=text -A3 "^# /etc/sysconfig/network/routes$" $FIL ) |\
		 grep -v "^#==\[ " | grep -v "^# /sbin/ethtool"

	( grep --binary-file=text -A13 "^# /etc/sysconfig/network/ifcfg-bond" $FIL |\
		grep -v "=''$"
		echo
	grep --binary-file=text -A21 "^# /proc/net/bonding/bond" $FIL

	grep --binary-file=text -A10 \
		-e "^# /etc/sysconfig/network/ifcfg-qeth" \
		-e "^# /etc/sysconfig/network/ifcfg-eth" $FIL |\
		 grep -v "=''$" ) |\
		 grep -v "^#==\[ " 
	echo
	# TODO firewall
	echo "iptables:"	
	grep "iptable_" $FIL
	echo
}


function chk_nameres(){

	echo_msgsep $FUNCNAME

	FIL="network.txt"
	( grep --binary-file=text -A1 "^# /etc/HOSTNAME" $FIL
	grep --binary-file=text -A1 "^# /bin/hostname" $FIL
	grep --binary-file=text -A1 "^# /sbin/chkconfig nscd" $FIL ) |\
		 grep -v "^#==\[ " 
	echo
	split_file $FIL 

	echo "# /etc/hosts:"
	f=$(grep -l "^# /etc/hosts$" ${TMP}/${FIL}/xx*)
        test -z "$f" ||\
        	grep -v "^#" $f | tr -s "\n"
	echo
	echo "# /etc/resolv.conf:"
	f=$(grep -l "^# /etc/resolv.conf$" ${TMP}/${FIL}/xx*)
        test -z "$f" ||\
        	grep -v "^#" $f | tr -s "\n"
	echo
#	echo "# /etc/nsswitch.conf:"
#	f=$(grep -l "^# /etc/nsswitch.conf$" ${TMP}/${FIL}/xx*)
#	test -z "$f" ||\
#       	grep -v "^#" $f | tr -s "\n"
#	echo
	echo "# /etc/host.conf:"
	f=$(grep -l "^# /etc/host.conf$" ${TMP}/${FIL}/xx*)
        test -z "$f" ||\
        	grep -v "^#" $f | tr -s "\n"

        rm -rf ${TMP}/${FIL}
	echo

# TODO DNS
}


function chk_timeres(){

	echo_msgsep $FUNCNAME

	FIL="ntp.txt"
	echo "software:"
	for p in "^#.rpm.-V" "^#.Verification.Status:"; do
		grep ${p} $FIL
	done
	echo
	echo "service:"
	grep -A1 "^# /sbin/chkconfig" $FIL
	grep -A5 "# /usr/sbin/ntpq -p" $FIL | grep -v "====="

	echo "config:"
	echo "# /etc/ntp.conf:"
	split_file $FIL 
	f=$(grep -l "^# /etc/ntp.conf$" ${TMP}/${FIL}/xx*)
        test -z "$f" ||\
		grep -v "^#" $f | tr -s "\n"
	f=$(grep -l "^NTPD_FORCE_SYNC_ON_STARTUP=" ${TMP}/${FIL}/xx*)
        test -z "$f" ||\
		grep -v "^#" $f | tr -s "\n"

        rm -rf ${TMP}/${FIL}
	echo

	test -z "${TIME_ERR}" &&\
	TIME_ERR="
"synchronized.to.LOCAL"
"ntpd.*time.reset"
"ntpd.*failed"
"
	echo "errors:"
	for p in $TIME_ERR; do
		echo -n "${p} : "
		grep -c ${p} $FIL
	done
	echo

	echo "hwclock:"
	FIL="sysconfig.txt"
	test -z "${TIME_PTRN}" &&\
	TIME_PTRN="
HWCLOCK
SYSTOHC
TIMEZONE
DEFAULT_TIMEZONE
"
	for p in $TIME_PTRN; do
		grep "^${p}" $FIL | sort -u
	done
	echo

	FIL="etc.txt"
	grep -A4 "^/etc/adjtime" $FIL
	echo

# TODO www.novell.com/support/kb/doc.php?id=7009834 208 days uptime limit
# TODO elevenmin mode, ...
}


function chk_sleha(){

	echo_msgsep $FUNCNAME
 
	test -z "${CLUSTER_PKG}" &&\
	CLUSTER_PKG="
corosync
cluster-glue
crmsh
drbd-heartbeat
drbd-pacemaker
libopenais3
libcorosync4
libglue2
libpacemaker3
ldirectord
openais
pacemaker
pacemaker-mgmt
pacemaker-mgmt-client
resource-agents
sbd
"
	test -z "${CLUSTER_PRCS}" &&\
	CLUSTER_PRCS="
sbd:
/usr/sbin/corosync
/usr/lib64/heartbeat/stonithd
/usr/lib64/heartbeat/cib
/usr/lib64/heartbeat/lrmd
/usr/lib64/heartbeat/attrd
/usr/lib64/heartbeat/pengine
/usr/lib64/heartbeat/crmd
/usr/lib64/heartbeat/mgmtd
"
	FIL="rpm.txt"
	for f in $CLUSTER_PGK; do
		grep $f $FIL 	
	done | sort
	echo
	# TODO check /etc/corosync/
	# TODO check ???, see cs_sum_baseconfig, cs_precheck_for_sap
	# TODO check watchdogs, define $WATCHMOD, see cs_disbale_other_watchdog
	# hangcheck_timer here, not "-" as in hangcheck-timer.ko
	WATCHMOD="hpwdt hangcheck_timer iTCO ibmasr softdog"
	echo -n "loaded watchdog: "
	for m in $WATCHMOD; do
		awk '$1=="'${m}'" {print $1}' modules.txt | sort -u
		awk '$2=="'${m}'" {print $0}' boot.txt | sort -u
	done
	echo
        echo -n "process on watchdog: "
        awk '$9=="/dev/watchdog" {print $1,$9}' open-files.txt
        echo
	echo

	FIL="chkconfig.txt"
	split_file $FIL
	f=$(grep -l "^# /sbin/chkconfig.--list$" ${TMP}/${FIL}/xx*)
	echo "/sbin/chkconfig --list:"
	test -z $f ||\
		grep -e "^openais.*0:" -e "^hp-asrd.*0:" $f
	echo
	f=$(grep -l "^/etc/init.d/rc3.d:$" ${TMP}/${FIL}/xx*)
        echo "/etc/init.d/rc3.d:"
        test -z $f ||\
		grep -e "K..openais" -e "S..hp-asrd" $f
	echo
	f=$(grep -l "^/etc/init.d/rc5.d:$" ${TMP}/${FIL}/xx*)
        echo "/etc/init.d/rc5.d:"
        test -z $f ||\
		grep -e "K..openais" -e "S..hp-asrd" $f
	echo

	FIL="sysconfig.txt"
        split_file $FIL
        f=$(grep -l "^# /etc/sysconfig/openais$" ${TMP}/${FIL}/xx*)
	echo "/etc/sysconfig/openais:"
	test -z $f || grep -v "^#" $f | tr -s "\n"
	echo
        f=$(grep -l "^# /etc/sysconfig/pacemaker$" ${TMP}/${FIL}/xx*)
	echo "/etc/sysconfig/pacemaker:"
	test -z $f || grep -v "^#" $f | tr -s "\n"
	echo	
	f=$(grep -l "^# /etc/sysconfig/sbd$" ${TMP}/${FIL}/xx*)
	echo "/etc/sysconfig/sbd:"
	test -z $f || grep -v "^#" $f | tr -s "\n"
	echo

	FIL="ha.txt"
        split_file $FIL
	f=$(grep -l "^# /usr/sbin/sbd.-d.*dump$" ${TMP}/${FIL}/xx*)
	echo "sbd -d /dev/... dump:"
	test -z $f || grep -v "^#" $f
	echo
	f=$(grep -l "^# /usr/sbin/sbd.-d.*list$" ${TMP}/${FIL}/xx*)
	echo "sbd -d /dev/... list:"
	test -z $f || grep -v "^#" $f
	echo
        f=$(grep -l "^# /etc/corosync/corosync.conf$" ${TMP}/${FIL}/xx*)
	echo "/etc/corosync/corosync.conf:"
	test -z $f || grep -v "^#" $f
	echo
	f=$(grep -l "^# /usr/sbin/sbd.-d.*dump$" ${TMP}/${FIL}/xx*)
        f=$(grep -l "^# /usr/sbin/crm_mon.-r.-1$" ${TMP}/${FIL}/xx*)
	echo "crm_mon -r -1:"
	test -z $f || grep -v "^#" $f
	echo
        f=$(grep -l "^# /var/lib/heartbeat/crm/cib.xml$" ${TMP}/${FIL}/xx*)
	echo "/var/lib/heartbeat/crm/cib.xml:"
	test -z $f || grep -v "^#" $f
	echo
	# TODO crm configure show?

	FIL="basic-health-check.txt"
	echo "cluster processes running:"
	for p in $CLUSTER_PRCS; do 
		awk '$10=="'${p}'" {print $0}' $FIL
	done
	echo

	FIL="lvm.txt"
        split_file $FIL
        echo "/etc/lvm/lvm.conf:"
        f=$(grep -l "# /etc/lvm/lvm.conf" $TMP/$FIL/xx*)
        test -z "$f" || grep -vh "^#" $f | grep "filter"
        echo
	FIL="fs-softraid.txt"
        split_file $FIL
        echo "/etc/mdadm.conf:"
        f=$(grep -l "# /etc/mdadm.conf" $TMP/$FIL/xx*)
        test -z "$f" || grep -vh "^#" $f # | grep "/dev/null"
        echo

}


function chk_vmware(){

	echo_msgsep $FUNCNAME

	# http://www.vmware.com/download/packages.html
	# http://packages.vmware.com/tools
	# TODO what is really needed?
	test -z "${VMW_PKG}" &&\
	VMW_PKG="
ntp-4
vmware-tools-core
vmware-tools-esx
vmware-tools-esx-kmods-default
vmware-tools-esx-nox
vmware-tools-guestlib
vmware-tools-guestsdk
vmware-tools-hgfs
vmware-tools-libraries-nox
vmware-tools-libraries-x
vmware-tools-plugins-autoUpgrade
vmware-tools-plugins-deployPkg
vmware-tools-plugins-desktopEvents
vmware-tools-plugins-dndcp
vmware-tools-plugins-guestInfo
vmware-tools-plugins-hgfsServer
vmware-tools-plugins-powerOps
vmware-tools-plugins-resolutionSet
vmware-tools-plugins-timeSync
vmware-tools-plugins-unity
vmware-tools-plugins-vix
vmware-tools-plugins-vmbackup
vmware-tools-pvscsi-common
vmware-tools-pvscsi-kmp-default
vmware-tools-services
vmware-tools-thinprint
vmware-tools-user
vmware-tools-vmblock-common
vmware-tools-vmblock-kmp-default
vmware-tools-vmci-common
vmware-tools-vmci-kmp-default
vmware-tools-vmhgfs-common
vmware-tools-vmhgfs-kmp-default
vmware-tools-vmmemctl-common
vmware-tools-vmmemctl-kmp-default
vmware-tools-vmsync-common
vmware-tools-vmsync-kmp-default
vmware-tools-vmxnet-common
vmware-tools-vmxnet-kmp-default
vmware-tools-vmxnet3-common
vmware-tools-vmxnet3-kmp-default
vmware-tools-vsock-common
vmware-tools-vsock-kmp-default
vmware-tools-xorg-drv-display-
vmware-tools-xorg-drv-mouse
"
	test -z "${VMW_MODUL}" &&\
	VMW_MODUL="
vmxnet3
vmmouse
vmw_pvscsi
vmw_balloon
mptbase
"
	# TODO vmware tools services?
	test -z "${VMW_SVC_YES}" &&\
	VMW_SVC_YES="
boot.clock
boot.sysstat
boot.sysctl
ntp
vmware-tools-services
"
	test -z "${VMW_SVC_OFF}" &&\
	VMW_SVC_OFF="
alsasound
ipmi
irq_balancer
joystick
libvirtd
microcode.ctl
powerd
smartd
xend
xendomains
"
	test -z "${VMW_ETC}" &&\
	VMW_ETC="
"/etc/ntp.conf"
"/etc/vmware-tools/tools.conf"
"/etc/vmware-tools/plugins"
"/etc/pam.d/vmtoolsd"
"/etc/ld.so.conf.d/vmware-tools-libraries.conf"
"
	# TODO  vmware user /usr/lib/vmware-tools/moduleScripts/user/vmware-config.pl --toConfigure=user

	echo "- packages installed:"
	FIL="rpm.txt"
	for p in $VMW_PKG; do
		echo -n "$p : "
		# TODO awk only
		grep $p $FIL | awk '{print $1}'
	done
	echo
	echo
	echo "- modules loaded:"
	FIL="modules.txt"
	for p in $VMW_MODUL; do
		echo -n "$p : "
		grep $p $FIL	
	done
	echo
	# TODO common function for services, see chk_services, chk_sap, chk_oracle
	echo "- service should be on:"
	FIL="chkconfig.txt"
	for s in $VMW_SVC_YES; do
		grep "${s}:..on" $FIL
		grep "${s}:..off" $FIL
		grep "^${s}.*0:.*3:on" $FIL | grep -v "boot\."
		grep "^${s}.*0.*3:off" $FIL | grep -v "boot\."
	done
	echo
	echo "- service should be off:"
	for s in $VMW_SVC_OFF; do
		grep "${s}:..off" $FIL
		grep "${s}:..on" $FIL
		grep "^${s}.*0.*3:off" $FIL | grep -v "boot\."
		grep "^${s}.*0.*3:on" $FIL | grep -v "boot\."
	done
	echo
	echo -n "- vmtoolsd running: "
	FIL="basic-health-check.txt"
	awk '$10~/vmtoolsd/ {print $10}' $FIL
	echo
	echo "- config files:"	
	FIL="etc.txt"
	split_file $FIL
	for p in $VMW_ETC; do
		echo
		echo "$p :"
		f=$(grep -l "^# ${p}$" ${TMP}/${FIL}/xx*)
		test -z $f && continue
		test -r $f && grep -v "^#" $f | tr -s "\n"
	done
	echo
}


function chk_sap(){

	echo_msgsep $FUNCNAME

	# TODO check /etc/sysconfig/sapconf
	echo "sysconfig :"
	FIL=etc.txt
	split_file $FIL
	f=$(grep -l "^# /etc/sysconfig/sapconf$" ${TMP}/${FIL}/xx*)
	test -z "$f" ||\
		grep -v "^#" $f | sort -u
	echo
	# TODO check ???, see sum_base_config, precheck_for_sap
	grep -e "^@sapsys" -e "^@sdba" crash.txt
	echo
	echo "proc :"
	FIL=proc.txt
	split_file $FIL
	f=$(grep -l "^# /proc/net/unix$" ${TMP}/${FIL}/xx*)
	test -z "$f" ||\
		awk '$8~/sap/ || $8~/oracle/ {print $8}' $f | sort -u
	echo
	FIL=pam.txt
	split_file $FIL
# TODO
#	echo "pam_limits :"
#	f=$(grep -l "^# /etc/security/limits.conf$" ${TMP}/${FIL}/xx*)
#	test -z "$f" ||\
#		awk'$1=="oracle" || $1~/sapsys/ || $1~/dba/ {print}' $f | sort -u
#	echo
# TODO
#	echo "getent_passwd :"
#	f=$(grep -l "^# /usr/bin/getent passwd$" ${TMP}/${FIL}/xx*)
#	test -z "$f" ||\
#		awk -F: '$2~/SAP.*Administrator/ {print}' $f | sort -u	
#	echo
	echo "getent_group :"
	f=$(grep -l "^# /usr/bin/getent group$" ${TMP}/${FIL}/xx*)
	test -z "$f" ||\
		awk -F: '$1=="sapinst" || $1=="sapsys" || $1=="oper"  {print}' $f | sort -u
	echo
	echo "proc :"
	FIL=memory.txt
	split_file $FIL
	f=$(grep -l "^# /usr/bin/pmap [1-9]" ${TMP}/${FIL}/xx*)
	test -z "$f" ||\
		awk '$2=="sapstartsrv" || $2=="sapstart" || $2~/ms\.sap/ || $2~/en\.sap/  || $2~/ig\.sap/ || $2=="igsmux_mt" || $2=="igspw_mt" || $2=="gwrd" || $2=="icman" {print}' $f | sort -u
		awk '$8~/sapstart/ || $8~/disp\+work/ || $8~/igswd_mt/ || $8~/igspw_mt/ || $8~/igsmux_mt/ || $8~/gwrd/ || $8~/icman/ || $8~/dboraslib/ || $8~/libaio/ || $8~/libclntsh.so/ || $8~/dw_/  {print $8}' $f | sort -u
	echo

# TODO 1310037   SUSE LINUX Enterprise Server 11: Installation notes?
# TODO SAP Application Server Base, Minimal System (Appliances), X Window System, Print Server, C/C++ Compiler and Tools
# TODO no 32bit pattern 
# TODO no /etc/hosts 127.0.0.2   <hostname>.<domain> <hostname>
# TODO 1275776	Linux: Preparing SLES for SAP environments
# TODO /usr/sbin/uuidd         uuidd:uuidd     6755, chkstat --set /etc/permissions.local
# TODO 1557506 - Linux paging improvements
# The recommended value is 1/16 (~6%) of the amount of RAM, but not less than 512 MByte and not more than 4 GByte.
#<  8 GB:  512  (remommended min. limit)
#  8 GB:  512  (=  8 * 1024 MB / 16)
#  16 GB: 1024  (= 16 * 1024 MB / 16)
#  32 GB: 2048  (= 32 * 1024 MB / 16)
#  64 GB: 4096  (= 64 * 1024 MB / 16)
#> 64 GB: 4096  (recommended max. limit)
#vm.pagecache_limit_ignore_dirty
# If there is a lot of local writes and it is OK to throttle them by limiting the writeback caching, we recommended that you set the value to 0. If writing mainly happens to NFS filesystems, the default 1 should be left untouched.(A value of 2 would be a middle ground, not limiting local write back caching as much, but potentially resulting in some paging.)

}


function chk_saphana(){

	echo_msgsep $FUNCNAME

# TODO chk_sap for 1310037   SUSE LINUX Enterprise Server 11: Installation notes?
# TODO chk_sap for 1275776   prepare SLES ...?
# 1514967   SAP HANA: Central Note
# 1523337   SAP In-Memory Database 1.0: Central Note
# 1944799   SAP HANA Guidelines for SLES Operating System Installation
# 1824819   SAP HANA DB: Recommended OS settings for SLES 11 / SLES for SAP Applications 11 SP2
# 1954788   SAP HANA DB: Recommended OS settings for SLES 11 / SLES for SAP Applications 11 SP3
# 1855805   Recommended SLES 11 packages for HANA support on OS level
# 1890444   Slow HANA system due to CPU power save mode
# 1867783   XFS Data Inconsistency Bug with SLES 11 SP2
# 1888072   SAP HANA DB: Indexserver crash in __strcmp_sse42
# TODO 1846872   "No space left on device" error reported from HANA
# TODO 1681360 - Support for SAP software in Linux / Fusion-IO
# 784391	SAP support terms and 3rd-party Linux kernel drivers
# 171356	SAP software on Linux: General information
# TID 7011982 cstate
# TODO krb5-32bit needs 32bit pattern 

	test -z "${HANA_SAP_BOOTARG}" &&\
	HANA_SAP_BOOTARG="
intel_idle.max_cstate=0
processor.max_cstate=0
"
	test -z "${HANA_SUSE_BOOTARG}" &&\
	HANA_SUSE_BOOTARG="
elevator=noop
cgroup_disable=memory
"
# TODO should sysctl go to chk_sap?
	test -z "${HANA_SAP_SYSCTL}" &&\
	HANA_SAP_SYSCTL="
vm.pagecache_limit_mb
vm.pagecache_limit_ignore_dirty
"
	test -z "${HANA_SUSE_SYSCTL}" &&\
	HANA_SUSE_SYSCTL="
vm.dirty_bytes
vm.dirty_background_bytes
"
	test -z "${HANA_VERS_MIN}" &&\
	HANA_VERS_MIN="
kernel-default:3.0.101-0.7.15.1
kernel-xen:3.0.101-0.7.15.1
glibc:2.11.3-17.56.2
"
	test -z "${HANA_SAP_PKG_YES}" &&\
	HANA_SAP_PKG_YES="
gtk2
java-1_6_0-ibm
libicu
mozilla-xulrunner192
ntp
sudo
syslog-ng
tcsh
libssh2-1
expect
autoyast2-installation
yast2-ncurses
bing
bonnie
cairo
findutils-locate
graphviz
iptraf
krb5-32bit
krb5-client
nfs-client
sensors
"

# TODO Fusion IO driver RPM?
	test -z "${HANA_SUSE_PKG_YES}" &&\
HANA_SUSE_PKG_YES="
rear
sysstat
supportutils
sapconf
xfsprogs
"

	FIL="rpm.txt"
	echo "RPM with specific version recommended by SAP:"
	for f in $HANA_VERS_MIN; do
		FULL=$(echo $f | awk -F: '{print $1"-"$2}')
		BASE=$(echo $f | awk -F: '{print $1}')
		echo -n "$FULL : "
		q="$FULL"
		# TODO do search, if package is -32bit
		grep -v -- "-32bit" $FIL | grep "^${FULL}" ||\
		 	q=$(	p=$(grep -v -- "-32bit" $FIL |\
					grep  "^${BASE}-[0-9]")
				echo "NO but $p"
			)
		echo "${q}"
	done | awk '{print $1,":",$3,$4,$5}' | sort
	echo
	FIL="boot.txt"
	grep -A1 "^# /bin/uname -a" $FIL
	echo

	FIL="rpm.txt"
	echo "RPMs recommended by SAP:"
	for f in $HANA_SAP_PKG_YES; do
		echo -n "$f : "
		# TODO do search, if package is -32bit
		grep -v -- "-32bit" $FIL | grep "^${f}-[0-9]" || echo "NO"
	done | awk '{print $1,":",$3}' | sort
	echo
	echo "RPMs recommended by SUSE:"
	for f in $HANA_SUSE_PKG_YES; do
		echo -n "$f : "
		# TODO do search, if package is -32bit
		grep -v -- "-32bit" $FIL | grep "^${f}-[0-9]" || echo "NO"
	done | awk '{print $1,":",$3}' | sort
	echo

	FIL="boot.txt"
	KRNLCMD=$(grep -A1 "^# /proc/cmdline" $FIL | tail -1)
	echo "SAP recommended /proc/cmdline args:"
	# TODO shorter output
	for f in $HANA_SAP_BOOTARG; do
		echo -n "$f : "
		p="NO"
		q=$( echo $KRNLCMD | grep "$f" ) && p=$q
		echo $p
	done
	echo
	echo "SUSE recommended /proc/cmdline args:"
	# TODO shorter output
	for f in $HANA_SUSE_BOOTARG; do
		echo -n "$f : "
		p="NO"
		q=$( echo $KRNLCMD | grep "$f" ) && p=$q
		echo $p
	done
	echo
	echo "Current /proc/cmdline:"
	echo "$KRNLCMD"
	echo
	
	FIL="sysconfig.txt"

	split_file $FIL

	hana_defapd=$(echo "${HANA_SAP_BOOTARG}" "${HANA_SUSE_BOOTARG}" | tr -d "\n")
	echo "SAP+SUSE recommended DEFAULT_APPEND: $hana_defapd"
	echo "# /etc/sysconfig/bootloader:"
	f=$(grep -l "^# /etc/sysconfig/bootloader$" ${TMP}/${FIL}/xx*)
	test -z "$f" ||\
		grep DEFAULT_APPEND $f
	echo
	rm -rf ${TMP}/${FIL}

	# TODO HANA_SAP_SYSCTL, HANA_SUSE_SYSCTL as SAP_SYSCTL in chk_sap? 
	FIL="etc.txt"

	split_file $FIL

	f=$(grep -l "^# /etc/sysctl.conf$" ${TMP}/${FIL}/xx*)
	echo "SAP recommended sysctl:"
	test -z "$f" ||\
		for p in $HANA_SAP_SYSCTL; do
			echo -n "$p : "
			grep $p $f || echo "NO"
		done
	echo
	echo "SUSE recommended sysctl:"
	test -z "$f" ||\
		for p in $HANA_SUSE_SYSCTL; do
			echo -n "$p : "
			grep $p $f || echo "NO"
		done
	echo
	rm -rf ${TMP}/${FIL}

	# TODO /boot/grub/menu.lst
	
	FIL="boot.txt"
	
	split_file $FIL

	# TODO
	echo "SAP recommended boot.local: echo never > /sys/kernel/mm/transparent_hugepage/enabled"
	echo "# /etc/init.d/boot.local:"
	F=$(grep -l "# /etc/init.d/boot.local" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')	
		test -z "$F" ||\
			grep -v "^#" $F

	FIL="sysfs.txt"
	echo "SAP recommeded nr_anon_transparent_hugepages: 0 , nr_hugepages: 0"
	grep "nr_anon_transparent_hugepages " $FIL | grep -v -- "--" 
	grep -A1 "# /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" $FIL | grep -v -- "--"
	
# TODO fix it: csplit runs on 100% CPU for ever
#	split_file $FIL
#
#	echo "SAP recommeded nr_anon_transparent_hugepages: 0 , nr_hugepages: 0"
#	# TODO n NUMA nodes
#	F=$(grep -l "# /sys/devices/system/node/node0/vmstat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node1/vmstat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node2/vmstat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node3/vmstat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node4/vmstat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node5/vmstat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node6/vmstat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node7/vmstat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node8/vmstat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#
#	F=$(grep -l "# /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			cat $F
	echo
	echo "SAP recommended cpuidle: acpi_idle"
	grep -A1 "# /sys/devices/system/cpu/cpuidle/current_driver" sysfs.txt
	echo

	FIL="fs-diskio.txt"

	split_file $FIL
	
	echo "SAP recommeded filesystem type: XFS"
	F=$(grep -l "# /etc/fstab" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
	grep -v "#==" $F | tr -s " "
	# TODO XFS create options

	# TODO calculate 2x RAM, max.50GB
	echo "SAP recommended swap space: 2x RAM, max.50GB"
	# TODO depends on name "swap"
	grep "Disk.*swap" $FIL

	# TODO search Fusion IO drives 
	# TODO boot msg shows max.4GB, use other info here	
	FIL="boot.txt"
	grep "^\[.*System RAM: " $FIL | awk '{print $11,$12,$13}'

}


function chk_oracle(){

	echo_msgsep $FUNCNAME

	# TODO put this in config file, use it in function
	# TODO see /srv/pinne/work/*/sap/technical/from_*/20130304/ps_*.txt
# orawe3    4637  0.0  2.4 10495496 3260024 ?    Ss   Mar03   3:21 oracleWE3 (LOCAL=NO)
# orawei    5038  0.0  0.0 10489296 19884 ?      Ss   Feb24   0:21 ora_q001_WEI
# orawe3   19399  0.0  0.0 10491972 29064 ?      Ss   Mar03   1:20 ora_pmon_WE3
# orawe3   19402  0.0  0.0 10489520 17080 ?      Ss   Mar03   0:58 ora_psp0_WE3
# orawe3   20152  0.0  0.0 10489520 16560 ?      Ss   Mar03   0:27 ora_diag_WE3
# orawe3   20175  0.0  0.0 10490032 41084 ?      Ss   Mar03   0:14 ora_dbrm_WE3
# orawe3   20191  0.2  0.0 10494128 44176 ?      Ss   Mar03  14:36 ora_dia0_WE3
# orawe3   20229  0.0  0.0 10489520 84100 ?      Ss   Mar03   0:13 ora_mman_WE3
# orawe3   20271  0.0  2.7 10496556 3600380 ?    Ss   Mar03   3:38 ora_dbw0_WE3
# orawet   20296  0.3  1.4 4977156 1958052 ?     Ss   Feb27  35:23 oracleWET (LOCAL=NO)
# orawe3   20310  0.0  2.7 10495532 3635364 ?    Ss   Mar03   3:36 ora_dbw1_WE3
# orawe3   20331  0.0  2.8 10496044 3794112 ?    Ss   Mar03   3:36 ora_dbw2_WE3
# orawe3   20357  0.0  2.7 10495532 3617344 ?    Ss   Mar03   3:34 ora_dbw3_WE3
# orawe3   20381  0.0  2.7 10496044 3653224 ?    Ss   Mar03   3:35 ora_dbw4_WE3
# orawe3   20402  0.6  0.0 10512176 39104 ?      Ss   Mar03  35:23 ora_lgwr_WE3
# orawe3   20422  0.0  0.1 10490536 210024 ?     Ss   Mar03   2:25 ora_ckpt_WE3
# orawe3   20439  0.0  0.7 10491064 925884 ?     Ss   Mar03   0:37 ora_smon_WE3
# orawe3   20459  0.0  0.0 10490032 35476 ?      Ss   Mar03   0:07 ora_reco_WE3
# orawe3   20488  0.0  0.2 10495560 352076 ?     Ss   Mar03   0:34 ora_mmon_WE3
# orawe3   20523  0.1  0.1 10489520 150708 ?     Ss   Mar03   7:05 ora_mmnl_WE3
# orawe2   25496  0.0  0.0  83832 12588 ?        Ssl  Feb24   0:25 /oracle/WE2/112_64/bin/tnslsnr listener_WE2 -inherit
# root     25625  4.7  0.0   6240  1176 ?        S    16:40   0:00 /oracle/WE3/libelle/trd/bin/trd
# orawe3   25926  0.0  0.0 10489512 20492 ?      Ss   Mar03   0:17 ora_smco_WE3
# orawe3   26972  0.0  0.0 10491052 33200 ?      Ss   Mar03   0:07 ora_q000_WE3
# orawe2   27280  0.0  0.0 6885040 17004 ?       Ss   Feb24   2:46 ora_psp0_WE2
# orawe2   31816  0.0  0.0 6885040 20592 ?       Ss   Feb24   0:25 ora_qmnc_WE2
	# 
        ORA_PRCS="
ora_pmon_
ora_vktm_
ora_gen._
ora_diag_
ora_dbrm_
ora_dia._
ora_dbw._
ora_lgwr_
ora_ckpt_
ora_smon_
ora_reco_
ora_mmon_
ora_mmnl_
ora_smco_
ora_psp._
ora_q.*_
ora_qmnc_
oracle.*
/.*/bin/tnslsnr
"

	echo "standalone Oracle installation"
	echo
	echo "sysconfig_oracle :"
        FIL="sysconfig.txt"
        split_file $FIL
        F=$(grep -l "# /etc/sysconfig/oracle" $TMP/$FIL/xx* |\
		 tr -d ":#" | awk '{print $1}')
        grep -v "^#" $F  | awk -F: '{print $2}' | tr -s "\n "
        echo

	grep -e "^oracle" -e "^@dba" crash.txt
	echo
	grep "^tmpfs" basic-health-check.txt	
	grep "^vm.nr_hugepages.=" env.txt	
	echo
	# TODO check ???, see sum_base_config, whbsaprecheck
	# TODO check async-IO, direct-IO
	# TODO hugetblfs from mount
	# TODO oracleasmlib oracleasm-support still needed?	

	FIL="rpm.txt"
        test -z "${ORA_SFT_PTRN}" &&\
        ORA_SFT_PTRN="
orarun
oracleasmlib oracleasm-support oracleasm-kmp-default
oracle-instantclient11.2-basic
oracle-instantclient11.2-basiclite
oracle-instantclient11.2-jdbc
oracle-instantclient11.2-sqlplus
oracle-instantclient11.2-devel
oracle-instantclient11.2-odbc
oracle-instantclient11.2-tools
cups-client cups-drivers cups-libs cups-libs-32bit cups cups-autoconfig
xorg-x11-libs xorg-x11
nfs-client
glibc glibc-32bit glibc-locale glibc-locale-32bit glibc-devel
kernel-default linux-kernel-headers
binutils gcc gcc-32bit gcc43 gcc-locale gcc-locale libgcc43 libgcc43-32bit
libaio libaio-devel
sysstat supportutils suse-sam
multipath-tools mdadm device-mapper device-mapper-32bit libext2fs2
ClusterTools2
"
	echo "RPM needed :"
        for p in $ORA_SFT_PTRN; do
                echo ${p}
        done
	echo
	echo "RPM found :"
        for p in $ORA_SFT_PTRN; do
                awk '$1=="'${p}'" {print $0}' $FIL
        done
        echo
# TODO
	test -z "${ORA_SVC_YES}" &&\
	ORA_SVC_YES="
boot.clock
boot.dmraid
boot.md
boot.devicemapper
boot.lvm
boot.multipath
boot.sysstat
boot.sysctl
multipathd
mdadmd
ntp
oracle
"
	test -z "${ORA_SVC_OFF}" &&\
	ORA_SVC_OFF="
boot.sapconf
alsasound
ipmi
openais
powerd
slpd
smartd
sapinit
"
	FIL="chkconfig.txt"
	echo "service needed :"
	for s in $ORA_SVC_YES; do
		echo $s
	done
	echo
	echo "service found :"
	for s in $ORA_SVC_YES; do
		grep "${s}:..on" $FIL
		grep "${s}:..off" $FIL
		grep "^${s}.*0:.*3:on" $FIL | grep -v "boot\."
		grep "^${s}.*0.*3:off" $FIL | grep -v "boot\."
	done
	echo
	echo "service should be off :"
	for s in $ORA_SVC_OFF; do
		grep "${s}:..off" $FIL
		grep "${s}:..on" $FIL
		grep "^${s}.*0.*3:off" $FIL | grep -v "boot\."
		grep "^${s}.*0.*3:on" $FIL | grep -v "boot\."
	done
        echo
	# TODO rpm.txt /usr/local/bin: coraenv, dbhome, oraenv
	echo "proc :"
	FIL=memory.txt
	split_file $FIL
	f=$(grep -l "^# /usr/bin/pmap [1-9]" ${TMP}/${FIL}/xx*)
	test -z "$f" ||\
		awk '$8~/dboraslib/ || $8~/libaio/ || $8~/libclntsh.so/ || $8~/tnslsnr/ || $8~/libodmd11/ || $8~/oracle$/ {print $8}' $f | sort -u
	echo
	# TODO oracle user, pfile
	# TODO memory.txt: tnslsnr libclntsh oracle$ libodmd11
	# TODO env.txt: ORA_
	FIL=env.txt
	split_file $FIL
	f=$(grep -l "^# /usr/bin/env$" ${TMP}/${FIL}/xx*)
	test -z "$f" ||\
		awk '$1~/ORA/ {print}' $f | sort -u
	echo
}


function chk_kernel(){
	
	echo_msgsep $FUNCNAME

	FIL="boot.txt"
	grep -A1 "^# /bin/uname -a" $FIL
	grep -A1 "^# /proc/cmdline" $FIL

	grep "^\[.*Booting paravirtualized kernel on bare hardware" $FIL

	echo -n "kernel.tainted : "
	grep "^kernel.tainted.=" env.txt
	echo
}


function chk_software(){

	echo_msgsep $FUNCNAME
	
	FIL="basic-environment.txt"

	grep -A3 "# /etc/SuSE-release" $FIL
	echo
	grep -A1 "# /etc/lsb-release" $FIL
	echo	

	FIL="sam.txt"
	grep -A4 "\.prod:" $FIL
	echo

	FIL="rpm.txt"
	grep "(none)" $FIL | grep -v "gpg-pubkey"
	echo	

	FIL="sam.txt"
	test -z "${SOFTW_PTRN}" &&\
	SOFTW_PTRN="
System.architecture:
Baseproduct:
not.our.package:
"	
	for p in $SOFTW_PTRN; do
		grep "${p}" $FIL
	done
	echo
}


function chk_updates(){

	echo_msgsep $FUNCNAME
	
	FIL="updates.txt"
	# TODO sles-for-sap
	# TODO fix doubled entries artefact "SLES11-*"
	test -z "${UPDT_PTRN}" &&\
	UPDT_PTRN="
"SLES11-"
"SLES11-SP"
"SLE11-HAE-SP"
"SLE11-SP1-SAP-AIO-"
"SLE11-SP2-SAP-"
"SLE11-SP3-SAP-"
"
	for p in ${UPDT_PTRN}; do
		grep "|.*|.${p}.*|.*|" $FIL | sort -u
	done
	echo
	grep "patches.needed" $FIL
	echo
}


function chk_multipath(){

	echo_msgsep $FUNCNAME

	FIL="mpio.txt"

	split_file $FIL

	F=$(grep -l "# /etc/multipath.conf" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
	echo "# /etc/multipath.conf:"
	test -z "$F" || grep -v "^#" $F | tr -s "\n"

	for p in "^#.rpm.-V" "^#.Verification.Status:"; do
		grep "${p}" $FIL
	done
	grep "^#.RPM.Not.Installed:" $FIL
	echo

	for p in "^boot.device-mapper.*o" "^boot.multipath.*o" "^multipathd.*0:.*3:on"; do
		grep ${p} $FIL
	done
	echo
	
	grep -B1 "^size=.*features=.*hwhandler=" $FIL |\
		sed s/^size/"	size"/g | grep -v "^--$"
	echo
	# TODO by-id
	# TODO dmsetup
	# TODO show complete multipath.conf
	# TODO multipath.conf: compare failback, rr_min_io, rr_min_io_rq, no_path_retry, polling_interval, dev_loss_tmo, user_friendly_names, async???
	# TODO check for un-matched userfriendly names (mpath[a-z])
	for p in "loading.*prioritizer" "path.checker.=" "prio.="; do
		grep ${p} $FIL
	done | awk '{print $6,$7,$8}' | sort -u
	echo
}
		

function chk_filesys(){
	
	echo_msgsep $FUNCNAME
	
	FIL="fs-diskio.txt"
	
	split_file $FIL 
	
	F=$(grep -l "# /etc/fstab" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
	grep -v "#==" $F | tr -s " "
	F=$(grep -l "# /bin/mount" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
	grep -v "#==" $F | tr -s " "
	# TODO define meaningful limits for await and svctm	
	# TODO make limits for await and svctm a config parameter
	# TODO autofs	
	F=$(grep "# /usr/bin/iostat" $TMP/$FIL/xx* | tr -d ":#" | awk '{print $1}')
	echo $F | awk ' $1=="Device:" {print $0}
		$10 > 100.0 {print $0}
		$11 > 20.0 {print $0}' | sort -u
	echo
}


function chk_storage(){
	
	echo_msgsep $FUNCNAME

	FIL="rpm.txt"
	# TODO see whbsaprecheck
	test -z "${STOR_PKG}" &&\
	STOR_PKG="
dmraid
lvm2
libext2fs2
mdadm
multipath-tools
nfs-client
open-iscsi
ocfs2-tools
ocfs2-kmp-default
xfsprogs
"
	echo "software:"
	for p in $STOR_PKG; do
		awk '$1=="'$p'" {print $1,$NF} ' $FIL
	done
	echo
	# TODO check if local disk is in multipath
	# TODO nfs
	test -z "${STOR_MOD}" &&\
	STOR_MOD="
ext2
ext3
ocfs2
xfs
"	
	echo "disk:"
	FIL="fs-diskio.txt"
	#grep -B1 "^Disk./dev/" $FIL
	grep -A1 "^Model:" $FIL | grep -v "^--$"
	echo

	# TODO FIL="fs-softraid.txt"
	# TODO better elevator guessing 
	FIL="boot.txt"
	echo "/boot/grub/menu.lst:"
	grep "io.scheduler.*registered.(default)" $FIL
	grep "kernel.*elevator" $FIL
	echo
	FIL="sysconfig.txt"
	echo "/etc/sysconfig/bootloader:"
	grep "DEFAULT_APPEND.*elevator" $FIL
	echo

	split_file $FIL
	echo "/etc/sysconfig/mdadm:"
	f=$(grep -l "# /etc/sysconfig/mdadm" $TMP/$FIL/xx*)
        test -z "$f" || grep -v "^#" $f | tr -s "\n"
        echo
	
	FIL="fs-softraid.txt"
	split_file $FIL
	echo "/etc/sysconfig/mdadm:"
	f=$(grep -l "# /etc/mdadm.conf" $TMP/$FIL/xx*)
        test -z "$f" || cat $f | tr -s "\n"
        echo
	echo "/proc/mdstat:"
	f=$(grep -l "# /proc/mdstat" $TMP/$FIL/xx*)
        test -z "$f" || cat $f | tr -s "\n"
        echo

	FIL="lvm.txt"
	split_file $FIL
	echo "/sbin/vgs:"
        f=$(grep -l "# /sbin/vgs" $TMP/$FIL/xx*)
	test -z "$f" || grep -vh "^#" $f | tr -s "\n"
	echo
	echo "/etc/sysconfig/lvm:"
        f=$(grep -l "# /etc/sysconfig/lvm" $TMP/$FIL/xx*)
	test -z "$f" || grep -vh "^#" $f | tr -s "\n"
	echo
	echo "/etc/lvm/lvm.conf:"
	f=$(grep -l "# /etc/lvm/lvm.conf" $TMP/$FIL/xx*)
	test -z "$f" || grep -vh "^#" $f | grep -vh "^    #" | tr -s "\n"
	echo
}


function chk_overview(){
	
	echo_msgsep $FUNCNAME

	#FIL="supportconfig.txt"
	#grep "Script.Version:" $FIL
	FIL="network.txt"
	echo -n "Hostname: "
	grep --binary-file=text -A1 "#./etc/HOSTNAME" $FIL | tail -1
	FIL="basic-health-report.txt"
	echo "Supportconfig: " $DIR
	for p in "^Date.Checked:" "^Status:" "\[...Red...\]" "\[.Yellow..\]"; do
		grep ${p} $FIL
	done
	echo
}


function chk_sysconf(){

	echo_msgsep $FUNCNAME

	SYSC_PTRN="
fs.file-max
fs.aio-max-nr
kernel.panic_on_oops
kernel.panic
kernel.sem
kernel.shmall
kernel.shmmax
kernel.shmmni
kernel.threads-max
net.ipv4.ip_local_port_range
net.core.rmem_default
net.core.rmem_max
net.core.wmem_default
net.core.wmem_max
vm.dirty_*
vm.zone_reclaim_mode
vm.numa_zonelist_order
vm.nr_pdflush_threads
vm.min_free_kbytes
vm.nr_hugepages
vm.swappiness
vm.overcommit_memory
vm.max_map_count
vm.hugetlb_shm_group
vm.pagecache_limit_*
"
	# TODO get really active settings
#	FIL="proc.txt"
#	PROC_PTRN=$(echo $SYSC_PTRN | tr "." "/" )
	FIL="env.txt"
	PROC_PTRN=$SYSC_PTRN
	echo "proc : "
	for m in ${PROC_PTRN}; do
#		grep -A1 "${m}" $FIL
		grep "^${m}" $FIL
	done | tr -s " " | tr "	" " " | sort -u
	echo
	FIL="etc.txt"
	echo "sysctl.conf : "
	for m in ${SYSC_PTRN}; do	
		grep ^$m $FIL
	done | sort -u
	echo
	# TODO security/limits.conf
	# TODO tmpfs
	FIL="env.txt"
	split_file $FIL
	F=$(grep -h "# ulimit -a" $TMP/$FIL/xx* | tr -d ":#")
        echo $F | grep -v "#==" | tr -s " "

	FIL="sysconfig.txt"
	split_file $FIL
	F=$(grep -l "# /etc/sysconfig/ulimit" $TMP/$FIL/xx* | tr -d ":#" |\
		 awk '{print $1}')
	# TODO make it work:
	# test -r $F && grep -v "^#" $F | tr -s "\n "
	echo

	# TODO /sys/kernel/mm/transparent_hugepage/enabled
	# TODO /sys/kernel/mm/transparent_hugepage/defrag

	# TODO how to find scheduler from supportdonfig? plugin needed?
	FIL="boot.txt"
	echo -n "io_scheduler: "; grep "io scheduler.*registered..default" $FIL
	echo
}


function chk_sysload(){

	echo_msgsep $FUNCNAME
	
	FIL="basic-health-report.txt"
	echo "basic-health-report:"
	grep "^[A-Z].*\[....e....\]$" $FIL | grep -v "Kernel.Taint"
	# TODO make dfull work
	dfull="X"
	dfull=$( grep "%.>=.90%" $FIL )
	test "${dfull}" = "X" &&\
	(	echo "Used Disk Space:"
		echo $dfull
	)
	echo

	OWD=$PWD
	cd sar || exit
	# TODO print date of sar file to each output line
	#	either by adding date from sar-fiel header,
	#	or by grep-ing the alreday known maxima out of the files
	# TODO reduce number of columns

	echo "max. load-avg15:"
	echo "hh:mm:ss xM   runq-sz  plist-sz   ldavg-1   ldavg-5  ldavg-15"
	for f in sa??; do
		sar -q -f $f
	done | grep "^[0-9][0-9]:" | sort -n -k7 | tail

	echo "max. user CPU utilisation:"
	echo "hh:mm:ss xM     CPU     %user     %nice   %system   %iowait    %steal     %idle"
	for f in sa??; do
		sar -f $f
	done | grep "^[0-9][0-9]:" | grep -v "%" | sort -n -k4 | tail

#	echo "max. system CPU utilisation:"
#	echo "hh:mm:ss xM     CPU     %user     %nice   %system   %iowait    %steal     %idle"
#	for f in sa??; do
#		sar -f $f 
#	done | grep "^[0-9][0-9]:" | grep -v "%" | sort -n -k6 | tail

	echo "max. iowait CPU utilisation:"
	echo "hh:mm:ss xM     CPU     %user     %nice   %system   %iowait    %steal     %idle"
	for f in sa??; do
		sar -f $f
	done | grep "^[0-9][0-9]:" | grep -v "%" | sort -n -k7 | tail

#	echo "max. interrupts:"
#	echo "hh:mm:ss xM       INTR    intr/s"
#	for f in sa??; do
#		sar -I SUM -f $f
#	done | grep "^[0-9][0-9]:" | sort -n -k4 | tail

#	echo "max. disk write tps:"
#	echo "hh:mm:ss xM       tps      rtps      wtps   bread/s   bwrtn/s"
#	for f in sa??; do
#		sar -f $f 
#	done | grep "^[0-9][0-9]:" | sort -n -k5 | tail

	echo "max. disk write blocks:"
	echo "hh:mm:ss xM       tps      rtps      wtps   bread/s   bwrtn/s"
	for f in sa??; do
		sar -f $f 
	done | grep "^[0-9][0-9]:" | sort -n -k7 | tail

#	echo "max. disk read tps:"
#	echo "hh:mm:ss xM       tps      rtps      wtps   bread/s   bwrtn/s"
#	for f in sa??; do
#		sar -f $f 
#	done | grep "^[0-9][0-9]:" | sort -n -k4 | tail

	echo "max. disk read blocks:"
	echo "hh:mm:ss xM       tps      rtps      wtps   bread/s   bwrtn/s"
	for f in sa??; do
		sar -f $f 
	done | grep "^[0-9][0-9]:" | sort -n -k6 | tail

	echo "max. swap used:"
	echo "hh:mm:ss xM kbswpfree kbswpused  %swpused  kbswpcad   %swpcad"
	for f in sa??; do
		sar -f $f 
	done | grep "^[0-9][0-9]:" | sort -n -k4 | tail

	echo "max. pageout:"
	echo "hh:mm:ss xM pgpgin/s pgpgout/s   fault/s  majflt/s  pgfree/s pgscank/s pgscand/s pgsteal/s    %vmeff "
	for f in sa??; do
		sar -B -f $f 
	done | grep "^[0-9][0-9]:" | sort -n -k4 | tail

#	# find lowest vmeff% that is not 0.00. 0.00 means "nothing done".
#	echo "min. vmeff:"
#	echo "hh:mm:ss xM pgpgin/s pgpgout/s   fault/s  majflt/s  pgfree/s pgscank/s pgscand/s pgsteal/s    %vmeff "
#	for f in sa??; do
#		sar -B -f $f 
#	done | grep "^[0-9][0-9]:" | grep -v "%" |\
#	 	awk '$11 != "0.00" {print}' | sort -n -k11 | head

	# TODO sum up buffer+cache, estimated use memory 
	echo "max. memory commit:"
	echo "hh:mm:ss xM kbmemfree kbmemused  %memused kbbuffers  kbcached  kbcommit   %commit"
	for f in sa??; do
		sar -r -f $f 
	done | grep "^[0-9][0-9]:" | sort -n -k8 | tail

	echo "max. memory cache:"
	echo "hh:mm:ss xM kbmemfree kbmemused  %memused kbbuffers  kbcached  kbcommit   %commit"
	for f in sa??; do
		sar -r -f $f 
	done | grep "^[0-9][0-9]:" | sort -n -k7 | tail

#	echo "max. network write tps:"
#	echo "hh:mm:ss xM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s"
#	for f in sa??; do
#		sar -n DEV -f $f 
#	done | grep "^[0-9][0-9]:" | grep bond | sort -n -k5 | tail

	echo "max. network write kB:"
	echo "hh:mm:ss xM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s"
	for f in sa??; do
		sar -n DEV -f $f 
	done | grep "^[0-9][0-9]:" | grep bond | sort -n -k7 | tail

#	echo "max. network read tps :"
#	echo "hh:mm:ss xM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s"
#	for f in sa??; do
#		sar -n DEV -f $f 
#	done | grep "^[0-9][0-9]:" | grep bond | sort -n -k4 | tail

	echo "max. network read kB:"
	echo "hh:mm:ss xM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s"
	for f in sa??; do
		sar -n DEV -f $f 
	done | grep "^[0-9][0-9]:" | grep bond | sort -n -k6 | tail

	# TODO network errors?
	# TODO NFS client and server reports?
	# TODO grep -A6 "numastat" sysctl.txt 

	cd $OWD 
	echo
}


function chk_errors(){
	
	echo_msgsep $FUNCNAME

	# TODO see cs_show_error_patterns
	test -z "${ERRCFG}" &&\
		ERRCFG="/etc/ClusterTools2/cs_show_error_patterns"
	test -s $ERRCFG && source $ERRCFG
 
	# TODO taint.flag ?
	echo -n "kernel.tainted : "
	grep "^kernel.tainted.=" env.txt

	# TODO add plugin-messages.txt, man supportconfig for plugin
	# TODO find start and end time for each log files

	FIL="messages.txt"
	FRST=$(grep -A1 "# /var/log/messages - Last.*Lines" $FIL |\
		tail -1 | awk '{print $1,$2,$3}')
	echo "$FIL starts $FRST"
	
	LOG="boot.txt messages.txt proc.txt ha.txt"
	for e in ${ERROR_PATTERN} ${CLUERR_PATTERN}; do
		echo -n "${e} : "
		for f in ${LOG}; do
			cat $f
		done | grep -ic ${e} 
	done
	echo

	# TODO dropped packets from proc.txt
	echo "dropped network packets:"
	FIL="proc.txt"
	split_file $FIL
	f=$(grep -l "^# /proc/net/dev$" ${TMP}/${FIL}/xx*)
	test -z "$f" ||\
		cat $f

NET_ERR="
ICMP.message.failed
packets.pruned.from.receive.queue
delayed.acks
listen.queue.of.a.socket.overflowed
"
	FIL="network.txt"
	for p in $NET_ERR; do
		grep $p $FIL
	done
	echo
}


# main()

case $1 in
	--help|-h)
		BEX=$(basename $EXE)
		echo "usage: $BEX OPTION"
		echo "usage: $BEX --generic <DIR>"
		echo "usage: $BEX --function <DIR> <FUNC>"
		echo
		echo "OPTION:"
                echo " --help		show help."
                echo " --listfunc	show available functions."
                echo " --version	show version."
                echo " --generic	show generic info."
                echo " --function	show info from selected function."
		exit
	;;
	--version|-v)
		echo -n "$(basename $EXE) "
		head -11 $EXE | grep "^# Version: "
		exit
	;;
	--listfunc|-l)
		awk '$1=="function" && $2~/^chk_/ {print $2}' $EXE |\
			tr -d "{()" | sort
		exit
	;;
	--generic|-g)
		cd $DIR
		chk_overview
		chk_hardware
		chk_kernel
		chk_modules
		chk_multipath
		chk_storage
		chk_filesys
		chk_network
		chk_nameres
		chk_timeres
		chk_software
		chk_updates
		chk_services
		chk_sysconf
		chk_errors
		chk_sysload

		rm -rf $TMP
		exit
	;;
	--function|-f)
		cd $DIR
		shift
		if [ $# -gt 1 ]; then	
			for f in $*; do
				EXE=$(type -p $0)
				grep "^function ${f}().$" $EXE && $f
				shift
			done
		else $FUN
		fi

		rm -rf $TMP
		exit
	;;
	--test|-t)
		cd $DIR
		#set -x
		chk_sleha

		rm -rf $TMP
		exit
	;;
esac
#
