#!/bin/bash 

die() {
	echo "$@" 1>&2
	exit 1
}

usage() {
        echo "usage: $0 [-a|-h] CONF_FILE" 1>&2
        echo "options:" 1>&2
        echo "    -a    abort recovery" 1>&2
        echo "    -h    this message" 1>&2
        exit 2
}

# Don't abort recovery by default
OPTS=

while getopts "ah" opt; do
        case "$opt" in
        a)      OPTS="-o abort_recov" ;;
        h|\?)   usage ;;
        *)      echo "Unexpected option return" 1>&2 ;;
        esac
done

shift $((OPTIND - 1))

[[ $1 ]] || die "Need configuration file"
. /chexport/lustre/hostlist.sh || die "Helper functions missing"
. $1 || die "Problem sourcing configuration file"

# XXX 1) validate that we have everything we need
# XXX 2) Check that all hosts are up
# XXX 3) modprobe lustre everywhere
# XXX 4) ensure connectivity to MGS

PDSH="pdsh -N -w"
#PDSH="echo $PDSH"

# Get the real number of disk
disk_cnt=0              
max=0                   
i=1                     
#for oss in $(hostlist_expand "$OSS"); do
#        # Okay find what we really have
#        luns=$($PDSH $oss "ls --sort=version /dev/mpath/*")
#        let "disk_cnt=$(get_node_count $luns)/2" # Handle fail over
                        
        # Deal with what we want
#        var="OST${i}DEVBASE"
#        osts=$(hostlist_expand "${!var}" | sort -n)
#        let "count=$(get_node_count "$osts")"

#        [[ $count -gt $disk_cnt ]] && {
#                let "ratio=$count/$disk_cnt"

#                volslist="${!var%%]*}]"
#                rootname=$(echo ${volslist//\/dev\/} | awk ' { printf $1 }')
#                vols=$(hostlist_expand "${volslist//\/dev\/}")
#                for vol in $vols; do
#                        $PDSH $oss "pvcreate -ff -y /dev/mpath/$vol"
#                        $PDSH $oss "vgcreate $vol /dev/mpath/$vol"
#                        vg=$($PDSH $oss "vgdisplay $vol | grep \"Free  PE\"")
#                        size=$(echo $vg | awk ' { print $5 }')
#                        let "size/=$ratio"
#                        for j in $(seq 0 $((--ratio))); do
#                                $PDSH $oss "lvcreate -l$size $vol /dev/mpath/$vol -n ${rootname%%[*}$j"
#                        done;
#                done
#        }
#        ((i++))
#        [[ $max -lt $count ]] && max=$count;
#done
max=7

echo "bring up lustre modules"
$PDSH $OSS,${MGTDEV%%:*},${MDTDEV%%:*} "modprobe lustre"

echo "start lustre debug"
. /chexport/lustre/debug.sh

# First format and mount the MGS
# XXX Don't reformat it if it is up and mounted
host=${MGTDEV%%:*}
dev=${MGTDEV#*:}; dev=${dev%%:*}
$PDSH $host "lctl del_route 10.37.228.1@o2ib1; lctl del_route 10.37.228.2@o2ib1; lctl del_route 10.37.228.3@o2ib1; lctl del_route 10.37.228.4@o2ib1"
mntpt=`$PDSH $host "[ -e /proc/fs/lustre/mgs/MGS/mntdev ] && cat /proc/fs/lustre/mgs/MGS/mntdev"`
if [ -z "$mntpt" ]; then {
	echo "Mounting MGS"
	$PDSH $host "lctl debug_daemon start /chexport/lustre/logs/lustre-log.$host 2048"
	$PDSH $host "mkdir -p /tmp/lustre/$MGSNAME && mount -t lustre $OPTS $dev /tmp/lustre/$MGSNAME" ||
		die "Unable to mount MGT $dev on /tmp/lustre/$MGSNAME"
	echo "mounted at time $(date)"
} else {
	echo "MGS already mounted"
} fi

# Mount everything one at a time
#
echo "OST bring up"
$PDSH $OSS "lctl del_route 10.37.228.1@o2ib1; lctl del_route 10.37.228.2@o2ib1; lctl del_route 10.37.228.3@o2ib1; lctl del_route 10.37.228.4@o2ib1"

index=1
for i in $(seq $max); do
	j=1
	for oss in $(hostlist_expand "$OSS"); do
		var="OST${j}DEVBASE"
		disk=$(hostlist_expand "${!var}" $i)
		ost=$(printf "%s-OST%04x" $FSNAME $index) 
		echo "Mounting $disk labled $ost as OST$index on $oss"
		dev=`$PDSH $oss "readlink $disk"`
		[ -n "$dev" ] && {
			dev=${dev//..\///dev/}
			mntpt=`$PDSH $oss "cat /proc/mounts | grep $dev"`
			[ -z "$mntpt" ] && {
				$PDSH $oss "lctl debug_daemon start /chexport/lustre/logs/lustre-log.$oss 2048"
				$PDSH $oss "mkdir -p /tmp/$FSNAME/ost$index && mount -t lustre $OPTS $disk /tmp/$FSNAME/ost$index" ||
					die "Unable to mount OST $index $dev on /tmp/$FSNAME/ost$index"
				echo "mounted at time $(date)"
			}
		}
		((index++))
		((j++))
	done
done

# Mount MDT
#
host=${MDTDEV%%:*}
dev=${MDTDEV#*:}; dev=${dev%%:*}
$PDSH $host "lctl del_route 10.37.228.1@o2ib1; lctl del_route 10.37.228.2@o2ib1; lctl del_route 10.37.228.3@o2ib1; lctl del_route 10.37.228.4@o2ib1"
mntpt=`$PDSH $host "[ -e /proc/fs/lustre/osd-ldiskfs/$FSNAME-MDT0000/mntdev ] && cat /proc/fs/lustre/osd-ldiskfs/$FSNAME-MDT0000/mntdev"`
if [ -z "$mntpt" ];then {
	echo "Mounting MDT"
	$PDSH $host "lctl debug_daemon start /chexport/lustre/logs/lustre-log.$host 2048"
	$PDSH $host "mkdir -p /tmp/lustre/$FSNAME/mdt && mount -t lustre $OPTS $dev /tmp/lustre/$FSNAME/mdt" ||
		die "Unable to mount MDT $dev on /tmp/lustre/$FSNAME/mdt"
	echo "mounted at time $(date)"
} else {
	echo "MDT already mounted"
} fi
$PDSH $host "lctl set_param mdt.lustre-MDT0000.identity_upcall=NONE"

echo "Disabling OSS read cache"
$PDSH $OSS -u 30 'lctl set_param obdfilter.*.read_cache_enable=0' || die "Unable to disable OSS read cache"

echo "Disabling OSS writethroughcache"
$PDSH $OSS -u 30 'lctl set_param obdfilter.*.writethrough_cache_enable=0' || die "Unable to disable OSS writethrough cache"

echo "$FSNAME created and mounted"
[ -n "$CLIENTS" -a -n "$MOUNT" ] && {
	echo "Mounting clients"
	$PDSH $CLIENTS "modprobe lustre"
	$PDSH $CLIENTS "lctl del_route 10.37.228.1@o2ib1; lctl del_route 10.37.228.2@o2ib1; lctl del_route 10.37.228.3@o2ib1; lctl del_route 10.37.228.4@o2ib1"
	$PDSH $CLIENTS "mount -t lustre -o user_xattr,flock $MGS:/$FSNAME $MOUNT"
}