#!/bin/sh
# DF-2631 trigger: umount -f of a same-clid multi-device hammer2 cluster
# hangs in hammer2_thr_wait ("h2twait") on UNMODIFIED images.
# All evidence to /root/poc/df2631/.
set -x
D=/root/poc/df2631
mkdir -p $D
for v in vn0 vn1 vn2; do vnconfig -u $v 2>/dev/null; done

echo '=== 0. prepare UNMODIFIED images: dd clones of one newfs image ==='
cp /root/poc/base.img /root/poc/df2631/m1.img
cp /root/poc/base.img /root/poc/df2631/m2.img
cp /root/poc/base.img /root/poc/df2631/m3.img
md5 /root/poc/df2631/m1.img /root/poc/df2631/m2.img

vnconfig -c vn0 /root/poc/df2631/m1.img
vnconfig -c vn1 /root/poc/df2631/m2.img
mkdir -p /mnt/h2 /mnt/h2b

echo '=== 1. mount MASTER #1 ==='
mount -t hammer2 -o ro /dev/vn0@testvol /mnt/h2
echo "MOUNT1_RC=$?"
echo -n 'T1_h2xop='; ps axlw | grep -c h2xop-testvol
echo -n 'T1_h2nod='; ps axlw | grep -c h2nod

echo '=== 2. same-clid merge via second device (EBUSY expected, chain merges) ==='
mount -t hammer2 -o ro /dev/vn1@testvol /mnt/h2b
echo "MOUNT2_RC=$?"
echo -n 'T2_h2xop='; ps axlw | grep -c h2xop-testvol
echo -n 'T2_h2nod='; ps axlw | grep -c h2nod
echo '--- h2nod wchan census (merged cluster) ---'
ps axlw | grep h2nod | grep -v grep

echo '=== 3. ls works? (quorum 2/2 both masters present) ==='
ls /mnt/h2
echo "LS_RC=$?"

echo '=== 4. umount -f  (expect: WEDGE in h2twait) ==='
date +%s > $D/ts_umount_start
( umount -f /mnt/h2 > $D/umountf.out 2>&1; echo "UMF_RC=$?" > $D/umountf.rc ) &
UMF=$!
sleep 3
echo "--- +3s: umount thread ---"
ps -o pid,stat,wchan,command -p $(pgrep -f "umount -f /mnt/h2" | head -1) 2>/dev/null || ps axlw | grep "umount -f" | grep -v grep
echo "--- +3s: h2nod census ---"
ps axlw | grep h2nod | grep -v grep
sleep 5
echo "--- +8s: umount + h2nod + survivor counts ---"
ps axlw | grep -E "umount -f" | grep -v grep
ps axlw | grep h2nod | grep -v grep
echo -n 'SURV_h2xop='; ps axlw | grep -c h2xop-testvol
sleep 55
echo "--- +63s: same census (post 60s tsleep timeouts) ---"
ps axlw | grep -E "umount -f" | grep -v grep
ps axlw | grep h2nod | grep -v grep
echo -n 'SURV63_h2xop='; ps axlw | grep -c h2xop-testvol
echo -n 'SURV63_h2xop_ALL='; ps axlw | grep -c h2xop-
ps axlw | grep -E "h2coll|h2feed|h2twait|frozen" | grep -v grep | head -50

echo '=== 5. mount still present? third hammer2 mount blocks on mntlk ==='
vnconfig -c vn2 /root/poc/df2631/m3.img
mkdir -p /mnt/h2d
( timeout 20 mount -t hammer2 -o ro /dev/vn2@testvol /mnt/h2d > $D/mount3.out 2>&1; echo "MOUNT3_RC=$?" > $D/mount3.rc ) &
M3=$!
sleep 6
echo "--- third mount thread wchan (expect mntlk or h2mount) ---"
ps axlw | grep -E "vn2@testvol" | grep -v grep
cat $D/mount3.rc 2>/dev/null

echo '=== 6. final full wchan histogram of hammer2/waiting threads ==='
ps axlw | grep -vE "grep" | awk '{print $(NF-3), $0}' 2>/dev/null | grep -E "h2|mntlk|umount" | head -40
echo '=== TRIGGER COMPLETE — guest wedged; recovery test follows ==='