#!/bin/sh
# DF-2657 fix validation (participant-quorum kernel):
#  A. identical-clone cluster: sync threads must reach h2idle (NOT h2coll),
#     umount -f must complete quickly, third mount + single-device sanity.
#  B. diverged cluster (m1 has file X, m2 does not): m2's sync thread must
#     PULL X in (2-master sync has never functioned before this fix), m1
#     must NOT regress (forward-only gate), no panics.
# All evidence to /root/poc/df2657_fix/.
D=/root/poc/df2657_fix
mkdir -p $D
exec > $D/fix_run.log 2>&1
set -x
uname -a

echo '===== PHASE A: identical clones (wedge regression) ====='
for v in vn0 vn1 vn2 vn3; do vnconfig -u $v 2>/dev/null; done
for m in /mnt/h2 /mnt/h2b /mnt/h2d /mnt/h2e; do timeout 15 umount -f $m 2>/dev/null; done
mkdir -p /mnt/h2 /mnt/h2b /mnt/h2d /mnt/h2e

vnconfig -c vn0 /root/poc/base.img
vnconfig -c vn1 /root/poc/base.img
mount -t hammer2 -o ro /dev/vn0@testvol /mnt/h2
echo "A_MOUNT1_RC=$?"
mount -t hammer2 -o ro /dev/vn1@testvol /mnt/h2b
echo "A_MOUNT2_RC=$?"
sleep 12
echo '--- A: h2nod census (EXPECT h2idle, NO h2coll) ---'
ps axlw | grep h2nod | grep -v grep
echo -n 'A_H2COLL_COUNT='; ps axlw | grep h2nod | grep -v grep | grep -c h2coll
ls /mnt/h2
echo "A_LS_RC=$?"

echo '--- A: umount -f must complete FAST (stock: 65s stall + failure) ---'
date +%s > $D/a_ts_start
umount -f /mnt/h2
A_UMF_RC=$?
date +%s > $D/a_ts_end
echo "A_UMOUNTF_RC=$A_UMF_RC"
echo "A_ELAPSED=$(( $(cat $D/a_ts_end) - $(cat $D/a_ts_start) ))s"
echo '--- A: survivor census ---'
ps axlw | grep h2nod | grep -v grep
echo -n 'A_SURV_H2XOP='; ps axlw | grep -c h2xop-testvol

echo '--- A: third mount + single-device sanity ---'
vnconfig -c vn2 /root/poc/base.img
timeout 20 mount -t hammer2 -o ro /dev/vn2@testvol /mnt/h2d
echo "A_MOUNT3_RC=$?"
ls /mnt/h2d; echo "A_MOUNT3_LS_RC=$?"
timeout 20 umount /mnt/h2d; echo "A_MOUNT3_UM_RC=$?"
timeout 20 umount -f /mnt/h2b; echo "A_H2B_UM_RC=$?"
vnconfig -u vn0; vnconfig -u vn1; vnconfig -u vn2
timeout 20 mount -t hammer2 -o ro /dev/vn3@testvol /mnt/h2e 2>/dev/null || true
vnconfig -c vn3 /root/poc/base.img
timeout 20 mount -t hammer2 -o ro /dev/vn3@testvol /mnt/h2e
echo "A_SANITY_MOUNT_RC=$?"
timeout 20 umount /mnt/h2e
echo "A_SANITY_UM_RC=$?"
vnconfig -u vn3

echo '===== PHASE B: diverged cluster (real sync + no regression) ====='
# m1 = base + file X (solo rw mount + clean umount); m2 = base
cp /root/poc/base.img $D/b1.img
cp /root/poc/base.img $D/b2.img
vnconfig -c vn0 $D/b1.img
mount -t hammer2 /dev/vn0@testvol /mnt/h2
echo "B_PREP_MOUNT_RC=$?"
echo "converge-me-hammer2" > /mnt/h2/X
ls -l /mnt/h2/X; cat /mnt/h2/X
umount /mnt/h2
echo "B_PREP_UM_RC=$?"
vnconfig -u vn0

vnconfig -c vn0 $D/b1.img
vnconfig -c vn1 $D/b2.img
mount -t hammer2 /dev/vn0@testvol /mnt/h2
echo "B_MOUNT1_RC=$?"
mount -t hammer2 -o ro /dev/vn1@testvol /mnt/h2b
echo "B_MOUNT2_RC=$?"
echo '--- B: pre-sync view of merged root ---'
ls /mnt/h2
echo '--- B: let sync threads run (3 x 5s passes) ---'
sleep 20
ps axlw | grep h2nod | grep -v grep
echo -n 'B_H2COLL_COUNT='; ps axlw | grep h2nod | grep -v grep | grep -c h2coll
dmesg | tail -15

echo '--- B: teardown (plain umount must complete; stock: impossible) ---'
date +%s > $D/b_ts_start
umount /mnt/h2
B_UM_RC=$?
date +%s > $D/b_ts_end
echo "B_UMOUNT_RC=$B_UM_RC ELAPSED=$(( $(cat $D/b_ts_end) - $(cat $D/b_ts_start) ))s"
timeout 20 umount -f /mnt/h2b; echo "B_H2B_UM_RC=$?"
vnconfig -u vn0; vnconfig -u vn1

echo '--- B: verify convergence on media: solo-mount m2 (stale clone) ---'
vnconfig -c vn0 $D/b2.img
mount -t hammer2 -o ro /dev/vn0@testvol /mnt/h2d
echo "B_M2_MOUNT_RC=$?"
ls -l /mnt/h2d/
cat /mnt/h2d/X > $D/b2_X_content.txt 2>&1
echo "B_M2_X_CAT_RC=$?"
cat $D/b2_X_content.txt
umount /mnt/h2d; echo "B_M2_UM_RC=$?"
vnconfig -u vn0

echo '--- B: verify no regression on media: solo-mount m1 (fresh clone) ---'
vnconfig -c vn0 $D/b1.img
mount -t hammer2 -o ro /dev/vn0@testvol /mnt/h2d
echo "B_M1_MOUNT_RC=$?"
ls -l /mnt/h2d/
cat /mnt/h2d/X
echo "B_M1_X_CAT_RC=$?"
umount /mnt/h2d; echo "B_M1_UM_RC=$?"
vnconfig -u vn0

echo '===== PHASE C: stability watch ====='
sleep 90
dmesg | tail -10
ps axlw | grep -E "h2nod|h2xop-testvol" | grep -v grep | awk '{print $8}' | sort | uniq -c
echo VALIDATION_DF2657_DONE