DF-2657 / fix_validate.sh
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | #!/bin/sh # DF-2657 fix validation (participant-quorum kernel): # A. identical-clone cluster: sync threads must reach h2idle (NOT h2coll), # umount -f must complete quickly, third mount + single-device sanity. # B. diverged cluster (m1 has file X, m2 does not): m2's sync thread must # PULL X in (2-master sync has never functioned before this fix), m1 # must NOT regress (forward-only gate), no panics. # All evidence to /root/poc/df2657_fix/. D=/root/poc/df2657_fix mkdir -p $D exec > $D/fix_run.log 2>&1 set -x uname -a echo '===== PHASE A: identical clones (wedge regression) =====' for v in vn0 vn1 vn2 vn3; do vnconfig -u $v 2>/dev/null; done for m in /mnt/h2 /mnt/h2b /mnt/h2d /mnt/h2e; do timeout 15 umount -f $m 2>/dev/null; done mkdir -p /mnt/h2 /mnt/h2b /mnt/h2d /mnt/h2e vnconfig -c vn0 /root/poc/base.img vnconfig -c vn1 /root/poc/base.img mount -t hammer2 -o ro /dev/vn0@testvol /mnt/h2 echo "A_MOUNT1_RC=$?" mount -t hammer2 -o ro /dev/vn1@testvol /mnt/h2b echo "A_MOUNT2_RC=$?" sleep 12 echo '--- A: h2nod census (EXPECT h2idle, NO h2coll) ---' ps axlw | grep h2nod | grep -v grep echo -n 'A_H2COLL_COUNT='; ps axlw | grep h2nod | grep -v grep | grep -c h2coll ls /mnt/h2 echo "A_LS_RC=$?" echo '--- A: umount -f must complete FAST (stock: 65s stall + failure) ---' date +%s > $D/a_ts_start umount -f /mnt/h2 A_UMF_RC=$? date +%s > $D/a_ts_end echo "A_UMOUNTF_RC=$A_UMF_RC" echo "A_ELAPSED=$(( $(cat $D/a_ts_end) - $(cat $D/a_ts_start) ))s" echo '--- A: survivor census ---' ps axlw | grep h2nod | grep -v grep echo -n 'A_SURV_H2XOP='; ps axlw | grep -c h2xop-testvol echo '--- A: third mount + single-device sanity ---' vnconfig -c vn2 /root/poc/base.img timeout 20 mount -t hammer2 -o ro /dev/vn2@testvol /mnt/h2d echo "A_MOUNT3_RC=$?" ls /mnt/h2d; echo "A_MOUNT3_LS_RC=$?" timeout 20 umount /mnt/h2d; echo "A_MOUNT3_UM_RC=$?" timeout 20 umount -f /mnt/h2b; echo "A_H2B_UM_RC=$?" vnconfig -u vn0; vnconfig -u vn1; vnconfig -u vn2 timeout 20 mount -t hammer2 -o ro /dev/vn3@testvol /mnt/h2e 2>/dev/null || true vnconfig -c vn3 /root/poc/base.img timeout 20 mount -t hammer2 -o ro /dev/vn3@testvol /mnt/h2e echo "A_SANITY_MOUNT_RC=$?" timeout 20 umount /mnt/h2e echo "A_SANITY_UM_RC=$?" vnconfig -u vn3 echo '===== PHASE B: diverged cluster (real sync + no regression) =====' # m1 = base + file X (solo rw mount + clean umount); m2 = base cp /root/poc/base.img $D/b1.img cp /root/poc/base.img $D/b2.img vnconfig -c vn0 $D/b1.img mount -t hammer2 /dev/vn0@testvol /mnt/h2 echo "B_PREP_MOUNT_RC=$?" echo "converge-me-hammer2" > /mnt/h2/X ls -l /mnt/h2/X; cat /mnt/h2/X umount /mnt/h2 echo "B_PREP_UM_RC=$?" vnconfig -u vn0 vnconfig -c vn0 $D/b1.img vnconfig -c vn1 $D/b2.img mount -t hammer2 /dev/vn0@testvol /mnt/h2 echo "B_MOUNT1_RC=$?" mount -t hammer2 -o ro /dev/vn1@testvol /mnt/h2b echo "B_MOUNT2_RC=$?" echo '--- B: pre-sync view of merged root ---' ls /mnt/h2 echo '--- B: let sync threads run (3 x 5s passes) ---' sleep 20 ps axlw | grep h2nod | grep -v grep echo -n 'B_H2COLL_COUNT='; ps axlw | grep h2nod | grep -v grep | grep -c h2coll dmesg | tail -15 echo '--- B: teardown (plain umount must complete; stock: impossible) ---' date +%s > $D/b_ts_start umount /mnt/h2 B_UM_RC=$? date +%s > $D/b_ts_end echo "B_UMOUNT_RC=$B_UM_RC ELAPSED=$(( $(cat $D/b_ts_end) - $(cat $D/b_ts_start) ))s" timeout 20 umount -f /mnt/h2b; echo "B_H2B_UM_RC=$?" vnconfig -u vn0; vnconfig -u vn1 echo '--- B: verify convergence on media: solo-mount m2 (stale clone) ---' vnconfig -c vn0 $D/b2.img mount -t hammer2 -o ro /dev/vn0@testvol /mnt/h2d echo "B_M2_MOUNT_RC=$?" ls -l /mnt/h2d/ cat /mnt/h2d/X > $D/b2_X_content.txt 2>&1 echo "B_M2_X_CAT_RC=$?" cat $D/b2_X_content.txt umount /mnt/h2d; echo "B_M2_UM_RC=$?" vnconfig -u vn0 echo '--- B: verify no regression on media: solo-mount m1 (fresh clone) ---' vnconfig -c vn0 $D/b1.img mount -t hammer2 -o ro /dev/vn0@testvol /mnt/h2d echo "B_M1_MOUNT_RC=$?" ls -l /mnt/h2d/ cat /mnt/h2d/X echo "B_M1_X_CAT_RC=$?" umount /mnt/h2d; echo "B_M1_UM_RC=$?" vnconfig -u vn0 echo '===== PHASE C: stability watch =====' sleep 90 dmesg | tail -10 ps axlw | grep -E "h2nod|h2xop-testvol" | grep -v grep | awk '{print $8}' | sort | uniq -c echo VALIDATION_DF2657_DONE |