#!/bin/sh
# DF-2660 baseline (stock kernel): hammer2_sync_insert() TOCTOU race.
#
# 3-master same-clid cluster: A/B (identical clones) hold f001..fN; C is the
# pristine base (missing all of them).  On merge, C's sync thread (idx=2,
# quorum 2-of-3 reachable among A,B on stock) walks its insert backlog:
#     hammer2_sync_insert()  synchro.c:741-758
#       unlock parent(shared) -> lock EX -> re-lookup key -> KKASSERT(chain==NULL)
# The scan ran with the inode unlocked and the parent shared, so a VOP create
# that inserts the same dirent key on C between the scan and the re-lookup
# makes the assert fire: panic on INVARIANTS kernels.
#
# Race choreography (attempt lessons):
#  a/e: m2/m3 subdirs hidden under the mounted $M -> M2/M3 failed with
#       mount(8) usage-error 64 (indistinguishable from EBUSY by rc) ->
#       NO cluster at all.  Use separate top-level mountpoints + verify the
#       merge via the h2nod-<label> census.
#  a:   full-speed hammer from M3-3s -- the sync thread's ipcluster collect
#       catches an A/B root-mtid skew -> EINPROGRESS -> 60s h2coll park;
#       pass never ran.
#  d:   0.1s/pair throttle -- still 80 txn/s on the root inode, pass never ran.
#  e:   dmesg was broken (msgbuf overflow, "Cannot allocate memory") -- use
#       /var/log/messages for backlog detection.
#  => hammer WAITS (quiescent) until syslog shows the first 'insert rec'
#     (pass entered, backlog running), then unleashes 6 full-speed loops to
#     race the rest of the backlog.
#
# usage: trigger_df2660.sh <attempt-label> <vn-base>   e.g. f 0
D=/root/poc/df2660
mkdir -p $D
sysctl -w debug.debugger_on_panic=0
exec > $D/run_$1.log 2>&1
set -x
LABEL=$1
VB=$2
V0=vn$VB
V1=vn$((VB + 1))
V2=vn$((VB + 2))
N=200
M=/mnt/race_$1
M2=/mnt/racem2_$1
M3=/mnt/racem3_$1
mkdir -p $M $M2 $M3

# 0. clean slate; ABORT if a unit will not attach cleanly
for v in $V0 $V1 $V2; do vnconfig -u $v 2>/dev/null; done
: > $D/.probe_$1
for v in $V0 $V1 $V2; do
	vnconfig -c $v $D/.probe_$1 || { echo "VN_ATTACH_FAILED_$v"; touch $D/stop_$LABEL; exit 1; }
	vnconfig -u $v
done
rm -f $D/.probe_$1

# 1. base image + newfs
dd if=/dev/zero of=$D/base_$1.img bs=1m count=512 status=none
vnconfig -c $V0 $D/base_$1.img || { echo VN_ATTACH_FAILED_$V0; exit 1; }
newfs_hammer2 -L $LABEL /dev/$V0
echo "NEWFS_RC=$?"
vnconfig -u $V0

# 2. column A: f001..fN (empty files: dirent+inode chains, no data)
cp $D/base_$1.img $D/A_$1.img
vnconfig -c $V0 $D/A_$1.img || { echo VN_ATTACH_FAILED_$V0; exit 1; }
mount -t hammer2 /dev/$V0@$LABEL $M
echo "MOUNT_A_RC=$?"
mount | grep -q " $M " || { echo "A_MOUNT_NOT_UP aborting"; touch $D/stop_$LABEL; exit 1; }
i=1
while [ $i -le $N ]; do
	> $M/f$i
	i=$((i + 1))
done
# dirent-hash order = readdir (RB key) order; the sync thread inserts in
# exactly this order, so hash-order hammer sweeps CROSS the insertion
# point every lap instead of hitting it 1/N-randomly.
ls -f $M | grep -E '^f[0-9]+$' > $D/order_$1.txt
wc -l $D/order_$1.txt
sync
sleep 2
umount $M
echo "UMOUNT_A_RC=$?"
vnconfig -u $V0

# 3. B = clone(A); C = pristine base
cp $D/A_$1.img $D/B_$1.img
cp $D/base_$1.img $D/C_$1.img
md5 $D/A_$1.img $D/B_$1.img

# 4. parked hammers: wait for GO, then full speed
hloop() {
	while [ ! -f $D/go_$LABEL ] && [ ! -f $D/stop_$LABEL ]; do sleep 0.2; done
	# REVERSED sweep: the sync thread advances FORWARD through the same
	# hash order, so a backward-sweeping hammer meets the insertion
	# frontier head-on (trailing sweeps never collide -- attempt f/g/h).
	for j in $(tail -r $D/order_$LABEL.txt); do
		[ -f $D/stop_$LABEL ] && break
		rm -f $M/$j
		echo x > $M/$j
	done
	echo "HLOOP$1_SWEEP_DONE $(date +%s)" >&2
	while [ ! -f $D/stop_$LABEL ]; do sleep 0.5; done
}
sysctl vfs.hammer2.debug=32768
vnconfig -c $V0 $D/A_$1.img
vnconfig -c $V1 $D/B_$1.img
vnconfig -c $V2 $D/C_$1.img
mount -t hammer2 /dev/$V0@$LABEL $M
echo "CLUSTER_M1_RC=$?"
mount | grep -q " $M " || { echo "CLUSTER_M1_NOT_UP aborting"; touch $D/stop_$LABEL; exit 1; }
mount -t hammer2 /dev/$V1@$LABEL $M2
echo "CLUSTER_M2_RC=$?"		# EBUSY + merge
mount -t hammer2 /dev/$V2@$LABEL $M3
echo "CLUSTER_M3_RC=$?"		# EBUSY + merge; sync thread idx2 starts NOW
sleep 2
NH=$(ps axlw | grep "h2nod-$LABEL" | grep -v grep | wc -l)
echo "H2NOD_COUNT=$NH (need >=3 sync threads for a real 3-master cluster)"
[ "$NH" -ge 3 ] || { echo "NO_SYNC_THREADS aborting"; touch $D/stop_$LABEL; exit 1; }
ps axlw | grep "h2nod-$LABEL" | grep -v grep
hloop 1 & hloop 2 & hloop 3 & hloop 4 & hloop 5 & hloop 6 &

# 5. wait for the backlog to START (quiescent entry), then unleash.
#    /var/log/messages carries the kern prints (dmesg msgbuf is saturated).
i=0
while [ $i -lt 150 ]; do
	if grep -q "sync_slaves pfs $LABEL" /var/log/messages 2>/dev/null; then
		echo "PASS_SEEN at $(date +%s) poll=$i -- UNLEASHING"
		touch $D/go_$LABEL
		break
	fi
	sleep 0.2
	i=$((i + 1))
done
[ -f $D/go_$LABEL ] || { echo "NO_PASS_SEEN"; touch $D/stop_$LABEL; exit 1; }

# 6. race window: up to 15s of full-speed hammering against the backlog
sleep 15
touch $D/stop_$LABEL
sleep 2
n=$(grep -c 'insert rec par=' /var/log/messages 2>/dev/null)
echo "POST inserts_seen=$n files=$(ls $M 2>/dev/null | grep -c '^f')"
echo "=== DF-2660 ATTEMPT $1 COMPLETE (no panic if you can read this) ==="