DF-2658 / trigger_df2658.sh
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 | #!/bin/sh # DF-2658 baseline (stock kernel): 3-master same-clid hammer2 cluster built # from dd clones; columns A/B (identical) hold, in this creation order: # d1 (4KB), F (1KB), d2 (4KB) # and column C (diverged, same creation order => same inums, same dirent # keys, same lbase-0 data-chain key): # d1 (4KB), F (1 byte, sync, then 64KB), d2 (4KB) # The two-stage F write on C keeps F's inode number but shifts F's # inode-chain and data-chain modify_tids past A/B's, so C's sync thread # (a) rescans (root mtid differs) and (b) defers into F's inode and # "replaces" F's DATA chain: # chain->bytes(65536) != focus->bytes(1024) enters the resize branch, which # computes nradix = hammer2_getradix(chain->bytes) (the OLD size) # => hammer2_chain_resize() early-returns a no-op (chain.c:1363-1366) # => bcopy(data, chain->data, chain->bytes) at synchro.c:1041 copies 65536 # bytes out of the focus's 1024-byte allocation. # # Expected on-media evidence in C's live F block: # [1024B focus content][d2's 4KB content][rest of the 64KB dio media # region][bytes past the end of the 64KB kernel dio buffer] # another file's content + kernel buffer overread persisted by the sync # thread into F's data block, plus F's logical size collapsing 65536 -> 1024 # (silent truncation). # # Run-1/2 post-mortems baked into this version: # - run 1: unmatched A/B-only decoy files dragged the deferred-inode # machinery into the NULL-deref at hammer2_inode.c:453 (chain==NULL -> # chain->parent, fault va 0x118) AFTER the bug path ran; d1/d2 are now # created on ALL columns (aligned, safe). # - run 2: identical creation sequences on cloned allocators made every # modify_tid equal => want_update==0, no sync at all. The two-stage F # write on C breaks mtid equality without changing the inum order. # - the flusher NULL-deref (DF-2620 family) can kill the guest seconds # after the pass, so this rig polls dmesg at 0.3s, syncs the instant the # DATA replace fires, and extracts evidence FIRST. D=/root/poc/df2658 sysctl -w debug.debugger_on_panic=0 mkdir -p $D exec > $D/run.log 2>&1 set -x LABEL=tv58 # 0. clean slate for v in vn0 vn1 vn2; do vnconfig -u $v 2>/dev/null; done for m in /mnt/h2a /mnt/h2b /mnt/h2c; do timeout 5 umount -f $m 2>/dev/null; done mkdir -p /mnt/h2a /mnt/h2b /mnt/h2c # 1. stage incompressible random content dd if=/dev/urandom of=$D/d1.bin bs=4096 count=1 status=none dd if=/dev/urandom of=$D/f_ab.bin bs=1024 count=1 status=none # A/B's F dd if=/dev/urandom of=$D/d2.bin bs=4096 count=1 status=none dd if=/dev/urandom of=$D/f_c.bin bs=65536 count=1 status=none # C's F dd if=/dev/urandom of=$D/d1_c.bin bs=4096 count=1 status=none # C's d1 dd if=/dev/urandom of=$D/d2_c.bin bs=4096 count=1 status=none # C's d2 # 2. base image + newfs (one MASTER PFS "tv58") dd if=/dev/zero of=$D/base.img bs=1m count=256 status=none vnconfig -c vn0 $D/base.img newfs_hammer2 -L $LABEL /dev/vn0 echo "NEWFS_RC=$?" vnconfig -u vn0 # 3. column A: solo rw mount; creation order d1, F(1KB one-shot), d2 cp $D/base.img $D/A.img vnconfig -c vn0 $D/A.img mount -t hammer2 /dev/vn0@$LABEL /mnt/h2a echo "MOUNT_A_SOLO_RC=$?" dd if=$D/d1.bin of=/mnt/h2a/d1 bs=4096 count=1 status=none dd if=$D/f_ab.bin of=/mnt/h2a/F bs=1024 count=1 status=none dd if=$D/d2.bin of=/mnt/h2a/d2 bs=4096 count=1 status=none hammer2 stat /mnt/h2a/d1 /mnt/h2a/F /mnt/h2a/d2 sync sleep 2 umount /mnt/h2a echo "UMOUNT_A_SOLO_RC=$?" vnconfig -u vn0 # 4. column B = bit-identical clone of A cp $D/A.img $D/B.img md5 $D/A.img $D/B.img # 5. column C: solo rw mount; SAME creation order, but F written in two # stages (1 byte, sync, then the full 64KB) so F's inode/data mtids # land past A/B's while the inum order stays identical cp $D/base.img $D/C.img vnconfig -c vn2 $D/C.img mount -t hammer2 /dev/vn2@$LABEL /mnt/h2c echo "MOUNT_C_SOLO_RC=$?" dd if=$D/d1_c.bin of=/mnt/h2c/d1 bs=4096 count=1 status=none dd if=/dev/urandom of=/mnt/h2c/F bs=1 count=1 status=none sync sleep 1 dd if=$D/f_c.bin of=/mnt/h2c/F bs=65536 count=1 status=none dd if=$D/d2_c.bin of=/mnt/h2c/d2 bs=4096 count=1 status=none hammer2 stat /mnt/h2c/d1 /mnt/h2c/F /mnt/h2c/d2 ls -l /mnt/h2c/F sync sleep 2 umount /mnt/h2c echo "UMOUNT_C_SOLO_RC=$?" vnconfig -u vn2 # 6. assemble the 3-master cluster (first mount rw, merges via EBUSY mounts) sysctl vfs.hammer2.debug=1 vnconfig -c vn0 $D/A.img vnconfig -c vn1 $D/B.img vnconfig -c vn2 $D/C.img mount -t hammer2 /dev/vn0@$LABEL /mnt/h2a echo "CLUSTER_M1_RC=$?" mount -t hammer2 /dev/vn1@$LABEL /mnt/h2b echo "CLUSTER_M2_RC=$?" # EBUSY + merge mount -t hammer2 /dev/vn2@$LABEL /mnt/h2c echo "CLUSTER_M3_RC=$?" # EBUSY + merge; sync thread idx2 starts NOW # 6b. poll dmesg at 0.3s for the DATA-chain replace (the bug executing); # sync hard the moment it lands; then EXTRACT EVIDENCE IMMEDIATELY # (the DF-2620-family flusher NULL-deref can kill the guest soon after) i=0 while [ $i -lt 300 ]; do if dmesg | grep -q 'replace rec .* slave 2 3\.0000000000000000'; then echo "DATA_REPLACE_SEEN at $(date +%s) poll=$i" break fi sleep 0.3 i=$((i + 1)) done sync sync sleep 1 sync echo '--- sync pass lines ---' dmesg | grep -E 'replace rec|insert rec|destroy rec' | tail -20 # 7. evidence extraction FIRST (narrow 8MB..40MB window: fresh-volume data # allocations live right past the 8MB boot + 8MB aux areas) cd $D echo '=== locate F on B (source geometry) and on C (victim) ===' ./scanbin find B.img f_ab.bin 8 8388608 41943040 > find_B_fab.txt ./scanbin find C.img f_ab.bin 8 8388608 41943040 > find_C_fab.txt cat find_B_fab.txt find_C_fab.txt OFF_B=$(sed -n 's/HIT //p' find_B_fab.txt | head -1) OFF_C=$(sed -n 's/HIT //p' find_C_fab.txt | head -1) echo "OFF_B=$OFF_B OFF_C=$OFF_C" echo "OFF_B_in_64K=$((OFF_B % 65536)) (in-dio offset of the focus allocation)" echo "expected read past the 64KB kernel dio buffer end = $((OFF_B % 65536)) bytes" echo '=== alignment sanity on B: d1 before F, d2 after F ===' ./scanbin find B.img d1.bin 4 8388608 41943040 > find_B_d1.txt ./scanbin find B.img d2.bin 4 8388608 41943040 > find_B_d2.txt cat find_B_d1.txt find_B_d2.txt ./scanbin dump C.img $OFF_C 65536 F_block_on_C.bin ./scanbin dump B.img $OFF_B 65536 F_block_on_B.bin ./scanbin dump C.img $((OFF_C + 1024)) 64512 F_tail_on_C.bin ./scanbin dump B.img $((OFF_B + 1024)) 64512 F_tail_on_B.bin echo '=== victim block: 64K live F block on C ===' ./scanbin stat F_block_on_C.bin 0 65536 ./scanbin stat F_tail_on_C.bin 0 64512 echo '=== victim tail vs B media tail at the same relative position ===' echo '(MATCH = read from the focus dio buffer = B media region content;' echo' DIFFER after the 64K-buffer seam = read from kernel memory past the' echo' end of the dio buffer)' ./scanbin cmp F_tail_on_C.bin 0 64512 F_tail_on_B.bin echo '=== cross-file leak: d1/d2 content inside the victim tail ===' for p in d1 d2; do echo -n "$p: " ./scanbin find F_tail_on_C.bin $p.bin 2 | head -2 done echo '=== hexdump: victim head (focus content), tail start, tail end ===' hexdump -C -n 64 F_block_on_C.bin echo '... victim tail start (block byte 1024):' hexdump -C -n 48 F_tail_on_C.bin echo '... victim tail end (last 32B of the 64K block):' tail -c 32 F_tail_on_C.bin | hexdump -C echo '=== B media tail at the same relative position (what legitimately lives there) ===' hexdump -C -n 48 F_tail_on_B.bin md5 F_block_on_C.bin F_block_on_B.bin f_ab.bin f_c.bin sync # 8. post-pass observations (if the guest is still alive) echo '--- cluster view of F (expect size 1024: silent truncation) ---' ls -l /mnt/h2a/F cmp -n 1024 $D/f_ab.bin /mnt/h2a/F echo "CMP_FOCUS_HEAD_RC=$?" echo '--- h2nod census ---' ps axlw | grep h2nod | grep -v grep echo "=== DF-2658 BASELINE COMPLETE ===" |