DF-2630 / fix.combined.diff
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 | --- a/sys/vfs/hammer2/hammer2.h 2026-06-29 12:51:19.000000000 +0000 +++ b/sys/vfs/hammer2/hammer2.h 2026-08-29 02:14:06.557049211 +0000 @@ -1254,6 +1254,8 @@ */ #define HAMMER2_PMPF_SPMP 0x00000001 #define HAMMER2_PMPF_EMERG 0x00000002 /* Emergency delete mode */ +#define HAMMER2_PMPF_ROOTFAILED 0x00000004 /* root quorum unreachable */ +#define HAMMER2_PMPF_TEARDOWN 0x00000008 /* unmount teardown in progress */ #define HAMMER2_DIRTYCHAIN_WAITING 0x80000000 #define HAMMER2_DIRTYCHAIN_MASK 0x7FFFFFFF @@ -1408,6 +1410,7 @@ extern int hammer2_aux_flags; extern int hammer2_debug; +extern int hammer2_xop_collect_timeout; extern int hammer2_xop_nthreads; extern int hammer2_xop_sgroups; extern int hammer2_xop_xgroups; --- a/sys/vfs/hammer2/hammer2_admin.c 2026-06-29 12:51:19.000000000 +0000 +++ b/sys/vfs/hammer2/hammer2_admin.c 2026-08-29 05:01:32.105817956 +0000 @@ -488,8 +488,32 @@ ip1 = xop->ip1; pmp = ip1->pmp; - if (pmp->has_xop_threads == 0) + if (pmp->has_xop_threads == 0) { + /* + * DF-2631: never (re-)create helper threads once the pmp + * is in unmount teardown. Doing so would rebuild + * xop_groups mid-teardown while pfsfree_scan() rips the + * cluster apart, leaving worker threads running inside + * memory that hammer2_xop_helper_cleanup() already freed + * (observed as h2xop survivors and a delayed page fault + * in hammer2_primary_xops_thread). + * + * Feed an immediate EOF + error for every node instead so + * the frontend's hammer2_xop_collect() fails cleanly in + * one pass (CITEM_NULL + error) rather than waiting for a + * quorum that can never form. + */ + if (pmp->flags & HAMMER2_PMPF_TEARDOWN) { + for (i = 0; i < xop->cluster.nchains; ++i) { + if (i != notidx) { + hammer2_xop_feed(xop, NULL, i, + HAMMER2_ERROR_EIO); + } + } + return; + } hammer2_xop_helper_create(pmp); + } /* * The sequencer assigns a worker thread to the XOP. @@ -891,11 +915,13 @@ hammer2_chain_t *chain; hammer2_key_t lokey; uint64_t mask; + int64_t deadline; int error; int keynull; int adv; /* advance the element */ int i; + deadline = 0; loop: /* * First loop tries to advance pieces of the cluster which @@ -993,7 +1019,47 @@ tsleep_interlock(xop, 0); if (atomic_cmpset_64(&xop->run_mask, mask, mask | HAMMER2_XOPMASK_WAIT)) { - tsleep(xop, PINTERLOCKED, "h2coll", hz*60); + /* + * DF-2631: bound the total wait. Some situations + * can never satisfy quorum for this collect: a + * synchronization thread collects a xop started + * with hammer2_xop_start_except() on its own + * cluster index (that column never feeds and is + * counted as 'umasters' forever), and during + * unmount xop helper columns can be deleted or + * frozen while a collect is still pending. The + * old code looped here forever with no failure + * path, wedging VFS_ROOT, the sync threads (which + * then never observe FREEZE/STOP) and umount -f + * (hammer2_thr_wait, "h2twait"). + */ + if (hammer2_xop_collect_timeout == 0 || + (xop->cluster.pmp && + (xop->cluster.pmp->flags & + HAMMER2_PMPF_SPMP))) { + /* + * Legacy behavior (also used for the + * super-root pmp, whose synchronization + * threads legitimately run long scans + * that must not be aborted). + */ + tsleep(xop, PINTERLOCKED, "h2coll", hz*60); + } else { + /* + * Elapsed-time bound (NOT event-counted): + * continuous feed events from a healthy + * column must not starve this timeout when + * another column can never contribute + * (start_except self-exclusion, frozen or + * deleted helpers). + */ + if (deadline == 0) + deadline = ticks + + hammer2_xop_collect_timeout * hz; + tsleep(xop, PINTERLOCKED, "h2coll", hz); + if ((int64_t)ticks - deadline >= 0) + return HAMMER2_ERROR_EIO; + } } goto loop; } --- a/sys/vfs/hammer2/hammer2_vfsops.c 2026-06-29 12:51:19.000000000 +0000 +++ b/sys/vfs/hammer2/hammer2_vfsops.c 2026-08-29 02:14:06.569049061 +0000 @@ -112,6 +112,14 @@ SYSCTL_INT(_vfs_hammer2, OID_AUTO, supported_version, CTLFLAG_RD, &hammer2_supported_version, 0, ""); +int hammer2_xop_collect_timeout = 60; /* secs; 0 = forever */ +SYSCTL_INT(_vfs_hammer2, OID_AUTO, xop_collect_timeout, CTLFLAG_RW, + &hammer2_xop_collect_timeout, 0, + "max seconds one xop collect waits for quorum (0 = forever)"); +static int hammer2_root_timeout = 60; /* secs; 0 = forever (legacy) */ +SYSCTL_INT(_vfs_hammer2, OID_AUTO, root_timeout, CTLFLAG_RW, + &hammer2_root_timeout, 0, + "max seconds VFS_ROOT waits for cluster quorum (0 = forever)"); SYSCTL_INT(_vfs_hammer2, OID_AUTO, aux_flags, CTLFLAG_RW, &hammer2_aux_flags, 0, ""); SYSCTL_INT(_vfs_hammer2, OID_AUTO, debug, CTLFLAG_RW, @@ -768,11 +776,30 @@ /* * Make sure all synchronization threads are locked * down. + * + * DF-2631: the synchronization threads must reach FROZEN + * BEFORE any xop helper column is frozen. A frozen xop + * helper stops servicing its xop queue, which starves any + * in-progress hammer2_xop_collect() issued by a sync + * thread (e.g. the start_except() ipcluster in + * hammer2_sync_slaves()); that thread then never returns + * to its main loop, never sees FREEZE, and + * hammer2_thr_freeze() below blocks forever ("h2twait"), + * hanging umount -f while it holds hammer2_mntlk. */ for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { if (pmp->pfs_hmps[i] == NULL) continue; hammer2_thr_freeze_async(&pmp->sync_thrs[i]); + } + for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { + if (pmp->pfs_hmps[i] == NULL) + continue; + hammer2_thr_freeze(&pmp->sync_thrs[i]); + } + for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { + if (pmp->pfs_hmps[i] == NULL) + continue; if (pmp->xop_groups) { for (j = 0; j < hammer2_xop_nthreads; ++j) { hammer2_thr_freeze_async( @@ -783,7 +810,6 @@ for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { if (pmp->pfs_hmps[i] == NULL) continue; - hammer2_thr_freeze(&pmp->sync_thrs[i]); if (pmp->xop_groups) { for (j = 0; j < hammer2_xop_nthreads; ++j) { hammer2_thr_freeze( @@ -1637,6 +1663,14 @@ if (pmp == NULL) return(0); + /* + * DF-2631: once unmount starts, xop helper threads must not be + * lazily re-created (hammer2_xop_start_except would otherwise + * rebuild xop_groups mid-teardown while pfsfree_scan() rips the + * cluster apart, leaving threads running in freed memory). + */ + atomic_set_int(&pmp->flags, HAMMER2_PMPF_TEARDOWN); + lockmgr(&hammer2_mntlk, LK_EXCLUSIVE); /* @@ -1951,6 +1985,7 @@ hammer2_pfs_t *pmp; struct vnode *vp; int error; + int loops; pmp = MPTOPMP(mp); if (pmp->iroot == NULL) { @@ -1960,7 +1995,19 @@ return EINVAL; } + /* + * DF-2630: A quorum failure is permanent for this pmp (e.g. + * pfs_nmasters ingested from a forged/corrupt PFS inode exceeds + * the number of attached chains). Fail immediately instead of + * re-waiting the full timeout on every lookup crossing the mount. + */ + if (pmp->flags & HAMMER2_PMPF_ROOTFAILED) { + *vpp = NULL; + return EIO; + } + error = 0; + loops = 0; hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED); while (pmp->inode_tid == 0) { @@ -2008,6 +2055,18 @@ hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED); if (error == EINTR) break; + if (hammer2_root_timeout && ++loops >= hammer2_root_timeout) { + kprintf("hammer2 (%s): root quorum cannot be reached " + "(nmasters=%d nchains=%d), failing VFS_ROOT " + "after %ds\n", + mp->mnt_stat.f_mntfromname, + pmp->pfs_nmasters, + pmp->iroot->cluster.nchains, + hammer2_root_timeout); + atomic_set_int(&pmp->flags, HAMMER2_PMPF_ROOTFAILED); + error = EIO; + break; + } } if (error) { |