--- a/sys/vfs/hammer2/hammer2.h 2026-06-29 12:51:19.000000000 +0000 +++ b/sys/vfs/hammer2/hammer2.h 2026-08-29 02:14:06.557049211 +0000 @@ -1254,6 +1254,8 @@ */ #define HAMMER2_PMPF_SPMP 0x00000001 #define HAMMER2_PMPF_EMERG 0x00000002 /* Emergency delete mode */ +#define HAMMER2_PMPF_ROOTFAILED 0x00000004 /* root quorum unreachable */ +#define HAMMER2_PMPF_TEARDOWN 0x00000008 /* unmount teardown in progress */ #define HAMMER2_DIRTYCHAIN_WAITING 0x80000000 #define HAMMER2_DIRTYCHAIN_MASK 0x7FFFFFFF @@ -1408,6 +1410,7 @@ extern int hammer2_aux_flags; extern int hammer2_debug; +extern int hammer2_xop_collect_timeout; extern int hammer2_xop_nthreads; extern int hammer2_xop_sgroups; extern int hammer2_xop_xgroups; --- a/sys/vfs/hammer2/hammer2_admin.c 2026-06-29 12:51:19.000000000 +0000 +++ b/sys/vfs/hammer2/hammer2_admin.c 2026-08-29 05:01:32.105817956 +0000 @@ -488,8 +488,32 @@ ip1 = xop->ip1; pmp = ip1->pmp; - if (pmp->has_xop_threads == 0) + if (pmp->has_xop_threads == 0) { + /* + * DF-2631: never (re-)create helper threads once the pmp + * is in unmount teardown. Doing so would rebuild + * xop_groups mid-teardown while pfsfree_scan() rips the + * cluster apart, leaving worker threads running inside + * memory that hammer2_xop_helper_cleanup() already freed + * (observed as h2xop survivors and a delayed page fault + * in hammer2_primary_xops_thread). + * + * Feed an immediate EOF + error for every node instead so + * the frontend's hammer2_xop_collect() fails cleanly in + * one pass (CITEM_NULL + error) rather than waiting for a + * quorum that can never form. + */ + if (pmp->flags & HAMMER2_PMPF_TEARDOWN) { + for (i = 0; i < xop->cluster.nchains; ++i) { + if (i != notidx) { + hammer2_xop_feed(xop, NULL, i, + HAMMER2_ERROR_EIO); + } + } + return; + } hammer2_xop_helper_create(pmp); + } /* * The sequencer assigns a worker thread to the XOP. @@ -891,11 +915,13 @@ hammer2_chain_t *chain; hammer2_key_t lokey; uint64_t mask; + int64_t deadline; int error; int keynull; int adv; /* advance the element */ int i; + deadline = 0; loop: /* * First loop tries to advance pieces of the cluster which @@ -993,7 +1019,47 @@ tsleep_interlock(xop, 0); if (atomic_cmpset_64(&xop->run_mask, mask, mask | HAMMER2_XOPMASK_WAIT)) { - tsleep(xop, PINTERLOCKED, "h2coll", hz*60); + /* + * DF-2631: bound the total wait. Some situations + * can never satisfy quorum for this collect: a + * synchronization thread collects a xop started + * with hammer2_xop_start_except() on its own + * cluster index (that column never feeds and is + * counted as 'umasters' forever), and during + * unmount xop helper columns can be deleted or + * frozen while a collect is still pending. The + * old code looped here forever with no failure + * path, wedging VFS_ROOT, the sync threads (which + * then never observe FREEZE/STOP) and umount -f + * (hammer2_thr_wait, "h2twait"). + */ + if (hammer2_xop_collect_timeout == 0 || + (xop->cluster.pmp && + (xop->cluster.pmp->flags & + HAMMER2_PMPF_SPMP))) { + /* + * Legacy behavior (also used for the + * super-root pmp, whose synchronization + * threads legitimately run long scans + * that must not be aborted). + */ + tsleep(xop, PINTERLOCKED, "h2coll", hz*60); + } else { + /* + * Elapsed-time bound (NOT event-counted): + * continuous feed events from a healthy + * column must not starve this timeout when + * another column can never contribute + * (start_except self-exclusion, frozen or + * deleted helpers). + */ + if (deadline == 0) + deadline = ticks + + hammer2_xop_collect_timeout * hz; + tsleep(xop, PINTERLOCKED, "h2coll", hz); + if ((int64_t)ticks - deadline >= 0) + return HAMMER2_ERROR_EIO; + } } goto loop; } --- a/sys/vfs/hammer2/hammer2_vfsops.c 2026-06-29 12:51:19.000000000 +0000 +++ b/sys/vfs/hammer2/hammer2_vfsops.c 2026-08-29 02:14:06.569049061 +0000 @@ -112,6 +112,14 @@ SYSCTL_INT(_vfs_hammer2, OID_AUTO, supported_version, CTLFLAG_RD, &hammer2_supported_version, 0, ""); +int hammer2_xop_collect_timeout = 60; /* secs; 0 = forever */ +SYSCTL_INT(_vfs_hammer2, OID_AUTO, xop_collect_timeout, CTLFLAG_RW, + &hammer2_xop_collect_timeout, 0, + "max seconds one xop collect waits for quorum (0 = forever)"); +static int hammer2_root_timeout = 60; /* secs; 0 = forever (legacy) */ +SYSCTL_INT(_vfs_hammer2, OID_AUTO, root_timeout, CTLFLAG_RW, + &hammer2_root_timeout, 0, + "max seconds VFS_ROOT waits for cluster quorum (0 = forever)"); SYSCTL_INT(_vfs_hammer2, OID_AUTO, aux_flags, CTLFLAG_RW, &hammer2_aux_flags, 0, ""); SYSCTL_INT(_vfs_hammer2, OID_AUTO, debug, CTLFLAG_RW, @@ -768,11 +776,30 @@ /* * Make sure all synchronization threads are locked * down. + * + * DF-2631: the synchronization threads must reach FROZEN + * BEFORE any xop helper column is frozen. A frozen xop + * helper stops servicing its xop queue, which starves any + * in-progress hammer2_xop_collect() issued by a sync + * thread (e.g. the start_except() ipcluster in + * hammer2_sync_slaves()); that thread then never returns + * to its main loop, never sees FREEZE, and + * hammer2_thr_freeze() below blocks forever ("h2twait"), + * hanging umount -f while it holds hammer2_mntlk. */ for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { if (pmp->pfs_hmps[i] == NULL) continue; hammer2_thr_freeze_async(&pmp->sync_thrs[i]); + } + for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { + if (pmp->pfs_hmps[i] == NULL) + continue; + hammer2_thr_freeze(&pmp->sync_thrs[i]); + } + for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { + if (pmp->pfs_hmps[i] == NULL) + continue; if (pmp->xop_groups) { for (j = 0; j < hammer2_xop_nthreads; ++j) { hammer2_thr_freeze_async( @@ -783,7 +810,6 @@ for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { if (pmp->pfs_hmps[i] == NULL) continue; - hammer2_thr_freeze(&pmp->sync_thrs[i]); if (pmp->xop_groups) { for (j = 0; j < hammer2_xop_nthreads; ++j) { hammer2_thr_freeze( @@ -1637,6 +1663,14 @@ if (pmp == NULL) return(0); + /* + * DF-2631: once unmount starts, xop helper threads must not be + * lazily re-created (hammer2_xop_start_except would otherwise + * rebuild xop_groups mid-teardown while pfsfree_scan() rips the + * cluster apart, leaving threads running in freed memory). + */ + atomic_set_int(&pmp->flags, HAMMER2_PMPF_TEARDOWN); + lockmgr(&hammer2_mntlk, LK_EXCLUSIVE); /* @@ -1951,6 +1985,7 @@ hammer2_pfs_t *pmp; struct vnode *vp; int error; + int loops; pmp = MPTOPMP(mp); if (pmp->iroot == NULL) { @@ -1960,7 +1995,19 @@ return EINVAL; } + /* + * DF-2630: A quorum failure is permanent for this pmp (e.g. + * pfs_nmasters ingested from a forged/corrupt PFS inode exceeds + * the number of attached chains). Fail immediately instead of + * re-waiting the full timeout on every lookup crossing the mount. + */ + if (pmp->flags & HAMMER2_PMPF_ROOTFAILED) { + *vpp = NULL; + return EIO; + } + error = 0; + loops = 0; hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED); while (pmp->inode_tid == 0) { @@ -2008,6 +2055,18 @@ hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_SHARED); if (error == EINTR) break; + if (hammer2_root_timeout && ++loops >= hammer2_root_timeout) { + kprintf("hammer2 (%s): root quorum cannot be reached " + "(nmasters=%d nchains=%d), failing VFS_ROOT " + "after %ds\n", + mp->mnt_stat.f_mntfromname, + pmp->pfs_nmasters, + pmp->iroot->cluster.nchains, + hammer2_root_timeout); + atomic_set_int(&pmp->flags, HAMMER2_PMPF_ROOTFAILED); + error = EIO; + break; + } } if (error) {