--- a/sys/vfs/hammer2/hammer2.h 2026-06-29 12:51:19.000000000 +0000 +++ b/sys/vfs/hammer2/hammer2.h 2026-08-29 02:14:06.585048860 +0000 @@ -1254,6 +1254,7 @@ */ #define HAMMER2_PMPF_SPMP 0x00000001 #define HAMMER2_PMPF_EMERG 0x00000002 /* Emergency delete mode */ +#define HAMMER2_PMPF_TEARDOWN 0x00000008 /* unmount teardown in progress */ #define HAMMER2_DIRTYCHAIN_WAITING 0x80000000 #define HAMMER2_DIRTYCHAIN_MASK 0x7FFFFFFF @@ -1408,6 +1409,7 @@ extern int hammer2_aux_flags; extern int hammer2_debug; +extern int hammer2_xop_collect_timeout; extern int hammer2_xop_nthreads; extern int hammer2_xop_sgroups; extern int hammer2_xop_xgroups; --- a/sys/vfs/hammer2/hammer2_admin.c 2026-06-29 12:51:19.000000000 +0000 +++ b/sys/vfs/hammer2/hammer2_admin.c 2026-08-29 05:01:32.105817956 +0000 @@ -488,8 +488,32 @@ ip1 = xop->ip1; pmp = ip1->pmp; - if (pmp->has_xop_threads == 0) + if (pmp->has_xop_threads == 0) { + /* + * DF-2631: never (re-)create helper threads once the pmp + * is in unmount teardown. Doing so would rebuild + * xop_groups mid-teardown while pfsfree_scan() rips the + * cluster apart, leaving worker threads running inside + * memory that hammer2_xop_helper_cleanup() already freed + * (observed as h2xop survivors and a delayed page fault + * in hammer2_primary_xops_thread). + * + * Feed an immediate EOF + error for every node instead so + * the frontend's hammer2_xop_collect() fails cleanly in + * one pass (CITEM_NULL + error) rather than waiting for a + * quorum that can never form. + */ + if (pmp->flags & HAMMER2_PMPF_TEARDOWN) { + for (i = 0; i < xop->cluster.nchains; ++i) { + if (i != notidx) { + hammer2_xop_feed(xop, NULL, i, + HAMMER2_ERROR_EIO); + } + } + return; + } hammer2_xop_helper_create(pmp); + } /* * The sequencer assigns a worker thread to the XOP. @@ -891,11 +915,13 @@ hammer2_chain_t *chain; hammer2_key_t lokey; uint64_t mask; + int64_t deadline; int error; int keynull; int adv; /* advance the element */ int i; + deadline = 0; loop: /* * First loop tries to advance pieces of the cluster which @@ -993,7 +1019,47 @@ tsleep_interlock(xop, 0); if (atomic_cmpset_64(&xop->run_mask, mask, mask | HAMMER2_XOPMASK_WAIT)) { - tsleep(xop, PINTERLOCKED, "h2coll", hz*60); + /* + * DF-2631: bound the total wait. Some situations + * can never satisfy quorum for this collect: a + * synchronization thread collects a xop started + * with hammer2_xop_start_except() on its own + * cluster index (that column never feeds and is + * counted as 'umasters' forever), and during + * unmount xop helper columns can be deleted or + * frozen while a collect is still pending. The + * old code looped here forever with no failure + * path, wedging VFS_ROOT, the sync threads (which + * then never observe FREEZE/STOP) and umount -f + * (hammer2_thr_wait, "h2twait"). + */ + if (hammer2_xop_collect_timeout == 0 || + (xop->cluster.pmp && + (xop->cluster.pmp->flags & + HAMMER2_PMPF_SPMP))) { + /* + * Legacy behavior (also used for the + * super-root pmp, whose synchronization + * threads legitimately run long scans + * that must not be aborted). + */ + tsleep(xop, PINTERLOCKED, "h2coll", hz*60); + } else { + /* + * Elapsed-time bound (NOT event-counted): + * continuous feed events from a healthy + * column must not starve this timeout when + * another column can never contribute + * (start_except self-exclusion, frozen or + * deleted helpers). + */ + if (deadline == 0) + deadline = ticks + + hammer2_xop_collect_timeout * hz; + tsleep(xop, PINTERLOCKED, "h2coll", hz); + if ((int64_t)ticks - deadline >= 0) + return HAMMER2_ERROR_EIO; + } } goto loop; } --- a/sys/vfs/hammer2/hammer2_vfsops.c 2026-06-29 12:51:19.000000000 +0000 +++ b/sys/vfs/hammer2/hammer2_vfsops.c 2026-08-29 02:14:06.593048758 +0000 @@ -112,6 +112,10 @@ SYSCTL_INT(_vfs_hammer2, OID_AUTO, supported_version, CTLFLAG_RD, &hammer2_supported_version, 0, ""); +int hammer2_xop_collect_timeout = 60; /* secs; 0 = forever */ +SYSCTL_INT(_vfs_hammer2, OID_AUTO, xop_collect_timeout, CTLFLAG_RW, + &hammer2_xop_collect_timeout, 0, + "max seconds one xop collect waits for quorum (0 = forever)"); SYSCTL_INT(_vfs_hammer2, OID_AUTO, aux_flags, CTLFLAG_RW, &hammer2_aux_flags, 0, ""); SYSCTL_INT(_vfs_hammer2, OID_AUTO, debug, CTLFLAG_RW, @@ -768,11 +772,30 @@ /* * Make sure all synchronization threads are locked * down. + * + * DF-2631: the synchronization threads must reach FROZEN + * BEFORE any xop helper column is frozen. A frozen xop + * helper stops servicing its xop queue, which starves any + * in-progress hammer2_xop_collect() issued by a sync + * thread (e.g. the start_except() ipcluster in + * hammer2_sync_slaves()); that thread then never returns + * to its main loop, never sees FREEZE, and + * hammer2_thr_freeze() below blocks forever ("h2twait"), + * hanging umount -f while it holds hammer2_mntlk. */ for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { if (pmp->pfs_hmps[i] == NULL) continue; hammer2_thr_freeze_async(&pmp->sync_thrs[i]); + } + for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { + if (pmp->pfs_hmps[i] == NULL) + continue; + hammer2_thr_freeze(&pmp->sync_thrs[i]); + } + for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { + if (pmp->pfs_hmps[i] == NULL) + continue; if (pmp->xop_groups) { for (j = 0; j < hammer2_xop_nthreads; ++j) { hammer2_thr_freeze_async( @@ -783,7 +806,6 @@ for (i = 0; i < HAMMER2_MAXCLUSTER; ++i) { if (pmp->pfs_hmps[i] == NULL) continue; - hammer2_thr_freeze(&pmp->sync_thrs[i]); if (pmp->xop_groups) { for (j = 0; j < hammer2_xop_nthreads; ++j) { hammer2_thr_freeze( @@ -1637,6 +1659,14 @@ if (pmp == NULL) return(0); + /* + * DF-2631: once unmount starts, xop helper threads must not be + * lazily re-created (hammer2_xop_start_except would otherwise + * rebuild xop_groups mid-teardown while pfsfree_scan() rips the + * cluster apart, leaving threads running in freed memory). + */ + atomic_set_int(&pmp->flags, HAMMER2_PMPF_TEARDOWN); + lockmgr(&hammer2_mntlk, LK_EXCLUSIVE); /*