DF-2657 / fix2657.diff
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | diff -urN a/sys/vfs/hammer2/hammer2.h b/sys/vfs/hammer2/hammer2.h --- a/sys/vfs/hammer2/hammer2.h 2026-08-30 04:20:54.789834671 +0000 +++ b/sys/vfs/hammer2/hammer2.h 2026-08-30 04:20:54.789834671 +0000 @@ -1083,6 +1083,12 @@ #define HAMMER2_XOP_INODE_STOP 0x00000004 #define HAMMER2_XOP_VOLHDR 0x00000008 #define HAMMER2_XOP_FSSYNC 0x00000010 +/* + * XOP issued via hammer2_xop_start_except(): the excluded cluster index + * never feeds the collect, so quorum must be calculated from the nodes + * the xop was actually issued to (chk_mask), not from pfs_nmasters. + */ +#define HAMMER2_XOP_PARTQUORUM 0x00000020 /* * Device vnode management structure @@ -1913,7 +1919,7 @@ void hammer2_cluster_rehold(hammer2_cluster_t *cluster); void hammer2_cluster_lock(hammer2_cluster_t *cluster, int how); int hammer2_cluster_check(hammer2_cluster_t *cluster, hammer2_key_t lokey, - int flags); + int flags, uint64_t pmask); void hammer2_cluster_unlock(hammer2_cluster_t *cluster); void hammer2_bulkfree_init(hammer2_dev_t *hmp); diff -urN a/sys/vfs/hammer2/hammer2_admin.c b/sys/vfs/hammer2/hammer2_admin.c --- a/sys/vfs/hammer2/hammer2_admin.c 2026-08-30 04:20:54.789834671 +0000 +++ b/sys/vfs/hammer2/hammer2_admin.c 2026-08-30 04:20:54.789834671 +0000 @@ -547,11 +547,20 @@ xop->desc = desc; /* + * Quorum for start_except() XOPs must be calculated from the + * participants only (the excluded index can never vote). Used by + * hammer2_xop_collect() when calling hammer2_cluster_check(). + */ + if (notidx >= 0) + xop->flags |= HAMMER2_XOP_PARTQUORUM; + + /* * The instant xop is queued another thread can pick it off. In the * case of asynchronous ops, another thread might even finish and * deallocate it. */ hammer2_spin_ex(&pmp->xop_spin); + xop->chk_mask = 0; nchains = ip1->cluster.nchains; for (i = 0; i < nchains; ++i) { /* @@ -891,6 +900,7 @@ hammer2_chain_t *chain; hammer2_key_t lokey; uint64_t mask; + uint64_t pmask; int error; int keynull; int adv; /* advance the element */ @@ -985,7 +995,20 @@ (mask & HAMMER2_XOPMASK_ALLDONE) != HAMMER2_XOPMASK_VOP) { error = HAMMER2_ERROR_EINPROGRESS; } else { - error = hammer2_cluster_check(&xop->cluster, lokey, keynull); + /* + * For XOPs issued via hammer2_xop_start_except() the quorum + * must be calculated from the participating nodes only, + * otherwise the collect can never reach quorum and will + * retry forever (this parks the sync threads of any cluster + * whose participating masters are fewer than pfs_nmasters/2+1, + * e.g. every MASTER+MASTER cluster). + */ + if (xop->flags & HAMMER2_XOP_PARTQUORUM) + pmask = xop->chk_mask; + else + pmask = ~0ULL; + error = hammer2_cluster_check(&xop->cluster, lokey, + keynull, pmask); } if (error == HAMMER2_ERROR_EINPROGRESS) { if (flags & HAMMER2_XOP_COLLECT_NOWAIT) diff -urN a/sys/vfs/hammer2/hammer2_cluster.c b/sys/vfs/hammer2/hammer2_cluster.c --- a/sys/vfs/hammer2/hammer2_cluster.c 2026-08-30 04:20:54.789834671 +0000 +++ b/sys/vfs/hammer2/hammer2_cluster.c 2026-08-30 04:20:54.789834671 +0000 @@ -318,7 +318,8 @@ * XXX needs to handle SOFT_MASTER and SOFT_SLAVE */ int -hammer2_cluster_check(hammer2_cluster_t *cluster, hammer2_key_t key, int flags) +hammer2_cluster_check(hammer2_cluster_t *cluster, hammer2_key_t key, int flags, + uint64_t pmask) { hammer2_chain_t *chain; hammer2_chain_t *focus; @@ -330,6 +331,7 @@ int ttlslaves; int nmasters; int nmasters_keymatch; + int npmasters; int nslaves; int nquorum; int umasters; /* unknown masters (still in progress) */ @@ -344,8 +346,33 @@ /* * Calculate quorum + * + * pmask contains the cluster indices the XOP was actually issued + * to (~0 for a normal all-node XOP). When a node was excluded via + * hammer2_xop_start_except() it can never vote (its element stays + * NULL and its fifo is never fed), so it must not be counted in + * the quorum nor as an in-progress master. Otherwise, for any + * cluster whose participating masters are fewer than + * pfs_nmasters/2+1 (e.g. every 2-node MASTER+MASTER cluster, where + * the quorum of 2 can never be reached by the single remaining + * participant), the check returns EINPROGRESS forever. */ - nquorum = pmp ? pmp->pfs_nmasters / 2 + 1 : 0; + npmasters = 0; + if (pmp) { + for (i = 0; i < cluster->nchains; ++i) { + if ((pmask & (1LLU << i)) == 0) + continue; + switch (pmp->pfs_types[i]) { + case HAMMER2_PFSTYPE_MASTER: + case HAMMER2_PFSTYPE_SUPROOT: + ++npmasters; + break; + default: + break; + } + } + } + nquorum = pmp ? npmasters / 2 + 1 : 0; nflags = 0; ttlmasters = 0; ttlslaves = 0; @@ -452,6 +479,14 @@ continue; } + /* + * Non-participating nodes (the excluded index of a + * hammer2_xop_start_except() XOP) can never vote and + * must not be counted as in-progress masters. + */ + if ((pmask & (1LLU << i)) == 0) + continue; + chain = cluster->array[i].chain; error = cluster->array[i].error; diff -urN a/sys/vfs/hammer2/hammer2_synchro.c b/sys/vfs/hammer2/hammer2_synchro.c --- a/sys/vfs/hammer2/hammer2_synchro.c 2026-08-30 04:20:54.789834671 +0000 +++ b/sys/vfs/hammer2/hammer2_synchro.c 2026-08-30 04:20:54.789834671 +0000 @@ -418,7 +418,15 @@ &parent, HAMMER2_RESOLVE_ALWAYS | HAMMER2_RESOLVE_SHARED); - want_update = (chain->bref.modify_tid != sync_tid); + /* + * Forward-only synchronization: with the quorum now + * calculated from the participating (i.e. remote) + * nodes only, a diverged peer may have an OLDER tid + * than ours. Never regress our own newer state + * towards it, or a diverged 2-master cluster will + * ping-pong destructively between passes. + */ + want_update = (chain->bref.modify_tid < sync_tid); if (chain) { hammer2_chain_unlock(chain); hammer2_chain_drop(chain); |