DF-2712 / fix.diff
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 | --- sys/kern/usched_bsd4.c 2026-06-29 12:51:19.000000000 +0000 +++ /tmp/opencode/df2712fix/usched_bsd4.c 2026-10-20 05:58:57.298608580 +0000 @@ -102,8 +102,16 @@ static void bsd4_yield(struct lwp *lp); static void bsd4_need_user_resched_remote(void *dummy); static int bsd4_batchy_looser_pri_test(struct lwp* lp); -static struct lwp *bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp); -static void bsd4_kick_helper(struct lwp *lp); +struct bsd4_kickreq { + globaldata_t gd; + int pri; +}; +#define BSD4_KICK_MAX 32 + +static struct lwp *bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp, + struct bsd4_kickreq *kr, int *nk); +static void bsd4_record_kick(struct bsd4_kickreq *kr, int *nk, struct lwp *lp); +static void bsd4_issue_kicks(struct bsd4_kickreq *kr, int nk); static struct lwp *bsd4_chooseproc_locked(struct lwp *chklp); static void bsd4_remrunqueue_locked(struct lwp *lp); static void bsd4_setrunqueue_locked(struct lwp *lp); @@ -527,12 +535,15 @@ bsd4_pcpu_t dd = &bsd4_pcpu[gd->gd_cpuid]; struct lwp *nlp; int cpuid = gd->gd_cpuid; + struct bsd4_kickreq kr[BSD4_KICK_MAX]; + int nk = 0; crit_enter_gd(gd); spin_lock(&bsd4_spin); if(usched_bsd4_cache_coherent) - nlp = bsd4_chooseproc_locked_cache_coherent(dd->uschedcp); + nlp = bsd4_chooseproc_locked_cache_coherent(dd->uschedcp, + kr, &nk); else nlp = bsd4_chooseproc_locked(dd->uschedcp); @@ -556,6 +567,7 @@ } else { spin_unlock(&bsd4_spin); } + bsd4_issue_kicks(kr, nk); #if 0 } else if (bsd4_runqcount && CPUMASK_TESTBIT(bsd4_rdyprocmask, cpuid)) { @@ -1413,7 +1425,8 @@ */ static struct lwp * -bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp) +bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp, + struct bsd4_kickreq *kr, int *nk) { struct lwp *lp; struct rq *q; @@ -1470,7 +1483,7 @@ /* * No more left and we didn't reach the checks limit. */ - bsd4_kick_helper(min_level_lwp); + bsd4_record_kick(kr, nk, min_level_lwp); return NULL; } lp = TAILQ_FIRST(q); @@ -1480,7 +1493,7 @@ * Limit the number of checks/queue to a configurable value to * minimize the contention (we are in a locked region */ - while (checks < usched_bsd4_queue_checks) { + while (checks < imin(imax(usched_bsd4_queue_checks, 1), BSD4_KICK_MAX - 2)) { if (CPUMASK_TESTMASK(lp->lwp_cpumask, cpumask) == 0 || (CPUMASK_TESTMASK(siblings, lp->lwp_thread->td_gd->gd_cpumask) == 0 && @@ -1509,14 +1522,14 @@ if (level < min_level || (level == min_level && min_level_lwp && lp->lwp_priority < min_level_lwp->lwp_priority)) { - bsd4_kick_helper(min_level_lwp); + bsd4_record_kick(kr, nk, min_level_lwp); min_level_lwp = lp; min_level = level; min_q = q; min_which = which; min_pri = pri; } else { - bsd4_kick_helper(lp); + bsd4_record_kick(kr, nk, lp); } lp = TAILQ_NEXT(lp, lwp_procq); if (lp == NULL) { @@ -1558,7 +1571,7 @@ */ if (chklp) { if (chklp->lwp_priority < lp->lwp_priority + PPQ) { - bsd4_kick_helper(lp); + bsd4_record_kick(kr, nk, lp); return(NULL); } } @@ -1580,23 +1593,28 @@ } /* - * If we aren't willing to schedule a ready process on our cpu, give it's + * If we aren't willing to schedule a ready process on our cpu, give its * target cpu a kick rather than wait for the next tick. * + * DF-2712: IPIs/wakeups must NOT be issued while bsd4_spin is held. + * lwkt_send_ipiq3() liveloops with interrupts enabled when the per-cpu + * ipiq FIFO to the target is over half full (lwkt_ipiq.c:225,244); doing + * that under bsd4_spin wedges every cpu spinning on the lock. So we + * RECORD the kick target under the spin (cheap atomic ops only) and + * ISSUE the IPI/wakeup after the caller releases bsd4_spin. + * * Called with bsd4_spin held. */ static void -bsd4_kick_helper(struct lwp *lp) +bsd4_record_kick(struct bsd4_kickreq *kr, int *nk, struct lwp *lp) { globaldata_t gd; - bsd4_pcpu_t dd; cpumask_t tmpmask; if (lp == NULL) return; gd = lp->lwp_thread->td_gd; - dd = &bsd4_pcpu[gd->gd_cpuid]; tmpmask = smp_active_mask; CPUMASK_ANDMASK(tmpmask, usched_global_cpumask); @@ -1604,13 +1622,36 @@ CPUMASK_ANDMASK(tmpmask, gd->gd_cpumask); if (CPUMASK_TESTZERO(tmpmask)) return; + if (*nk >= BSD4_KICK_MAX) + return; ++usched_bsd4_kicks; ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, gd->gd_cpuid); - if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) { - lwkt_send_ipiq(gd, bsd4_need_user_resched_remote, NULL); - } else { - wakeup(dd->helper_thread); + kr[*nk].gd = gd; + kr[*nk].pri = lp->lwp_priority; /* snapshot under spin */ + ++*nk; +} + +/* + * Issue kicks recorded by bsd4_record_kick(). Must be called WITHOUT + * bsd4_spin. The target priority is re-checked at issue time; a + * spurious kick is harmless (same as the old under-spin behavior). + */ +static +void +bsd4_issue_kicks(struct bsd4_kickreq *kr, int nk) +{ + bsd4_pcpu_t dd; + int i; + + for (i = 0; i < nk; ++i) { + dd = &bsd4_pcpu[kr[i].gd->gd_cpuid]; + if ((dd->upri & ~PPQMASK) > (kr[i].pri & ~PPQMASK)) { + lwkt_send_ipiq(kr[i].gd, + bsd4_need_user_resched_remote, NULL); + } else { + wakeup(dd->helper_thread); + } } } |