--- sys/kern/usched_bsd4.c 2026-06-29 12:51:19.000000000 +0000 +++ /tmp/opencode/df2712fix/usched_bsd4.c 2026-10-20 05:58:57.298608580 +0000 @@ -102,8 +102,16 @@ static void bsd4_yield(struct lwp *lp); static void bsd4_need_user_resched_remote(void *dummy); static int bsd4_batchy_looser_pri_test(struct lwp* lp); -static struct lwp *bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp); -static void bsd4_kick_helper(struct lwp *lp); +struct bsd4_kickreq { + globaldata_t gd; + int pri; +}; +#define BSD4_KICK_MAX 32 + +static struct lwp *bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp, + struct bsd4_kickreq *kr, int *nk); +static void bsd4_record_kick(struct bsd4_kickreq *kr, int *nk, struct lwp *lp); +static void bsd4_issue_kicks(struct bsd4_kickreq *kr, int nk); static struct lwp *bsd4_chooseproc_locked(struct lwp *chklp); static void bsd4_remrunqueue_locked(struct lwp *lp); static void bsd4_setrunqueue_locked(struct lwp *lp); @@ -527,12 +535,15 @@ bsd4_pcpu_t dd = &bsd4_pcpu[gd->gd_cpuid]; struct lwp *nlp; int cpuid = gd->gd_cpuid; + struct bsd4_kickreq kr[BSD4_KICK_MAX]; + int nk = 0; crit_enter_gd(gd); spin_lock(&bsd4_spin); if(usched_bsd4_cache_coherent) - nlp = bsd4_chooseproc_locked_cache_coherent(dd->uschedcp); + nlp = bsd4_chooseproc_locked_cache_coherent(dd->uschedcp, + kr, &nk); else nlp = bsd4_chooseproc_locked(dd->uschedcp); @@ -556,6 +567,7 @@ } else { spin_unlock(&bsd4_spin); } + bsd4_issue_kicks(kr, nk); #if 0 } else if (bsd4_runqcount && CPUMASK_TESTBIT(bsd4_rdyprocmask, cpuid)) { @@ -1413,7 +1425,8 @@ */ static struct lwp * -bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp) +bsd4_chooseproc_locked_cache_coherent(struct lwp *chklp, + struct bsd4_kickreq *kr, int *nk) { struct lwp *lp; struct rq *q; @@ -1470,7 +1483,7 @@ /* * No more left and we didn't reach the checks limit. */ - bsd4_kick_helper(min_level_lwp); + bsd4_record_kick(kr, nk, min_level_lwp); return NULL; } lp = TAILQ_FIRST(q); @@ -1480,7 +1493,7 @@ * Limit the number of checks/queue to a configurable value to * minimize the contention (we are in a locked region */ - while (checks < usched_bsd4_queue_checks) { + while (checks < imin(imax(usched_bsd4_queue_checks, 1), BSD4_KICK_MAX - 2)) { if (CPUMASK_TESTMASK(lp->lwp_cpumask, cpumask) == 0 || (CPUMASK_TESTMASK(siblings, lp->lwp_thread->td_gd->gd_cpumask) == 0 && @@ -1509,14 +1522,14 @@ if (level < min_level || (level == min_level && min_level_lwp && lp->lwp_priority < min_level_lwp->lwp_priority)) { - bsd4_kick_helper(min_level_lwp); + bsd4_record_kick(kr, nk, min_level_lwp); min_level_lwp = lp; min_level = level; min_q = q; min_which = which; min_pri = pri; } else { - bsd4_kick_helper(lp); + bsd4_record_kick(kr, nk, lp); } lp = TAILQ_NEXT(lp, lwp_procq); if (lp == NULL) { @@ -1558,7 +1571,7 @@ */ if (chklp) { if (chklp->lwp_priority < lp->lwp_priority + PPQ) { - bsd4_kick_helper(lp); + bsd4_record_kick(kr, nk, lp); return(NULL); } } @@ -1580,23 +1593,28 @@ } /* - * If we aren't willing to schedule a ready process on our cpu, give it's + * If we aren't willing to schedule a ready process on our cpu, give its * target cpu a kick rather than wait for the next tick. * + * DF-2712: IPIs/wakeups must NOT be issued while bsd4_spin is held. + * lwkt_send_ipiq3() liveloops with interrupts enabled when the per-cpu + * ipiq FIFO to the target is over half full (lwkt_ipiq.c:225,244); doing + * that under bsd4_spin wedges every cpu spinning on the lock. So we + * RECORD the kick target under the spin (cheap atomic ops only) and + * ISSUE the IPI/wakeup after the caller releases bsd4_spin. + * * Called with bsd4_spin held. */ static void -bsd4_kick_helper(struct lwp *lp) +bsd4_record_kick(struct bsd4_kickreq *kr, int *nk, struct lwp *lp) { globaldata_t gd; - bsd4_pcpu_t dd; cpumask_t tmpmask; if (lp == NULL) return; gd = lp->lwp_thread->td_gd; - dd = &bsd4_pcpu[gd->gd_cpuid]; tmpmask = smp_active_mask; CPUMASK_ANDMASK(tmpmask, usched_global_cpumask); @@ -1604,13 +1622,36 @@ CPUMASK_ANDMASK(tmpmask, gd->gd_cpumask); if (CPUMASK_TESTZERO(tmpmask)) return; + if (*nk >= BSD4_KICK_MAX) + return; ++usched_bsd4_kicks; ATOMIC_CPUMASK_NANDBIT(bsd4_rdyprocmask, gd->gd_cpuid); - if ((dd->upri & ~PPQMASK) > (lp->lwp_priority & ~PPQMASK)) { - lwkt_send_ipiq(gd, bsd4_need_user_resched_remote, NULL); - } else { - wakeup(dd->helper_thread); + kr[*nk].gd = gd; + kr[*nk].pri = lp->lwp_priority; /* snapshot under spin */ + ++*nk; +} + +/* + * Issue kicks recorded by bsd4_record_kick(). Must be called WITHOUT + * bsd4_spin. The target priority is re-checked at issue time; a + * spurious kick is harmless (same as the old under-spin behavior). + */ +static +void +bsd4_issue_kicks(struct bsd4_kickreq *kr, int nk) +{ + bsd4_pcpu_t dd; + int i; + + for (i = 0; i < nk; ++i) { + dd = &bsd4_pcpu[kr[i].gd->gd_cpuid]; + if ((dd->upri & ~PPQMASK) > (kr[i].pri & ~PPQMASK)) { + lwkt_send_ipiq(kr[i].gd, + bsd4_need_user_resched_remote, NULL); + } else { + wakeup(dd->helper_thread); + } } }