diff --git a/sys/libkern/qsort.c b/sys/libkern/qsort.c --- a/sys/libkern/qsort.c +++ b/sys/libkern/qsort.c @@ -170,16 +170,45 @@ vecswap(a, pb - r, r); r = min(pd - pc, pn - pd - es); vecswap(pb, pn - r, r); - if ((r = pb - pa) > es) + /* + * DF-2245: recurse on the SMALLER partition and iterate on the + * larger one. This bounds the recursion depth to ceil(log2(n)) + * regardless of how adversarial the input is, eliminating the + * O(n)-depth kernel stack overflow of the previous "always recurse + * on the left partition" code on a median-of-3 killer input. + * + * "<" partition: [a, a + (pb-pa)) count = (pb-pa)/es + * ">" partition: [pn-(pd-pc), pn) count = (pd-pc)/es + */ + { + char *small_a, *large_a; + size_t small_n, large_n; + + if ((pb - pa) < (pd - pc)) { + small_a = a; + small_n = (pb - pa) / es; + large_a = pn - (pd - pc); + large_n = (pd - pc) / es; + } else { + small_a = pn - (pd - pc); + small_n = (pd - pc) / es; + large_a = a; + large_n = (pb - pa) / es; + } + + /* Recurse on the smaller partition (depth bounded to O(log n)). */ + if (small_n > 1) #ifdef I_AM_KQSORT_R - kqsort_r(a, r / es, es, thunk, cmp); + kqsort_r(small_a, small_n, es, thunk, cmp); #else - kqsort(a, r / es, es, cmp); + kqsort(small_a, small_n, es, cmp); #endif - if ((r = pd - pc) > es) { - /* Iterate rather than recurse to save stack space */ - a = pn - r; - n = r / es; - goto loop; + + /* Iterate on the larger partition. */ + if (large_n > 1) { + a = large_a; + n = large_n; + goto loop; + } } }