DragonFlyBSD Kernel Audit
DF-2245 / fix.diff
← back to finding ↓ download raw
diff --git a/sys/libkern/qsort.c b/sys/libkern/qsort.c
--- a/sys/libkern/qsort.c
+++ b/sys/libkern/qsort.c
@@ -170,16 +170,45 @@
 	vecswap(a, pb - r, r);
 	r = min(pd - pc, pn - pd - es);
 	vecswap(pb, pn - r, r);
-	if ((r = pb - pa) > es)
+	/*
+	 * DF-2245: recurse on the SMALLER partition and iterate on the
+	 * larger one.  This bounds the recursion depth to ceil(log2(n))
+	 * regardless of how adversarial the input is, eliminating the
+	 * O(n)-depth kernel stack overflow of the previous "always recurse
+	 * on the left partition" code on a median-of-3 killer input.
+	 *
+	 * "<" partition: [a, a + (pb-pa))    count = (pb-pa)/es
+	 * ">" partition: [pn-(pd-pc), pn)    count = (pd-pc)/es
+	 */
+	{
+		char *small_a, *large_a;
+		size_t small_n, large_n;
+
+		if ((pb - pa) < (pd - pc)) {
+			small_a = a;
+			small_n = (pb - pa) / es;
+			large_a = pn - (pd - pc);
+			large_n = (pd - pc) / es;
+		} else {
+			small_a = pn - (pd - pc);
+			small_n = (pd - pc) / es;
+			large_a = a;
+			large_n = (pb - pa) / es;
+		}
+
+		/* Recurse on the smaller partition (depth bounded to O(log n)). */
+		if (small_n > 1)
 #ifdef	I_AM_KQSORT_R
-		kqsort_r(a, r / es, es, thunk, cmp);
+			kqsort_r(small_a, small_n, es, thunk, cmp);
 #else
-		kqsort(a, r / es, es, cmp);
+			kqsort(small_a, small_n, es, cmp);
 #endif
-	if ((r = pd - pc) > es) {
-		/* Iterate rather than recurse to save stack space */
-		a = pn - r;
-		n = r / es;
-		goto loop;
+
+		/* Iterate on the larger partition. */
+		if (large_n > 1) {
+			a = large_a;
+			n = large_n;
+			goto loop;
+		}
 	}
 }