⬢ DragonFlyBSD Kernel Audit
DF-2254 / fix.diff
← back to finding ↓ download raw
--- a/sys/libkern/qsort.c	2026-10-21 04:27:18.855407376 +0000
+++ b/sys/libkern/qsort.c	2026-10-21 04:27:18.855407376 +0000
@@ -3,16 +3,38 @@
 	vecswap(a, pb - r, r);
 	r = min(pd - pc, pn - pd - es);
 	vecswap(pb, pn - r, r);
-	if ((r = pb - pa) > es)
+
+	/*
+	 * DF-2254 / DF-2245: recurse on the SMALLER partition and iterate
+	 * on the larger one.  Previously the left ("< pivot") partition
+	 * was always the recursed one, so an adversarial input gave
+	 * O(n) recursion depth and overflowed the kernel stack.  This
+	 * bounds the depth at O(log2 n) for any input.
+	 */
+	if ((pb - pa) <= (pd - pc)) {
+		/* left side smaller: recurse left, iterate right */
+		if ((r = pb - pa) > es)
 #ifdef	I_AM_KQSORT_R
-		kqsort_r(a, r / es, es, thunk, cmp);
+			kqsort_r(a, r / es, es, thunk, cmp);
 #else
-		kqsort(a, r / es, es, cmp);
+			kqsort(a, r / es, es, cmp);
 #endif
-	if ((r = pd - pc) > es) {
-		/* Iterate rather than recurse to save stack space */
-		a = pn - r;
-		n = r / es;
-		goto loop;
+		if ((r = pd - pc) > es) {
+			a = pn - r;
+			n = r / es;
+			goto loop;
+		}
+	} else {
+		/* right side smaller: recurse right, iterate left */
+		if ((r = pd - pc) > es)
+#ifdef	I_AM_KQSORT_R
+			kqsort_r(pn - r, r / es, es, thunk, cmp);
+#else
+			kqsort(pn - r, r / es, es, cmp);
+#endif
+		if ((r = pb - pa) > es) {
+			n = r / es;
+			goto loop;
+		}
 	}
 }
\ No newline at end of file