--- sys/bus/u4b/controller/xhci_pci.c.orig
+++ sys/bus/u4b/controller/xhci_pci.c
@@ -352,7 +352,13 @@ xhci_pci_detach(device_t self)
 	struct xhci_softc *sc = device_get_softc(self);
 
 	/* during module unload there are lots of children leftover */
-	device_delete_children(self);
+	if (device_delete_children(self) != 0) {
+		device_printf(self,
+		    "child device failed to detach; "
+		    "refusing controller teardown\n");
+		return (EBUSY);
+	}
 
 	usb_callout_drain(&sc->sc_callout);
 	xhci_halt_controller(sc);