/* * nfereplica.c - deterministic replica of if_nfe.c's sc_tick_ch callout * protocol (nfe(4) instance of the DF-3715(em) / DF-4025(re) / DF-4046(stge) * AB-BA family). * * if_nfe.c arms ONE callout with plain callout_init() -- the NON-MPSAFE * variant (if_nfe.c:636, `callout_init(&sc->sc_tick_ch);`) -- whose handler * takes the driver serializer by hand and re-arms UNCONDITIONALLY while * holding it: * * nfe_tick() if_nfe.c:2181-2193 * :2187 lwkt_serialize_enter (BLOCKING acquire) * :2189 mii_tick(mii) * :2190 callout_reset(&sc->sc_tick_ch, hz, nfe_tick, sc) * <-- UNCONDITIONAL re-arm, no IFF_RUNNING gate, * under the serializer * :2192 lwkt_serialize_exit * * nfe_stop() (if_nfe.c:1585-1640) tears it down with * callout_stop(&sc->sc_tick_ch) at if_nfe.c:1593 -- the FIRST action after * ASSERT_SERIALIZED (if_nfe.c:1591) -- while HOLDING that serializer. * Every nfe_stop caller holds it: nfe_detach :669-672, nfe_shutdown * :705-708, nfe_suspend :716-719, nfe_ioctl SIOCSIFFLAGS :1025-1027, * nfe_init :1446 (nfe_stop is the first statement, :1446) -- and nfe_init * is reachable from nfe_ioctl SIOCSIFMTU :1005-1007, SIOCSIFCAP :1049-1051, * nfe_resume :730-732, and nfe_watchdog :1431. * * DragonFly callout_stop() is synchronous (sys/kern/kern_timeout.c:1091-1093 * -> _callout_cancel_or_stop(..., sync=1); ssleep "costp" at :920): it waits * for an INPROG callback to finish. A nfe_tick dequeued by a softclock * thread that is parked in lwkt_serialize_enter therefore deadlocks against * the stopper (AB-BA): both threads wedge permanently, the interface * serializer is never released (all later ioctls/nfe_start on nfex block), * and the affected softclock stops delivering ALL callouts on that CPU. * nfe_tick's unconditional re-arm (:2190) additionally means nothing in the * driver can prevent the parked callback from re-arming once it runs. * * Sysctls: * kern.nfereplica.arm =1 arm the nfe_tick replica (short period for a * deterministic window; real driver uses hz) * kern.nfereplica.stop =1 nfe_stop replica: hold serializer, wait until * the handler is parked, callout_stop() * -> if the AB-BA analysis is right this NEVER * returns. */ #include #include #include #include #include #include #include static struct lwkt_serialize ser; /* ifp->if_serializer replica */ static struct callout tick_c; /* sc->sc_tick_ch replica */ static volatile int running; /* IFF_RUNNING replica */ static volatile int tick_fired, tick_ran; static void tick_handler(void *); static void tick_handler(void *arg) /* if_nfe.c:2181-2193 shape */ { tick_fired++; /* pre-acquire */ lwkt_serialize_enter(&ser); /* if_nfe.c:2187 (BLOCKING) */ tick_ran++; /* mii_tick(mii) elided */ callout_reset(&tick_c, 1, tick_handler, arg); /* :2190, UNCONDITIONAL */ lwkt_serialize_exit(&ser); /* if_nfe.c:2192 */ } static int arm_sysctl(SYSCTL_HANDLER_ARGS) { int error, v = 0; error = sysctl_handle_int(oidp, &v, 0, req); if (error || req->newptr == NULL) return error; if (v == 1) { tick_fired = tick_ran = 0; running = 1; /* if_nfe.c:1570 callout_reset(&sc->sc_tick_ch, hz, nfe_tick, sc) */ callout_reset(&tick_c, 1, tick_handler, NULL); kprintf("nfereplica: armed (nfe_tick shape, 1-tick period; " "nfe uses hz)\n"); } return 0; } static int stop_sysctl(SYSCTL_HANDLER_ARGS) { int error, v = 0; error = sysctl_handle_int(oidp, &v, 0, req); if (error || req->newptr == NULL) return error; if (v != 1) return 0; /* * nfe_stop() shape - if_nfe.c:1591 ASSERT_SERIALIZED, * :1593 callout_stop(&sc->sc_tick_ch), :1596 clear IFF_RUNNING, in * that order, all while HOLDING the serializer. * * NOTE: the park-wait below deliberately uses a BUSY wait (DELAY), * not tsleep: a tsleep timeout is delivered by the softclock of the * stopper's own CPU, which is exactly the thread the tick replica * parks -- an early run of this replica showed the stopper stuck in * its own parking aid (wchan "nfetkwt") instead of reaching * callout_stop(). The busy wait keeps the observation pinned to the * real defect: callout_stop()'s synchronous 'costp' wait * (kern_timeout.c:920). */ lwkt_serialize_enter(&ser); while (tick_ran == tick_fired) /* wait until handler is parked */ DELAY(1000); kprintf("nfereplica: nfe_tick parked on our serializer " "(fired=%d ran=%d) - entering callout_stop()\n", tick_fired, tick_ran); kprintf("nfereplica: IF THIS IS THE LAST nfereplica LINE, " "AB-BA DEADLOCK IS PROVEN\n"); callout_stop(&tick_c); /* if_nfe.c:1593 */ running = 0; /* if_nfe.c:1596 */ kprintf("nfereplica: callout_stop returned (NO deadlock)\n"); lwkt_serialize_exit(&ser); return 0; } static SYSCTL_NODE(_kern, OID_AUTO, nfereplica, CTLFLAG_RW, 0, "nfe(4) callout replica"); SYSCTL_PROC(_kern_nfereplica, OID_AUTO, arm, CTLTYPE_INT | CTLFLAG_RW, 0, 0, arm_sysctl, "I", "arm nfe_tick replica"); SYSCTL_PROC(_kern_nfereplica, OID_AUTO, stop, CTLTYPE_INT | CTLFLAG_RW, 0, 0, stop_sysctl, "I", "nfe_stop replica vs nfe_tick (AB-BA)"); static int nfereplica_load(struct module *m, int what, void *arg) { switch (what) { case MOD_LOAD: lwkt_serialize_init(&ser); callout_init(&tick_c); /* if_nfe.c:636: plain NON-MP init */ running = 0; kprintf("nfereplica: loaded (hz=%d)\n", hz); return 0; case MOD_UNLOAD: callout_stop_async(&tick_c); return 0; } return 0; } static moduledata_t nfereplica_mod = { "nfereplica", nfereplica_load, NULL }; DECLARE_MODULE(nfereplica, nfereplica_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);