/* * alc_callout_replica.c - deterministic replica of if_alc.c's timer protocol. * * if_alc.c arms one MP-safe callout (callout_init_mp, if_alc.c:1353): * * alc_tick() if_alc.c:3408-3436 * lwkt_serialize_enter(ifp->if_serializer) BLOCKING (:3414) * mii_tick / alc_stats_update / alc_txeof / alc_watchdog * callout_reset(&sc->alc_tick_ch, hz, alc_tick, sc) (:3428) * lwkt_serialize_exit (:3435) * * alc_stop() if_alc.c:3957-4021 (ASSERT_SERIALIZED :3965) * callout_stop(&sc->alc_tick_ch) (:3973) * * alc_stop() is called with the serializer held from: * alc_ioctl SIOCSIFFLAGS down (:2826), alc_init (:3580, every reinit: * ifconfig up, watchdog :2774/:2779, alc_intr DMA-error :3103), * alc_detach (:1616), alc_suspend (:2437). * * DragonFly callout_stop() is the SYNCHRONOUS variant * (sys/kern/kern_timeout.c:1091 -> _callout_cancel_or_stop(..., sync=1)); * when the callback is INPROG it ssleep()s on "costp" (kern_timeout.c:923) * until the callback completes. If alc_tick is INPROG and parked inside * lwkt_serialize_enter on the serializer held by the alc_stop() caller, * neither thread can make progress: PERMANENT AB-BA DEADLOCK. * * This module reproduces exactly that protocol with two callouts: * "alc tick (as shipped)" - blocking acquire + self re-arm at hz * "alc tick (fix.diff shape)" - lwkt_serialize_try + 1-tick retry * and exposes sysctls to run the interleavings: * * kern.alcreplica.arm =1 arm both replicas * kern.alcreplica.stop_try =1 hold serializer, callout_stop() the * try-based (fixed) shape -> must return, * pending=0 (validates the fix mechanism) * kern.alcreplica.stop_blk =1 hold serializer, wait for the shipped-shape * handler to be BLOCKED on it, callout_stop() * -> if the analysis is right this NEVER * returns (sysctl thread D3 wchan "costp"). */ #include #include #include #include #include #include #include static struct lwkt_serialize ser; /* == ifp->if_serializer */ static struct callout tick_blk_c; /* alc_tick as shipped */ static struct callout tick_try_c; /* alc_tick with DF-3760 fix.diff */ static volatile int blk_fired, blk_ran; static volatile int try_fired, try_ran; /* if_alc.c:3408-3436 - alc_tick() EXACT protocol (blocking) */ static void alc_tick_blk(void *arg) { blk_fired++; /* if_alc.c:3414 - BLOCKING acquire */ lwkt_serialize_enter(&ser); blk_ran++; /* mii_tick/stats/txeof/watchdog live here */ /* if_alc.c:3428 - self re-arm at hz */ callout_reset(&tick_blk_c, hz, alc_tick_blk, NULL); lwkt_serialize_exit(&ser); } /* alc_tick() with the proposed fix: try-enter, 1-tick retry */ static void alc_tick_try(void *arg) { try_fired++; if (lwkt_serialize_try(&ser)) { try_ran++; callout_reset(&tick_try_c, hz, alc_tick_try, NULL); lwkt_serialize_exit(&ser); } else { /* serializer held (e.g. by alc_stop) - retry next tick */ callout_reset(&tick_try_c, 1, alc_tick_try, NULL); } } static int arm_sysctl(SYSCTL_HANDLER_ARGS) { int error, v = 0; error = sysctl_handle_int(oidp, &v, 0, req); if (error || req->newptr == NULL) return error; if (v == 1) { blk_fired = blk_ran = try_fired = try_ran = 0; callout_reset(&tick_try_c, 1, alc_tick_try, NULL); callout_reset(&tick_blk_c, 1, alc_tick_blk, NULL); kprintf("alcreplica: armed (hz=%d)\n", hz); } return 0; } /* * Fix-shape test: alc_stop() vs the try-based alc_tick. * callout_stop() must return (handler never blocks) and unload the * racing re-arm. Validates the DF-3760 fix.diff mechanism. */ static int stop_try_sysctl(SYSCTL_HANDLER_ARGS) { int error, v = 0, tries; error = sysctl_handle_int(oidp, &v, 0, req); if (error || req->newptr == NULL) return error; if (v != 1) return 0; for (tries = 0; tries < 100; tries++) { lwkt_serialize_enter(&ser); /* as alc_stop */ if (try_fired == try_ran && callout_pending(&tick_try_c) == 0) tsleep(&try_fired, 0, "tryw", 1); kprintf("alcreplica[try]: stop (fired=%d ran=%d)\n", try_fired, try_ran); callout_stop(&tick_try_c); /* :3973 */ kprintf("alcreplica[try]: callout_stop returned, " "pending=%d\n", callout_pending(&tick_try_c)); lwkt_serialize_exit(&ser); if (callout_pending(&tick_try_c) == 0) break; tsleep(&try_ran, 0, "tryr", hz / 10); } kprintf("alcreplica[try]: RESULT pending=%d " "(0 == try-shape fully stopped, no deadlock)\n", callout_pending(&tick_try_c)); return 0; } /* * Shipped-shape test: alc_stop() vs the blocking alc_tick. * Take the serializer (as alc_stop does), wait until the handler is * parked inside lwkt_serialize_enter (fired but not ran), then * callout_stop(). If the AB-BA analysis is right this never prints * the next line - the sysctl thread AND a softclock thread are wedged * permanently (D-state, wchan "costp" == kern_timeout.c:923). */ static int stop_blk_sysctl(SYSCTL_HANDLER_ARGS) { int error, v = 0; error = sysctl_handle_int(oidp, &v, 0, req); if (error || req->newptr == NULL) return error; if (v != 1) return 0; lwkt_serialize_enter(&ser); /* as alc_stop :3965 */ while (blk_ran == blk_fired) tsleep(&blk_ran, 0, "blkw", 2); kprintf("alcreplica[blk]: alc_tick handler BLOCKED on our " "serializer (fired=%d ran=%d) - entering callout_stop()\n", blk_fired, blk_ran); kprintf("alcreplica[blk]: IF THIS IS THE LAST alcreplica LINE, " "THE if_alc.c AB-BA DEADLOCK IS PROVEN\n"); callout_stop(&tick_blk_c); /* :3973 */ kprintf("alcreplica[blk]: callout_stop returned (NO deadlock)\n"); lwkt_serialize_exit(&ser); return 0; } static SYSCTL_NODE(_kern, OID_AUTO, alcreplica, CTLFLAG_RW, 0, "alc callout replica"); SYSCTL_PROC(_kern_alcreplica, OID_AUTO, arm, CTLTYPE_INT | CTLFLAG_RW, 0, 0, arm_sysctl, "I", "arm replica callouts"); SYSCTL_PROC(_kern_alcreplica, OID_AUTO, stop_try, CTLTYPE_INT | CTLFLAG_RW, 0, 0, stop_try_sysctl, "I", "alc_stop vs try-based (fixed) alc_tick"); SYSCTL_PROC(_kern_alcreplica, OID_AUTO, stop_blk, CTLTYPE_INT | CTLFLAG_RW, 0, 0, stop_blk_sysctl, "I", "alc_stop vs shipped (blocking) alc_tick"); static int alc_replica_load(struct module *m, int what, void *arg) { switch (what) { case MOD_LOAD: lwkt_serialize_init(&ser); callout_init_mp(&tick_try_c); callout_init_mp(&tick_blk_c); kprintf("alc_callout_replica: loaded (hz=%d ncpus=%d)\n", hz, ncpus); return 0; case MOD_UNLOAD: /* only safe if the AB-BA test was not run */ callout_stop(&tick_try_c); callout_stop(&tick_blk_c); return 0; } return 0; } static moduledata_t alc_replica_mod = { "alc_callout_replica", alc_replica_load, NULL }; DECLARE_MODULE(alc_callout_replica, alc_replica_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);