/*
 * alc_callout_replica.c - deterministic replica of if_alc.c's timer protocol.
 *
 * if_alc.c arms one MP-safe callout (callout_init_mp, if_alc.c:1353):
 *
 *   alc_tick()  if_alc.c:3408-3436
 *       lwkt_serialize_enter(ifp->if_serializer)   BLOCKING   (:3414)
 *       mii_tick / alc_stats_update / alc_txeof / alc_watchdog
 *       callout_reset(&sc->alc_tick_ch, hz, alc_tick, sc)     (:3428)
 *       lwkt_serialize_exit                                  (:3435)
 *
 *   alc_stop()  if_alc.c:3957-4021  (ASSERT_SERIALIZED :3965)
 *       callout_stop(&sc->alc_tick_ch)                        (:3973)
 *
 * alc_stop() is called with the serializer held from:
 *       alc_ioctl SIOCSIFFLAGS down (:2826), alc_init (:3580, every reinit:
 *       ifconfig up, watchdog :2774/:2779, alc_intr DMA-error :3103),
 *       alc_detach (:1616), alc_suspend (:2437).
 *
 * DragonFly callout_stop() is the SYNCHRONOUS variant
 * (sys/kern/kern_timeout.c:1091 -> _callout_cancel_or_stop(..., sync=1));
 * when the callback is INPROG it ssleep()s on "costp" (kern_timeout.c:923)
 * until the callback completes.  If alc_tick is INPROG and parked inside
 * lwkt_serialize_enter on the serializer held by the alc_stop() caller,
 * neither thread can make progress: PERMANENT AB-BA DEADLOCK.
 *
 * This module reproduces exactly that protocol with two callouts:
 *   "alc tick (as shipped)"     - blocking acquire + self re-arm at hz
 *   "alc tick (fix.diff shape)" - lwkt_serialize_try + 1-tick retry
 * and exposes sysctls to run the interleavings:
 *
 *   kern.alcreplica.arm      =1  arm both replicas
 *   kern.alcreplica.stop_try =1  hold serializer, callout_stop() the
 *                                try-based (fixed) shape -> must return,
 *                                pending=0 (validates the fix mechanism)
 *   kern.alcreplica.stop_blk =1  hold serializer, wait for the shipped-shape
 *                                handler to be BLOCKED on it, callout_stop()
 *                                -> if the analysis is right this NEVER
 *                                returns (sysctl thread D3 wchan "costp").
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/serialize.h>
#include <sys/callout.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

static struct lwkt_serialize ser;	/* == ifp->if_serializer */
static struct callout tick_blk_c;	/* alc_tick as shipped */
static struct callout tick_try_c;	/* alc_tick with DF-3760 fix.diff */

static volatile int blk_fired, blk_ran;
static volatile int try_fired, try_ran;

/* if_alc.c:3408-3436 - alc_tick() EXACT protocol (blocking) */
static void
alc_tick_blk(void *arg)
{
	blk_fired++;
	/* if_alc.c:3414 - BLOCKING acquire */
	lwkt_serialize_enter(&ser);
	blk_ran++;		/* mii_tick/stats/txeof/watchdog live here */
	/* if_alc.c:3428 - self re-arm at hz */
	callout_reset(&tick_blk_c, hz, alc_tick_blk, NULL);
	lwkt_serialize_exit(&ser);
}

/* alc_tick() with the proposed fix: try-enter, 1-tick retry */
static void
alc_tick_try(void *arg)
{
	try_fired++;
	if (lwkt_serialize_try(&ser)) {
		try_ran++;
		callout_reset(&tick_try_c, hz, alc_tick_try, NULL);
		lwkt_serialize_exit(&ser);
	} else {
		/* serializer held (e.g. by alc_stop) - retry next tick */
		callout_reset(&tick_try_c, 1, alc_tick_try, NULL);
	}
}

static int
arm_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v == 1) {
		blk_fired = blk_ran = try_fired = try_ran = 0;
		callout_reset(&tick_try_c, 1, alc_tick_try, NULL);
		callout_reset(&tick_blk_c, 1, alc_tick_blk, NULL);
		kprintf("alcreplica: armed (hz=%d)\n", hz);
	}
	return 0;
}

/*
 * Fix-shape test: alc_stop() vs the try-based alc_tick.
 * callout_stop() must return (handler never blocks) and unload the
 * racing re-arm.  Validates the DF-3760 fix.diff mechanism.
 */
static int
stop_try_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0, tries;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	for (tries = 0; tries < 100; tries++) {
		lwkt_serialize_enter(&ser);		/* as alc_stop */
		if (try_fired == try_ran &&
		    callout_pending(&tick_try_c) == 0)
			tsleep(&try_fired, 0, "tryw", 1);
		kprintf("alcreplica[try]: stop (fired=%d ran=%d)\n",
		    try_fired, try_ran);
		callout_stop(&tick_try_c);		/* :3973 */
		kprintf("alcreplica[try]: callout_stop returned, "
		    "pending=%d\n", callout_pending(&tick_try_c));
		lwkt_serialize_exit(&ser);
		if (callout_pending(&tick_try_c) == 0)
			break;
		tsleep(&try_ran, 0, "tryr", hz / 10);
	}
	kprintf("alcreplica[try]: RESULT pending=%d "
	    "(0 == try-shape fully stopped, no deadlock)\n",
	    callout_pending(&tick_try_c));
	return 0;
}

/*
 * Shipped-shape test: alc_stop() vs the blocking alc_tick.
 * Take the serializer (as alc_stop does), wait until the handler is
 * parked inside lwkt_serialize_enter (fired but not ran), then
 * callout_stop().  If the AB-BA analysis is right this never prints
 * the next line - the sysctl thread AND a softclock thread are wedged
 * permanently (D-state, wchan "costp" == kern_timeout.c:923).
 */
static int
stop_blk_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	lwkt_serialize_enter(&ser);			/* as alc_stop :3965 */
	while (blk_ran == blk_fired)
		tsleep(&blk_ran, 0, "blkw", 2);
	kprintf("alcreplica[blk]: alc_tick handler BLOCKED on our "
	    "serializer (fired=%d ran=%d) - entering callout_stop()\n",
	    blk_fired, blk_ran);
	kprintf("alcreplica[blk]: IF THIS IS THE LAST alcreplica LINE, "
	    "THE if_alc.c AB-BA DEADLOCK IS PROVEN\n");
	callout_stop(&tick_blk_c);			/* :3973 */
	kprintf("alcreplica[blk]: callout_stop returned (NO deadlock)\n");
	lwkt_serialize_exit(&ser);
	return 0;
}

static SYSCTL_NODE(_kern, OID_AUTO, alcreplica, CTLFLAG_RW, 0,
    "alc callout replica");
SYSCTL_PROC(_kern_alcreplica, OID_AUTO, arm, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    arm_sysctl, "I", "arm replica callouts");
SYSCTL_PROC(_kern_alcreplica, OID_AUTO, stop_try, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, stop_try_sysctl, "I", "alc_stop vs try-based (fixed) alc_tick");
SYSCTL_PROC(_kern_alcreplica, OID_AUTO, stop_blk, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, stop_blk_sysctl, "I", "alc_stop vs shipped (blocking) alc_tick");

static int
alc_replica_load(struct module *m, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		lwkt_serialize_init(&ser);
		callout_init_mp(&tick_try_c);
		callout_init_mp(&tick_blk_c);
		kprintf("alc_callout_replica: loaded (hz=%d ncpus=%d)\n",
		    hz, ncpus);
		return 0;
	case MOD_UNLOAD:
		/* only safe if the AB-BA test was not run */
		callout_stop(&tick_try_c);
		callout_stop(&tick_blk_c);
		return 0;
	}
	return 0;
}

static moduledata_t alc_replica_mod = {
	"alc_callout_replica",
	alc_replica_load,
	NULL
};
DECLARE_MODULE(alc_callout_replica, alc_replica_mod, SI_SUB_DRIVERS,
    SI_ORDER_ANY);