/*
 * callout_replica.c - deterministic replica of if_em.c's callout protocol.
 *
 * if_em.c arms three MP-safe callouts (callout_init_mp, if_em.c:548-550)
 * whose handlers take the driver serializer by hand:
 *
 *   em_timer()           if_em.c:2216-2239  lwkt_serialize_enter (BLOCKING, :2222)
 *                                             + self re-arm (:2236)
 *   em_82547_move_tail() if_em.c:2027-2036  lwkt_serialize_enter (BLOCKING, :2033)
 *                                             + re-arm from move_tail_serialized (:2016)
 *   em_txgc_timer()      if_em.c:484-507    lwkt_serialize_TRY (:494)
 *                                             + unconditional re-arm (:506)
 *
 * em_stop() (if_em.c:2343-2350) tears them down with callout_stop() while
 * holding the same serializer.
 *
 * This module reproduces exactly that protocol with two callouts:
 *   "blocking timer"  == em_timer shape
 *   "try gc timer"    == em_txgc_timer shape
 * and exposes sysctls to run the two interleavings:
 *
 *   test.em.arm   =1  arm both replicas
 *   test.em.stop_gc=1 hold serializer, wait for gc handler to be INPROG,
 *                     callout_stop() -> must return; reports callout_pending()
 *                     (expected 0: STOP post-processing unloads the racing
 *                      re-arm - i.e. the DF-3681 premise does NOT hold on
 *                      this kern_timeout.c, for the try-based handler)
 *   test.em.stop_blk=1 hold serializer, wait until blocking handler is
 *                     BLOCKED on it, then callout_stop() -> if the AB-BA
 *                     analysis is right, this NEVER returns (softclock
 *                     thread of that CPU wedged inside the callback).
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/serialize.h>
#include <sys/callout.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

static struct lwkt_serialize ser;
static struct callout blk_c;		/* em_timer replica */
static struct callout gc_c;		/* em_txgc_timer replica */

static volatile int blk_fired, blk_ran;
static volatile int gc_fired, gc_ran;

static void
blk_timer(void *arg)
{
	blk_fired++;
	/* if_em.c:2222 - BLOCKING acquire */
	lwkt_serialize_enter(&ser);
	blk_ran++;
	/* if_em.c:2236 - self re-arm */
	callout_reset(&blk_c, hz / 4, blk_timer, NULL);
	lwkt_serialize_exit(&ser);
}

static void
gc_timer(void *arg)
{
	gc_fired++;
	/* if_em.c:494 - try acquire */
	if (lwkt_serialize_try(&ser)) {
		gc_ran++;
		lwkt_serialize_exit(&ser);
	}
	/* if_em.c:505-506 - unconditional re-arm on try-failure */
	callout_reset(&gc_c, 1, gc_timer, NULL);
}

static int
arm_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v == 1) {
		blk_fired = blk_ran = gc_fired = gc_ran = 0;
		callout_reset(&gc_c, 1, gc_timer, NULL);
		callout_reset(&blk_c, 1, blk_timer, NULL);
		kprintf("replica: armed (gc every tick, blk every hz/4)\n");
	}
	return 0;
}

static int
stop_gc_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0, tries;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	/*
	 * Hold the serializer (as em_stop does), wait for a gc callback to be
	 * INPROG (fired but not yet re-armed), then callout_stop().  The gc
	 * handler never blocks, so callout_stop's sync wait must complete.
	 */
	for (tries = 0; tries < 100; tries++) {
		lwkt_serialize_enter(&ser);
		while (gc_fired == gc_ran &&		/* handler in try/re-arm */
		    callout_pending(&gc_c) == 0)
			tsleep(&gc_fired, 0, "gcw", 1);
		kprintf("replica[gc]: stop while INPROG (fired=%d ran=%d)\n",
		    gc_fired, gc_ran);
		callout_stop(&gc_c);
		kprintf("replica[gc]: callout_stop returned, pending=%d\n",
		    callout_pending(&gc_c));
		lwkt_serialize_exit(&ser);
		if (callout_pending(&gc_c) == 0)
			break;			/* stopped for good */
		tsleep(&gc_ran, 0, "gcr", hz / 10);
	}
	kprintf("replica[gc]: RESULT pending=%d (0 == re-arm did NOT survive "
	    "callout_stop; DF-3681 premise false for try-based handler here)\n",
	    callout_pending(&gc_c));
	return 0;
}

static int
stop_blk_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	/*
	 * AB-BA test: take the serializer (as em_stop does), wait until the
	 * blocking handler is parked inside lwkt_serialize_enter (fired but
	 * not ran), then callout_stop().
	 *
	 * callout_stop (kern_timeout.c:1091 -> _callout_cancel_or_stop,
	 * sync=1) sleeps on "costp" until the INPROG callback completes;
	 * the callback sleeps in lwkt_serialize_enter until we release the
	 * serializer; we sleep in callout_stop holding it.  If the analysis
	 * is right this never prints the next line - the writing thread AND
	 * a softclock thread are wedged permanently.
	 */
	lwkt_serialize_enter(&ser);
	while (blk_ran == blk_fired)
		tsleep(&blk_ran, 0, "blkw", 2);
	kprintf("replica[blk]: handler BLOCKED on our serializer "
	    "(fired=%d ran=%d) - entering callout_stop()\n", blk_fired, blk_ran);
	kprintf("replica[blk]: IF THIS IS THE LAST replica LINE, "
	    "AB-BA DEADLOCK IS PROVEN\n");
	callout_stop(&blk_c);
	kprintf("replica[blk]: callout_stop returned (NO deadlock)\n");
	lwkt_serialize_exit(&ser);
	return 0;
}

static SYSCTL_NODE(_kern, OID_AUTO, emreplica, CTLFLAG_RW, 0,
    "em callout replica");
SYSCTL_PROC(_kern_emreplica, OID_AUTO, arm, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    arm_sysctl, "I", "arm replica callouts");
SYSCTL_PROC(_kern_emreplica, OID_AUTO, stop_gc, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    stop_gc_sysctl, "I", "em_stop vs try-based gc timer");
SYSCTL_PROC(_kern_emreplica, OID_AUTO, stop_blk, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    stop_blk_sysctl, "I", "em_stop vs blocking timer (AB-BA)");

static int
callout_replica_load(struct module *m, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		lwkt_serialize_init(&ser);
		callout_init_mp(&gc_c);
		callout_init_mp(&blk_c);
		kprintf("callout_replica: loaded (hz=%d)\n", hz);
		return 0;
	case MOD_UNLOAD:
		/* only safe if tests were not run to the deadlock */
		callout_stop(&gc_c);
		callout_stop(&blk_c);
		return 0;
	}
	return 0;
}

static moduledata_t callout_replica_mod = {
	"callout_replica",
	callout_replica_load,
	NULL
};
DECLARE_MODULE(callout_replica, callout_replica_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);