/*
 * emx_callout_replica.c - deterministic replica of if_emx.c's callout
 * protocol.  (Pass-2 twin of DF-3715's callout_replica.c for em(4).)
 *
 * if_emx.c arms two MP-safe callouts (callout_init_mp at :553 and :587):
 *
 *   emx_timer()      if_emx.c:1908-1931   lwkt_serialize_enter(
 *                                             &sc->main_serialize)
 *					     BLOCKING at :1914
 *                                           + self re-arm at :1928
 *                                           (callout_reset(&sc->timer, hz,
 *                                            emx_timer, sc) INSIDE the
 *                                           serializer, before exit :1930)
 *   emx_txgc_timer() if_emx.c:426-449     lwkt_serialize_TRY(
 *                                             &tdata->tx_serialize)
 *					   non-blocking at :436
 *                                           + unconditional re-arm at :448
 *                                           (the `done:` fall-through)
 *
 * emx_stop() tears the timer down with callout_stop(&sc->timer) at :2030
 * while holding ALL serializers (ASSERT_IFNET_SERIALIZED_ALL at :2026;
 * callers: SIOCSIFFLAGS :1227, emx_init :1354, emx_detach :989,
 * emx_suspend :1059).  The same callout_stop-under-serializer exists in
 * emx_intr_body() LSC path :1534 (ISR holds main_serialize, then
 * emx_serialize_skipmain adds tx+rx) and emx_npoll_status() :4196.
 *
 * DragonFly callout_stop() is SYNCHRONOUS (kern_timeout.c:1091 ->
 * _callout_cancel_or_stop sync=1; ssleep "costp" at :920) - it waits for
 * an in-progress callback to finish.  lwkt_serialize_enter() BLOCKS
 * (tsleep "slize", kern/lwkt_serialize.c:234).  A callback dequeued and
 * parked on the serializer therefore deadlocks against the stopper
 * (AB-BA): the stopper sleeps in callout_stop holding the serializer;
 * the softclock thread of that CPU is parked inside the callback.
 *
 * This module reproduces exactly that protocol with two callouts:
 *   "blocking timer"  == emx_timer shape
 *   "try gc timer"    == emx_txgc_timer shape
 * and exposes sysctls to run the two interleavings:
 *
 *   kern.emxreplica.arm     =1  arm both replicas
 *   kern.emxreplica.stop_gc =1  hold serializer, wait for gc handler to be
 *                               INPROG, callout_stop() -> must return;
 *                               reports callout_pending() (expected 0:
 *                               STOP post-processing unloads the racing
 *                               re-arm for the try-based handler)
 *   kern.emxreplica.stop_blk=1  hold serializer, wait until blocking
 *                               handler is BLOCKED on it, then
 *                               callout_stop() -> if the AB-BA analysis is
 *                               right this NEVER returns (the writing
 *                               thread AND a softclock thread wedge
 *                               permanently - emx_stop shape).
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/serialize.h>
#include <sys/callout.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

static struct lwkt_serialize ser;	/* == sc->main_serialize replica */
static struct callout blk_c;		/* == sc->timer (emx_timer) replica */
static struct callout gc_c;		/* == tdata->tx_gc_timer replica */

static volatile int blk_fired, blk_ran;
static volatile int gc_fired, gc_ran;

static void
blk_timer(void *arg)
{
	blk_fired++;
	/* if_emx.c:1914 - BLOCKING acquire of main_serialize */
	lwkt_serialize_enter(&ser);
	blk_ran++;
	/* if_emx.c:1928 - self re-arm INSIDE the serializer */
	callout_reset(&blk_c, hz / 4, blk_timer, NULL);
	/* if_emx.c:1930 - exit */
	lwkt_serialize_exit(&ser);
}

static void
gc_timer(void *arg)
{
	gc_fired++;
	/* if_emx.c:436 - non-blocking try acquire of tx_serialize */
	if (lwkt_serialize_try(&ser)) {
		gc_ran++;
		lwkt_serialize_exit(&ser);
	}
	/* if_emx.c:447-448 - `done:` unconditional re-arm */
	callout_reset(&gc_c, 1, gc_timer, NULL);
}

static int
arm_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v == 1) {
		blk_fired = blk_ran = gc_fired = gc_ran = 0;
		callout_reset(&gc_c, 1, gc_timer, NULL);
		callout_reset(&blk_c, 1, blk_timer, NULL);
		kprintf("emxreplica: armed (gc every tick, blk every hz/4)\n");
	}
	return 0;
}

static int
stop_gc_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0, tries;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	/*
	 * Hold the serializer (as emx_stop does), wait for a gc callback to
	 * be INPROG (fired but not yet re-armed), then callout_stop().  The
	 * gc handler never blocks, so callout_stop's sync wait must
	 * complete.  Negative control for the AB-BA test.
	 */
	for (tries = 0; tries < 100; tries++) {
		lwkt_serialize_enter(&ser);
		while (gc_fired == gc_ran &&		/* handler in try/re-arm */
		    callout_pending(&gc_c) == 0)
			tsleep(&gc_fired, 0, "gcw", 1);
		kprintf("emxreplica[gc]: stop while INPROG (fired=%d ran=%d)\n",
		    gc_fired, gc_ran);
		callout_stop(&gc_c);
		kprintf("emxreplica[gc]: callout_stop returned, pending=%d\n",
		    callout_pending(&gc_c));
		lwkt_serialize_exit(&ser);
		if (callout_pending(&gc_c) == 0)
			break;			/* stopped for good */
		tsleep(&gc_ran, 0, "gcr", hz / 10);
	}
	kprintf("emxreplica[gc]: RESULT pending=%d (0 == re-arm did NOT "
	    "survive callout_stop for the try-based emx_txgc_timer shape)\n",
	    callout_pending(&gc_c));
	return 0;
}

static int
stop_blk_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	/*
	 * AB-BA test (emx_stop shape): take the serializer, wait until the
	 * blocking handler is parked inside lwkt_serialize_enter (fired but
	 * not ran), then callout_stop().
	 *
	 * callout_stop (kern_timeout.c:1091 -> _callout_cancel_or_stop
	 * sync=1) sleeps on "costp" until the INPROG callback completes;
	 * the callback sleeps in lwkt_serialize_enter until we release the
	 * serializer; we sleep in callout_stop holding it.  If the analysis
	 * is right this never prints the next line - the writing thread AND
	 * a softclock thread are wedged permanently.
	 */
	lwkt_serialize_enter(&ser);
	while (blk_ran == blk_fired)
		tsleep(&blk_ran, 0, "blkw", 2);
	kprintf("emxreplica[blk]: handler BLOCKED on our serializer "
	    "(fired=%d ran=%d) - entering callout_stop() [emx_stop shape]\n",
	    blk_fired, blk_ran);
	kprintf("emxreplica[blk]: IF THIS IS THE LAST emxreplica LINE, "
	    "AB-BA DEADLOCK IS PROVEN\n");
	callout_stop(&blk_c);
	kprintf("emxreplica[blk]: callout_stop returned (NO deadlock)\n");
	lwkt_serialize_exit(&ser);
	return 0;
}

static SYSCTL_NODE(_kern, OID_AUTO, emxreplica, CTLFLAG_RW, 0,
    "emx callout replica");
SYSCTL_PROC(_kern_emxreplica, OID_AUTO, arm, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    arm_sysctl, "I", "arm replica callouts");
SYSCTL_PROC(_kern_emxreplica, OID_AUTO, stop_gc, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    stop_gc_sysctl, "I", "emx_stop vs try-based gc timer");
SYSCTL_PROC(_kern_emxreplica, OID_AUTO, stop_blk, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    stop_blk_sysctl, "I", "emx_stop vs blocking timer (AB-BA)");

static int
emx_callout_replica_load(struct module *m, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		lwkt_serialize_init(&ser);
		callout_init_mp(&gc_c);
		callout_init_mp(&blk_c);
		kprintf("emx_callout_replica: loaded (hz=%d)\n", hz);
		return 0;
	case MOD_UNLOAD:
		/* only safe if the AB-BA test was not run to the deadlock */
		callout_stop(&gc_c);
		callout_stop(&blk_c);
		return 0;
	}
	return 0;
}

static moduledata_t emx_callout_replica_mod = {
	"emx_callout_replica",
	emx_callout_replica_load,
	NULL
};
DECLARE_MODULE(emx_callout_replica, emx_callout_replica_mod,
    SI_SUB_DRIVERS, SI_ORDER_ANY);