/*
 * bfe_tick_replica.c - deterministic replica of if_bfe.c's callout protocol.
 *
 * if_bfe.c (DragonFly) arms ONE NON-MP callout:
 *
 *   callout_init(&sc->bfe_stat_timer);                       if_bfe.c:323
 *   callout_reset(&sc->bfe_stat_timer, hz, bfe_tick, sc);    if_bfe.c:1386
 *
 *   bfe_tick()  if_bfe.c:1486-1508:
 *       mii = device_get_softc(sc->bfe_miibus);              :1492 (outside ser)
 *       lwkt_serialize_enter(ifp->if_serializer);            :1494 (BLOCKING)
 *       bfe_stats_update(sc);                                :1496
 *       callout_reset(&sc->bfe_stat_timer, hz, bfe_tick, sc);:1497 (re-arm INSIDE)
 *       ... mii_tick/link work ...                           :1499-1507
 *       lwkt_serialize_exit(ifp->if_serializer);             :1508
 *
 *   bfe_stop()  if_bfe.c:1516-1530:
 *       ASSERT_SERIALIZED(ifp->if_serializer);
 *       callout_stop(&sc->bfe_stat_timer);                   :1522 (SYNC stop)
 *
 * bfe_stop is reached WITH the serializer held from:
 *   bfe_ioctl SIOCSIFFLAGS down (:1449-1450), bfe_init (:1365, itself called
 *   from bfe_watchdog :1480 and from the bfe_intr error path :1186),
 *   bfe_detach (:441), bfe_shutdown (:475).
 *
 * DragonFly callout_stop() synchronously waits for an in-progress callback
 * (sys/kern/kern_timeout.c:1091 -> :859 _callout_cancel_or_stop sync=1,
 * ssleep(...,"costp",0) at :920).  A bfe_tick callback that was dequeued and
 * is blocked in lwkt_serialize_enter() (sys/kern/lwkt_serialize.c:105 ->
 * adaptive spin then lwkt_serialize_sleep) on the serializer held by the
 * bfe_stop caller deadlocks against it (AB-BA): both threads sleep forever,
 * the ifnet serializer is never released, and the wedged softclock thread
 * stops delivering all callouts for its CPU.
 *
 * This module reproduces exactly that protocol with two callouts:
 *   "tick timer"   == bfe_tick shape   (non-mp callout, BLOCKING enter,
 *                                       hz re-arm inside the serializer)
 *   "fixed timer"  == the family-fix shape (try-enter, re-arm outside) --
 *                    positive control: callout_stop under a held serializer
 *                    returns for this shape (mechanism-level fix validation).
 *
 * sysctls:
 *   kern.bferplica.arm=1        arm both replicas (tick fires in 1 tick,
 *                               then self re-arms at hz like if_bfe.c:1497)
 *   kern.bferplica.stop_fixed=1 hold serializer, callout_stop(fixed) -> returns
 *   kern.bferplica.stop_blk=1   hold serializer, wait until the tick handler
 *                               is BLOCKED on it, callout_stop(tick) ->
 *                               if the AB-BA analysis is right this NEVER
 *                               returns (wedges this thread + a softclock
 *                               thread; guest must be reset afterwards)
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/serialize.h>
#include <sys/callout.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

static struct lwkt_serialize ser;
static struct callout tick_c;		/* bfe_tick replica */
static struct callout fixed_c;		/* fixed (try-enter) shape */

static volatile int tick_fired, tick_ran;
static volatile int fixed_fired, fixed_ran;

static void
tick_timer(void *arg)
{
	tick_fired++;
	/* if_bfe.c:1494 - BLOCKING acquire */
	lwkt_serialize_enter(&ser);
	/* if_bfe.c:1496-1497 - work + hz re-arm INSIDE the serializer */
	tick_ran++;
	callout_reset(&tick_c, hz, tick_timer, NULL);
	/* if_bfe.c:1499-1507 link work elided */
	lwkt_serialize_exit(&ser);
}

static void
fixed_timer(void *arg)
{
	fixed_fired++;
	/* family fix: never block on the serializer */
	if (lwkt_serialize_try(&ser)) {
		fixed_ran++;
		lwkt_serialize_exit(&ser);
	}
	/* re-arm unconditionally, outside the serializer */
	callout_reset(&fixed_c, 1, fixed_timer, NULL);
}

static int
arm_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v == 1) {
		tick_fired = tick_ran = fixed_fired = fixed_ran = 0;
		callout_reset(&tick_c, 1, tick_timer, NULL);
		callout_reset(&fixed_c, 1, fixed_timer, NULL);
		kprintf("bferplica: armed (tick: 1 tick then hz; fixed: 1 tick)\n");
	}
	return 0;
}

static int
stop_fixed_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0, i;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	for (i = 0; i < 20; i++) {
		lwkt_serialize_enter(&ser);
		kprintf("bferplica[fixed]: holding serializer, calling callout_stop()\n");
		callout_stop(&fixed_c);
		kprintf("bferplica[fixed]: callout_stop RETURNED (no deadlock), "
		    "pending=%d\n", callout_pending(&fixed_c));
		lwkt_serialize_exit(&ser);
		if (callout_pending(&fixed_c) == 0)
			break;
		tsleep(&fixed_ran, 0, "fxr", hz / 10);
	}
	return 0;
}

static int
stop_blk_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	/*
	 * AB-BA test (bfe_stop shape): take the serializer, wait until the
	 * bfe_tick-shaped handler is parked inside lwkt_serialize_enter
	 * (fired but not ran), then callout_stop() -- exactly what
	 * if_bfe.c:1522 does from bfe_ioctl/bfe_watchdog/bfe_intr paths.
	 * If the analysis is right, the next kprintf never happens: this
	 * thread sleeps in "costp" (kern_timeout.c:920) while the handler
	 * sleeps on the serializer we hold.  Both wedge permanently.
	 */
	lwkt_serialize_enter(&ser);
	while (tick_ran == tick_fired)
		tsleep(&tick_ran, 0, "bfew", 2);
	kprintf("bferplica[blk]: bfe_tick handler BLOCKED on our serializer "
	    "(fired=%d ran=%d) - entering callout_stop()\n", tick_fired, tick_ran);
	kprintf("bferplica[blk]: IF THIS IS THE LAST bferplica LINE, "
	    "AB-BA DEADLOCK IS PROVEN (bfe_tick vs bfe_stop)\n");
	callout_stop(&tick_c);
	kprintf("bferplica[blk]: callout_stop returned (NO deadlock)\n");
	lwkt_serialize_exit(&ser);
	return 0;
}

static SYSCTL_NODE(_kern, OID_AUTO, bferplica, CTLFLAG_RW, 0,
    "bfe callout replica");
SYSCTL_PROC(_kern_bferplica, OID_AUTO, arm, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    arm_sysctl, "I", "arm bfe replica callouts");
SYSCTL_PROC(_kern_bferplica, OID_AUTO, stop_fixed, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, stop_fixed_sysctl, "I", "bfe_stop vs try-based fixed timer");
SYSCTL_PROC(_kern_bferplica, OID_AUTO, stop_blk, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, stop_blk_sysctl, "I", "bfe_stop vs bfe_tick (AB-BA)");

static int
bfe_replica_load(struct module *m, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		lwkt_serialize_init(&ser);
		/* if_bfe.c:323 - NON-mp callout, bfe's exact flavor */
		callout_init(&tick_c);
		callout_init(&fixed_c);
		kprintf("bferplica: loaded (hz=%d)\n", hz);
		return 0;
	case MOD_UNLOAD:
		/* only safe if stop_blk was not run (it wedges this module) */
		callout_stop(&fixed_c);
		callout_stop(&tick_c);
		return 0;
	}
	return 0;
}

static moduledata_t bfe_replica_mod = {
	"bfe_replica",
	bfe_replica_load,
	NULL
};
DECLARE_MODULE(bfe_replica, bfe_replica_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);