/*
 * ste_callout_replica.c - deterministic replica of if_ste.c's callout protocol.
 *
 * if_ste.c (DragonFly) arms ONE NON-MP callout:
 *
 *   callout_init(&sc->ste_stat_timer);                       if_ste.c:914
 *   callout_reset(&sc->ste_stat_timer, hz,
 *                 ste_stats_update, sc);                     if_ste.c:1220
 *
 *   ste_stats_update()  if_ste.c:780-813:
 *       mii = device_get_softc(sc->ste_miibus);              :788 (outside ser)
 *       lwkt_serialize_enter(ifp->if_serializer);            :790 (BLOCKING, 1st action)
 *       ...stats reads / mii_pollstat / link work...         :792-809
 *       callout_reset(&sc->ste_stat_timer, hz,
 *                     ste_stats_update, sc);                 :811 (re-arm INSIDE ser)
 *       lwkt_serialize_exit(ifp->if_serializer);             :812
 *
 *   ste_stop()  if_ste.c:1224-1232:
 *       callout_stop(&sc->ste_stat_timer);                   :1231 (SYNC stop)
 *       ...free all RX/TX mbufs, bzero rings...              :1248-1262
 *
 * ste_stop is reached WITH if_serializer held from:
 *   ste_detach (:997-998), ste_init (:1130; itself reached from
 *   ste_ioctl SIOCSIFFLAGS :1322/:1326, ste_intr HOSTERR :617,
 *   ste_txeoc :723/:736, ste_watchdog :1494), and ste_intr's
 *   STATS_OFLOW path calls callout_stop directly under the serializer
 *   from interrupt-thread context (:607-610).  ste_watchdog itself runs
 *   serialized on netisr0 (sys/net/if.c if_slowtimo_dispatch ->
 *   ifnet_tryserialize_all).
 *
 * DragonFly callout_stop() synchronously waits for an in-progress callback
 * (sys/kern/kern_timeout.c:1091 -> :859 _callout_cancel_or_stop sync=1,
 * ssleep(...,"costp",0) at :920).  ste_stats_update is a NON-MP callout, so
 * its callback runs holding the MP lock (kern_timeout.c:580-585) and its
 * FIRST action is the blocking lwkt_serialize_enter (:790).  A dequeued
 * callback parked on the serializer held by the ste_stop caller therefore
 * deadlocks against it (AB-BA): stopper sleeps forever in "costp" holding
 * if_serializer; the softclock thread sleeps forever on the serializer
 * WITH THE MP LOCK HELD -> system-wide legacy-path stall.
 *
 * Secondary outcome (same root cause): if the STOP lands while the tick is
 * INPROG *inside* the serializer body, the tick's own callout_reset (:811)
 * is processed recursively and overrides the STOP (kern_timeout.c RESET
 * beats STOP), so ste_stop returns with the timer re-armed; at detach the
 * resurrected tick dereferences the freed softc/miibus/ste_ldata one
 * second later (use-after-free).
 *
 * This module reproduces the protocol with two callouts:
 *   "tick timer"   == ste_stats_update shape (non-mp callout, BLOCKING
 *                     enter, hz re-arm inside the serializer)
 *   "fixed timer"  == the family-fix shape (try-enter, re-arm outside) --
 *                     positive control: callout_stop under a held
 *                     serializer returns for this shape.
 *
 * sysctls:
 *   kern.sterplica.arm=1        arm both replicas (tick fires in 1 tick,
 *                               then self re-arms at hz like if_ste.c:811)
 *   kern.sterplica.stop_fixed=1 hold serializer, callout_stop(fixed) -> returns
 *   kern.sterplica.stop_blk=1   hold serializer, wait until the tick handler
 *                               is BLOCKED on it, callout_stop(tick) ->
 *                               if the AB-BA analysis is right this NEVER
 *                               returns (wedges this thread + a softclock
 *                               thread; guest must be reset afterwards)
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/serialize.h>
#include <sys/callout.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

static struct lwkt_serialize ser;
static struct callout tick_c;		/* ste_stats_update replica */
static struct callout fixed_c;		/* fixed (try-enter) shape */

static volatile int tick_fired, tick_ran;
static volatile int fixed_fired, fixed_ran;

static void
tick_timer(void *arg)
{
	tick_fired++;
	/* if_ste.c:790 - BLOCKING acquire, first action */
	lwkt_serialize_enter(&ser);
	/* if_ste.c:792-809 stat work elided */
	tick_ran++;
	/* if_ste.c:811 - hz re-arm INSIDE the serializer */
	callout_reset(&tick_c, hz, tick_timer, NULL);
	/* if_ste.c:812 */
	lwkt_serialize_exit(&ser);
}

static void
fixed_timer(void *arg)
{
	fixed_fired++;
	/* family fix: never block on the serializer */
	if (lwkt_serialize_try(&ser)) {
		fixed_ran++;
		lwkt_serialize_exit(&ser);
	}
	/* re-arm unconditionally, outside the serializer */
	callout_reset(&fixed_c, 1, fixed_timer, NULL);
}

static int
arm_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v == 1) {
		tick_fired = tick_ran = fixed_fired = fixed_ran = 0;
		callout_reset(&tick_c, 1, tick_timer, NULL);
		callout_reset(&fixed_c, 1, fixed_timer, NULL);
		kprintf("sterplica: armed (tick: 1 tick then hz; fixed: 1 tick)\n");
	}
	return 0;
}

static int
stop_fixed_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0, i;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	for (i = 0; i < 20; i++) {
		lwkt_serialize_enter(&ser);
		kprintf("sterplica[fixed]: holding serializer, calling callout_stop()\n");
		callout_stop(&fixed_c);
		kprintf("sterplica[fixed]: callout_stop RETURNED (no deadlock), "
		    "pending=%d\n", callout_pending(&fixed_c));
		lwkt_serialize_exit(&ser);
		if (callout_pending(&fixed_c) == 0)
			break;
		tsleep(&fixed_ran, 0, "fxr", hz / 10);
	}
	return 0;
}

static int
stop_blk_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	/*
	 * AB-BA test (ste_stop shape, exactly what if_ste.c:1231 does from
	 * ste_ioctl/ste_intr STATS_OFLOW/ste_watchdog/ste_detach paths):
	 * take the serializer, wait until the ste_stats_update-shaped
	 * handler is parked inside lwkt_serialize_enter (fired but not
	 * ran), then callout_stop().  If the analysis is right, the next
	 * kprintf never happens: this thread sleeps in "costp"
	 * (kern_timeout.c:920) while the handler sleeps on the serializer
	 * we hold -- and, being a non-MP callout, it holds the MP lock.
	 * Both wedge permanently.
	 */
	lwkt_serialize_enter(&ser);
	while (tick_ran == tick_fired)
		tsleep(&tick_ran, 0, "stew", 2);
	kprintf("sterplica[blk]: ste_stats_update handler BLOCKED on our "
	    "serializer (fired=%d ran=%d) - entering callout_stop()\n",
	    tick_fired, tick_ran);
	kprintf("sterplica[blk]: IF THIS IS THE LAST sterplica LINE, "
	    "AB-BA DEADLOCK IS PROVEN (ste_stats_update vs ste_stop)\n");
	callout_stop(&tick_c);
	kprintf("sterplica[blk]: callout_stop returned (NO deadlock)\n");
	lwkt_serialize_exit(&ser);
	return 0;
}

static SYSCTL_NODE(_kern, OID_AUTO, sterplica, CTLFLAG_RW, 0,
    "ste callout replica");
SYSCTL_PROC(_kern_sterplica, OID_AUTO, arm, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    arm_sysctl, "I", "arm ste replica callouts");
SYSCTL_PROC(_kern_sterplica, OID_AUTO, stop_fixed, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, stop_fixed_sysctl, "I", "ste_stop vs try-based fixed timer");
SYSCTL_PROC(_kern_sterplica, OID_AUTO, stop_blk, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, stop_blk_sysctl, "I", "ste_stop vs ste_stats_update (AB-BA)");

static int
ste_replica_load(struct module *m, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		lwkt_serialize_init(&ser);
		/* if_ste.c:914 - NON-mp callout, ste's exact flavor */
		callout_init(&tick_c);
		callout_init(&fixed_c);
		kprintf("sterplica: loaded (hz=%d)\n", hz);
		return 0;
	case MOD_UNLOAD:
		/* only safe if stop_blk was not run (it wedges this module) */
		callout_stop(&fixed_c);
		callout_stop(&tick_c);
		return 0;
	}
	return 0;
}

static moduledata_t ste_replica_mod = {
	"ste_replica",
	ste_replica_load,
	NULL
};
DECLARE_MODULE(ste_replica, ste_replica_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);