/*
 * pcn_callout_replica.c - deterministic replica of if_pcn.c's callout protocol.
 *
 * if_pcn.c arms ONE legacy callout for its 1 Hz stats/media tick:
 *
 *   pcn_attach()  if_pcn.c:577   callout_init(&sc->pcn_stat_timer)
 *                                (plain, NON-MPSAFE -> callback executes
 *                                 under the Giant MP lock, kern_timeout.c:583)
 *   pcn_tick()    if_pcn.c:873   lwkt_serialize_enter(ifp->if_serializer)
 *                                (:879, BLOCKING - lwkt_serialize.c:105 ->
 *                                 atomic_intr_cond_enter(..., sleep))
 *                                ... mii_tick/mii_pollstat under serializer ...
 *                                self re-arm at :896
 *                                callout_reset(&sc->pcn_stat_timer, hz, ...),
 *                                exit at :898
 *   pcn_stop()    if_pcn.c:1329  callout_stop(&sc->pcn_stat_timer)
 *
 * EVERY caller of pcn_stop() holds ifp->if_serializer at that point:
 *   pcn_detach :641-643 (enter at :641, pcn_stop at :643),
 *   pcn_shutdown :1374-1376,
 *   pcn_init :1106 (-> from pcn_ioctl SIOCSIFFLAGS :1273/:1276,
 *   pcn_watchdog :1306-1310, pcn_intr ERR :929 - all serialized),
 *   pcn_intr :912-914 (direct pcn_stop when !IFF_UP, interrupt context).
 *
 * DragonFly callout_stop() is SYNCHRONOUS: _callout_stop() ssleep()'s on
 * 'costp' (kern_timeout.c:920) until an INPROG callback finishes
 * (kern_timeout.c:567 sets INPROG around the call).
 *
 * => A pcn_tick that has been dequeued and is parked in
 *    lwkt_serialize_enter() on the serializer held by the pcn_stop caller
 *    deadlocks against it (AB-BA): the stopper sleeps forever in callout_stop,
 *    the tick sleeps forever on the serializer, the CPU's softclock thread
 *    stops delivering ALL callouts, and - because pcn_stat_timer is a
 *    non-MPSAFE callout - the parked softclock thread holds the Giant MP
 *    lock while it sleeps.
 *
 * Sysctls (root):
 *   kern.pcnreplica_arm = 1   arm the pcn_tick-shaped timer (1 tick, so the
 *                             park is deterministic)
 *   kern.pcnreplica_stop = 1  enter the serializer (pcn_stop context), wait
 *                             until the tick is parked on it, then
 *                             callout_stop() -- NEVER RETURNS on the AB-BA.
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/serialize.h>
#include <sys/callout.h>

static struct lwkt_serialize ser;	/* ifp->if_serializer replica */
static struct callout tick_c;		/* sc->pcn_stat_timer replica (NON-MPSAFE) */

static volatile int tick_fired;		/* bumped before the park point */
static volatile int tick_ran;		/* bumped only when acquired */
static volatile int tick_parked_seen;

static void
pcn_tick_replica(void *arg)
{
	tick_fired++;
	/* if_pcn.c:879 - BLOCKING acquire (park point) */
	lwkt_serialize_enter(&ser);
	tick_ran++;
	/* if_pcn.c:896 - self re-arm under the serializer, 1 Hz like pcn */
	callout_reset(&tick_c, hz, pcn_tick_replica, NULL);
	lwkt_serialize_exit(&ser);
}

static int
arm_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL || v == 0)
		return (error);

	callout_reset(&tick_c, 1, pcn_tick_replica, NULL);
	kprintf("pcnreplica[arm]: pcn_stat_timer armed (1 tick), waiting\n");
	return (0);
}

static int
stop_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;
	int i, base;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL || v == 0)
		return (error);

	/*
	 * pcn_stop context: serializer held while calling callout_stop().
	 */
	lwkt_serialize_enter(&ser);
	base = tick_fired;
	for (i = 0; i < 5000; i++) {		/* wait <= 5s for a tick fire */
		if (tick_fired != base && tick_ran == base)
			break;			/* fired but parked on us */
		DELAY(1000);
	}
	if (tick_fired != base) {
		tick_parked_seen = 1;
		kprintf("pcnreplica[stop]: tick INPROG and PARKED on the "
		    "serializer (fired=%d ran=%d)\n", tick_fired, tick_ran);
		kprintf("pcnreplica[stop]: IF THIS IS THE LAST pcnreplica LINE "
		    "callout_stop NEVER RETURNED (AB-BA deadlock)\n");
	} else {
		kprintf("pcnreplica[stop]: no tick fired within 5s; "
		    "callout_stop will trivially return\n");
	}

	callout_stop(&tick_c);			/* if_pcn.c:1329 */

	kprintf("pcnreplica[stop]: callout_stop RETURNED (no deadlock in "
	    "this interleave)\n");
	lwkt_serialize_exit(&ser);
	return (0);
}

static int
pcnreplica_load(module_t mod, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		lwkt_serialize_init(&ser);
		callout_init(&tick_c);	/* plain: NON-MPSAFE, like if_pcn.c:577 */
		tick_fired = tick_ran = tick_parked_seen = 0;
		sysctl_add_oid(NULL, SYSCTL_STATIC_CHILDREN(_kern),
		    OID_AUTO, "pcnreplica_arm", CTLTYPE_INT|CTLFLAG_RW,
		    NULL, 0, arm_sysctl, "I", "arm pcn_tick replica");
		sysctl_add_oid(NULL, SYSCTL_STATIC_CHILDREN(_kern),
		    OID_AUTO, "pcnreplica_stop", CTLTYPE_INT|CTLFLAG_RW,
		    NULL, 0, stop_sysctl, "I", "pcn_stop replica (callout_stop "
		    "under serializer)");
		kprintf("pcnreplica: loaded (if_pcn.c callout protocol replica)\n");
		return (0);
	case MOD_UNLOAD:
		return (EBUSY);		/* nothing to clean while wedged */
	}
	return (EOPNOTSUPP);
}

static moduledata_t pcnreplica_mod = { "pcnreplica", pcnreplica_load, NULL };
DECLARE_MODULE(pcnreplica, pcnreplica_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);