/*
 * lge_callout_replica.c - deterministic replica of if_lge.c's callout protocol.
 *
 * if_lge.c arms ONE callout (callout_init, if_lge.c:435) whose handler takes
 * the driver serializer by hand:
 *
 *   lge_tick()             if_lge.c:978-987   lwkt_serialize_enter (BLOCKING, :984)
 *                                             -> lge_tick_serialized()
 *                                             -> self re-arm INSIDE the lock (:1017)
 *                                             -> lwkt_serialize_exit (:986)
 *
 * lge_stop() tears it down with callout_stop() (if_lge.c:1442) while holding
 * that same serializer -- every in-tree caller of lge_stop() holds
 * ifp->if_serializer:
 *
 *   lge_detach   :576-580  (explicit serialize_enter around lge_stop)
 *   lge_init     :1211     (if_ioctl SIOCSIFFLAGS :1387, lge_rxeoc :933 from
 *                           lge_intr, lge_watchdog :1425 -- all serialized)
 *   lge_intr     :1029     (bus_setup_intr(..., ifp->if_serializer) :553-555)
 *   lge_ioctl    :1391     (SIOCSIFFLAGS, !IFF_UP && IFF_RUNNING)
 * and additionally, from interrupt-thread context:
 *   lge_intr     :1055     (PHY_INTR: callout_stop + lge_tick_serialized)
 *
 * DragonFly callout_stop() is synchronous (kern_timeout.c:1091 ->
 * _callout_cancel_or_stop sync=1): when the callback is INPROG the caller
 * ssleep("costp")s until it completes.  A lge_tick dequeued while a stop-path
 * thread holds the serializer therefore parks in lwkt_serialize_enter and
 * deadlocks the stopper (AB-BA): the stopper sleeps in costp holding the
 * serializer, the callback spins on the serializer, the serializer is never
 * released, and the parked CPU's softclock thread stops delivering every
 * callout on that CPU.
 *
 * This module reproduces exactly that protocol with two callouts:
 *   "lge timer"   == lge_tick shape (blocking enter, re-arm inside)
 *   "fixed timer" == the recommended-fix shape (lwkt_serialize_try + 1-tick
 *                    retry) and doubles as the negative control: its
 *                    callout_stop() must RETURN even while the stopper holds
 *                    the serializer.
 *
 * sysctls:
 *   kern.lgereplica.arm      =1  arm both replicas (lge timer period = hz,
 *                                 exactly lge_init:1319 / lge_tick:1017)
 *   kern.lgereplica.stop_fix =1  hold serializer, stop fixed-shape callout,
 *                                 must return (deadlock-free control)
 *   kern.lgereplica.stop_lge =1  hold serializer, wait until lge-shaped
 *                                 handler is PARKED on it, callout_stop()
 *                                 -> if the analysis is right this NEVER
 *                                 returns.
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/serialize.h>
#include <sys/callout.h>
#include <sys/sysctl.h>
#include <sys/systm.h>

static struct lwkt_serialize ser;	/* == ifp->if_serializer */
static struct callout lge_c;		/* lge_stat_timer replica */
static struct callout fix_c;		/* patched-shape control */

static volatile int lge_fired, lge_ran;
static volatile int fix_fired, fix_ran;

static void lge_tick_replica(void *);

static void
lge_tick_serialized_replica(void)
{
	/* if_lge.c:996-1017 body: stats + mii_tick + RE-ARM INSIDE the lock */
	lge_ran++;
	callout_reset(&lge_c, hz, lge_tick_replica, NULL);
}

static void
lge_tick_replica(void *arg)
{
	lge_fired++;
	/* if_lge.c:984 - BLOCKING acquire */
	lwkt_serialize_enter(&ser);
	lge_tick_serialized_replica();
	/* if_lge.c:986 */
	lwkt_serialize_exit(&ser);
}

static void
fix_tick(void *arg)
{
	fix_fired++;
	/* recommended-fix shape: never block, retry next tick */
	if (lwkt_serialize_try(&ser)) {
		fix_ran++;
		callout_reset(&fix_c, hz, fix_tick, NULL);
		lwkt_serialize_exit(&ser);
	} else {
		callout_reset(&fix_c, 1, fix_tick, NULL);
	}
}

static int
arm_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v == 1) {
		lge_fired = lge_ran = fix_fired = fix_ran = 0;
		callout_reset(&lge_c, hz, lge_tick_replica, NULL);
		callout_reset(&fix_c, 1, fix_tick, NULL);
		kprintf("lgereplica: armed (lge timer every hz, fix timer every 1)\n");
	}
	return 0;
}

static int
stop_fix_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0, tries;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	/* negative control: the try-based (fixed) shape must stop cleanly */
	for (tries = 0; tries < 100; tries++) {
		lwkt_serialize_enter(&ser);
		callout_stop(&fix_c);
		kprintf("lgereplica[fix]: callout_stop returned under held "
		    "serializer, pending=%d\n", !!callout_pending(&fix_c));
		lwkt_serialize_exit(&ser);
		if (callout_pending(&fix_c) == 0)
			break;
		tsleep(&fix_ran, 0, "fixr", hz / 10);
	}
	kprintf("lgereplica[fix]: RESULT pending=%d (0 == fixed shape does NOT "
	    "deadlock lge_stop)\n", !!callout_pending(&fix_c));
	return 0;
}

static int
stop_lge_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1)
		return 0;

	/*
	 * AB-BA test, lge_stop() shape (if_lge.c:1435-1442): take the
	 * serializer (as every lge_stop caller does), wait until the
	 * lge_tick-shaped handler is parked inside lwkt_serialize_enter
	 * (fired but not ran), then callout_stop().
	 *
	 * If the analysis is right, the kprintf after callout_stop() is
	 * never reached: the caller sleeps forever on "costp"
	 * (kern_timeout.c:920) while the softclock thread spins on the
	 * serializer we hold.
	 */
	lwkt_serialize_enter(&ser);
	while (lge_ran == lge_fired)
		tsleep(&lge_ran, 0, "lgew", 2);
	kprintf("lgereplica[lge]: handler PARKED on our serializer "
	    "(fired=%d ran=%d) - entering callout_stop()\n", lge_fired, lge_ran);
	kprintf("lgereplica[lge]: IF THIS IS THE LAST lgereplica LINE, "
	    "THE LGE_STOP AB-BA DEADLOCK IS PROVEN\n");
	callout_stop(&lge_c);
	kprintf("lgereplica[lge]: callout_stop returned (NO deadlock)\n");
	lwkt_serialize_exit(&ser);
	return 0;
}

static SYSCTL_NODE(_kern, OID_AUTO, lgereplica, CTLFLAG_RW, 0,
    "lge callout replica");
SYSCTL_PROC(_kern_lgereplica, OID_AUTO, arm, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
    arm_sysctl, "I", "arm replica callouts");
SYSCTL_PROC(_kern_lgereplica, OID_AUTO, stop_fix, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, stop_fix_sysctl, "I", "lge_stop vs fixed-shape timer (control)");
SYSCTL_PROC(_kern_lgereplica, OID_AUTO, stop_lge, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, stop_lge_sysctl, "I", "lge_stop vs lge_tick (AB-BA deadlock)");

static int
lge_callout_replica_load(struct module *m, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		lwkt_serialize_init(&ser);
		callout_init(&lge_c);
		callout_init(&fix_c);
		kprintf("lge_callout_replica: loaded (hz=%d)\n", hz);
		return 0;
	case MOD_UNLOAD:
		/* only safe if the deadlock test was not run to the wedge */
		callout_stop(&fix_c);
		callout_stop(&lge_c);
		return 0;
	}
	return 0;
}

static moduledata_t lge_callout_replica_mod = {
	"lge_callout_replica",
	lge_callout_replica_load,
	NULL
};
DECLARE_MODULE(lge_callout_replica, lge_callout_replica_mod,
    SI_SUB_DRIVERS, SI_ORDER_ANY);