/*
 * DF-4275 protocol replica — the epic_stop()/epic_stats_update() AB-BA
 * deadlock in sys/dev/netif/tx/if_tx.c (DF-3715 family instance).
 *
 * Replicated protocol (identical lock discipline to the driver):
 *
 *   epic_stats_update() (if_tx.c:796)      epic_ifioctl()/epic_ifwatchdog()
 *   -----------------------------------    --------------------------------
 *   lwkt_serialize_enter(if_serializer);   lwkt_serialize_enter(if_serializer);
 *     mii_tick(mii);                        epic_stop(sc);          (if_tx.c:1337)
 *     callout_reset(...)                    callout_stop(&sc->tx_stat_timer);
 *   lwkt_serialize_exit();
 *
 * In this kernel callout_stop() is SYNCHRONOUS:
 * sys/kern/kern_timeout.c:1091 -> _callout_cancel_or_stop(cc, CALLOUT_STOP, 1)
 * -> ssleep(c, &c->spin, 0, "costp", 0)  waiting for CALLOUT_INPROG to clear.
 *
 * If the tick callback has been dequeued by softclock (CALLOUT_INPROG set at
 * kern_timeout.c:567) and is blocked in lwkt_serialize_enter() on the very
 * serializer the stopping thread holds, neither thread can ever make
 * progress: the stopper sleeps in 'costp' waiting for the callback, the
 * callback waits for the serializer.
 *
 * if_tx.c entry points that hold ifp->if_serializer across
 * epic_stop() -> callout_stop():
 *   - epic_ifioctl SIOCSIFMTU  (if_tx.c:360-375: epic_stop+epic_init)
 *   - epic_ifioctl SIOCSIFFLAGS down (if_tx.c:389-391)
 *   - epic_ifwatchdog          (if_tx.c:764-790: epic_stop+epic_init)
 *   - epic_intr PCI-fatal      (if_tx.c:691-705)
 *   - epic_detach              (if_tx.c:300-304)
 *
 * Because if_tx.c uses plain callout_init() (if_tx.c:200, non-MPSAFE), the
 * wedged softclock thread also holds the MP lock.
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/kthread.h>
#include <sys/proc.h>
#include <sys/serialize.h>
#include <sys/callout.h>
#include <sys/thread.h>
#include <sys/thread2.h>
#include <sys/mplock2.h>

static struct lwkt_serialize ser;	/* ifp->if_serializer analog   */
static struct callout timer;		/* sc->tx_stat_timer analog    */
static volatile int tick_inprog;	/* callback dequeued + parked  */
static struct thread *stopper_td;
static volatile int done;

/* ---- epic_stats_update() analog (softclock context) ---- */
static void
tick(void *arg)
{
	tick_inprog = 1;			/* INPROG + about to block */
	kprintf("df4275: tick() dequeued; entering serializer "
	    "(blocks: stopper holds it)\n");
	lwkt_serialize_enter(&ser);		/* == epic_stats_update:803 */
	kprintf("df4275: tick() ACQUIRED serializer (DEADLOCK DID NOT "
	    "HAPPEN -- unexpected)\n");
	lwkt_serialize_exit(&ser);
}

/* ---- the epic_stop()-under-serializer analog ---- */
static void
stopper(void *arg)
{
	get_mplock();
	stopper_td = curthread;

	/* arm the callout, then take the serializer — exactly what
	 * epic_ifioctl() does before calling epic_stop(). */
	callout_reset(&timer, 1, tick, NULL);
	lwkt_serialize_enter(&ser);

	/* Wait until softclock dequeued the callback and it is parked on the
	 * serializer (in the real driver this is epic_stats_update blocked on
	 * ifp->if_serializer while an ioctl/watchdog holds it).  NB: yield-wait,
	 * not a timeout sleep — the tsleep timeout would itself depend on the
	 * softclock wheel that this deadlock wedges — and not a busy DELAY,
	 * which would starve softclock on this CPU. */
	while (!tick_inprog) {
		if (tick_inprog)
			break;
		lwkt_yield();
	}

	kprintf("df4275: stopper HOLDS serializer; tick() INPROG+blocked; "
	    "calling callout_stop() [epic_stop analog] ...\n");
	kprintf("df4275: expecting permanent 'costp' sleep -- if this returns,"
	    " no deadlock\n");

	callout_stop(&timer);			/* == epic_stop:1341 */

	kprintf("df4275: callout_stop RETURNED (no deadlock)\n");
	lwkt_serialize_exit(&ser);
	done = 1;
	wakeup(&done);
	rel_mplock();
	kthread_exit();
}

/* ---- reporter: proves the stopper is parked in 'costp' ---- */
static void
reporter(void *arg)
{
	int i;

	get_mplock();
	for (i = 0; i < 5 && !done; i++) {
		tsleep(&done, 0, "df4275r", hz);
		kprintf("df4275: stopper_td=%p wchan=%p wmesg=\"%s\" "
		    "(tick_inprog=%d)\n",
		    stopper_td, stopper_td->td_wchan,
		    stopper_td->td_wmesg ? stopper_td->td_wmesg : "(null)",
		    tick_inprog);
	}
	if (done)
		kprintf("df4275: RESULT: no deadlock\n");
	else
		kprintf("df4275: RESULT: DEADLOCK REPRODUCED -- stopper "
		    "permanently asleep in \"%s\" while tick() is blocked on "
		    "the serializer the stopper holds\n",
		    stopper_td->td_wmesg ? stopper_td->td_wmesg : "?");
	rel_mplock();
	kthread_exit();
}

static int
df4275_load(module_t mod, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		lwkt_serialize_init(&ser);
		callout_init(&timer);	/* non-MPSAFE, same as if_tx.c:200 */
		kthread_create(stopper, NULL, NULL, "df4275stop");
		kthread_create(reporter, NULL, NULL, "df4275rep");
		kprintf("df4275: replica armed\n");
		break;
	default:
		break;
	}
	return (0);
}

static moduledata_t df4275rep_mod = {
	"df4275rep",
	df4275_load,
	NULL
};

DECLARE_MODULE(df4275rep, df4275rep_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
MODULE_VERSION(df4275rep, 1);