/*
 * DF-4260 PoC: deterministic reproduction of the if_vr.c
 * callout_stop()-under-serializer deadlock pattern.
 *
 * The pattern being reproduced (all on the shipping guest kernel):
 *
 *   vr_tick()   [if_vr.c:1165]  lwkt_serialize_enter(ifp->if_serializer)
 *   ...                              (softclock thread, possibly another CPU)
 *
 *   vr_stop()   [if_vr.c:1630]  callout_stop(&sc->vr_stat_timer)
 *   ...                          called *while ifp->if_serializer is held*
 *                                (vr_detach:824, vr_ioctl SIOCSIFFLAGS:1537
 *                                 via ifioctl's ifnet_serialize_all,
 *                                 vr_intr:1195, vr_watchdog:1609)
 *
 * callout_stop() is synchronous on this kernel (kern_timeout.c:1091-1094,
 * _callout_cancel_or_stop(cc, CALLOUT_STOP, sync=1)); if the callback is
 * CALLOUT_INPROG it ssleep("costp")s (kern_timeout.c:920) waiting for the
 * callback to finish -- while the caller still holds the lwkt serializer
 * the callback is blocked trying to acquire.  Neither side can progress:
 * permanent kernel deadlock (softclock thread on one CPU dead => every
 * timer / tsleep timeout queued on that CPU is dead; the ifconfig syscall
 * never returns).
 *
 * kern.poc_deadlock=2  runs the *fixed* pattern (callout_stop_async() under
 *                      the serializer + callout_drain() outside it) and is
 *                      expected to return cleanly.
 * kern.poc_deadlock=1  runs the *bugged* vr pattern (synchronous
 *                      callout_stop() under the serializer) and is expected
 *                      to wedge the kernel permanently.
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/serialize.h>
#include <sys/callout.h>
#include <sys/globaldata.h>
#include <machine/cpufunc.h>

static struct lwkt_serialize poc_s;
static struct callout poc_c;

/* 0 = idle, 1 = softclock dequeued us (CALLOUT_INPROG), 2 = cb completed */
static volatile int poc_state;

/*
 * Stand-in for vr_tick() (if_vr.c:1158-1181): takes the serializer, does
 * its work, re-arms itself (vr_tick re-arms at if_vr.c:1178).
 */
static void
poc_tick(void *arg __unused)
{
	poc_state = 1;		/* softclock set INPROG before calling us */
	DELAY(50000);		/* widen the window: let the trigger thread win */

	lwkt_serialize_enter(&poc_s);		/* vr_tick: if_vr.c:1165 */
	poc_state = 2;
	callout_reset(&poc_c, hz, poc_tick, NULL);	/* vr_tick: if_vr.c:1178 */
	lwkt_serialize_exit(&poc_s);
}

/*
 * Stand-in for the vr_ioctl(SIOCSIFFLAGS)/vr_detach path: this sysctl
 * write plays the role of `ifconfig vr0 down` (ifioctl holds
 * ifp->if_serializer around if_ioctl, sys/net/if.c ifnet_serialize_all).
 */
static int
poc_trigger(SYSCTL_HANDLER_ARGS)
{
	int v = 0, error, i, tgt;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return error;
	if (v != 1 && v != 2)
		return EINVAL;

	/* arm the tick on ANOTHER cpu's softclock (SMP requirement) */
	tgt = (mycpu->gd_cpuid + 1) % ncpus;
	poc_state = 0;
	callout_reset_bycpu(&poc_c, 1, poc_tick, NULL, tgt);

	for (i = 0; i < 10000 && poc_state == 0; i++)	/* <= 10s */
		DELAY(1000);
	if (poc_state == 0)
		return ETIMEDOUT;
	kprintf("poc_deadlock: mode %d, tick INPROG on cpu%d\n", v, tgt);

	if (v == 2) {
		/* FIXED pattern (fix.diff): async stop under the serializer,
		 * synchronous drain OUTSIDE it. */
		lwkt_serialize_enter(&poc_s);
		callout_stop_async(&poc_c);
		lwkt_serialize_exit(&poc_s);
		callout_drain(&poc_c);
		kprintf("poc_deadlock: fixed pattern completed, state=%d\n",
		    poc_state);
		return 0;
	}

	/*
	 * BUGGED pattern == vr_stop() (if_vr.c:1630) reached under
	 * if_serializer.  Never returns: sync callout_stop() sleeps
	 * ("costp", kern_timeout.c:920) holding poc_s, while poc_tick is
	 * blocked acquiring poc_s.
	 */
	lwkt_serialize_enter(&poc_s);
	callout_stop(&poc_c);		/* <-- hangs here, holding poc_s */
	lwkt_serialize_exit(&poc_s);

	kprintf("poc_deadlock: bugged pattern returned (unexpected!)\n");
	return 0;
}

SYSCTL_PROC(_kern, OID_AUTO, poc_deadlock,
    CTLTYPE_INT | CTLFLAG_RW, 0, 0, poc_trigger, "I",
    "1=trigger bugged vr_stop pattern, 2=fixed pattern");

static int
poc_modevent(module_t mod __unused, int type, void *arg __unused)
{
	switch (type) {
	case MOD_LOAD:
		lwkt_serialize_init(&poc_s);
		callout_init_mp(&poc_c);
		kprintf("poc_deadlock: loaded (ncpus=%d)\n", ncpus);
		return 0;
	case MOD_UNLOAD:
		callout_drain(&poc_c);
		return 0;
	default:
		return 0;
	}
}

static moduledata_t poc_mod = { "poc_deadlock", poc_modevent, 0 };
DECLARE_MODULE(poc_deadlock, poc_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);