/*
 * DF-2750 PoC - lockmgr_release() multi-count blind decrement lost-wakeup.
 *
 * sys/kern/kern_lock.c lockmgr_release(): the exclusive multi-count branch
 * does a BLIND atomic_fetchadd(-1) based on a stale pre-read of lk_count.
 * When lk_lockholder == LK_KERNTHREAD any cpu may legally release the lock
 * (buffer-lock biodone handoff protocol, sys/sys/buf2.h BUF_KERNPROC), so
 * two concurrent releasers that both read LKC_XMASK == 2 drive the count
 * 2 -> 0 with NEITHER executing one of the three single-count cases, i.e.:
 *
 *   - no wakeup(lkp) is issued  (EXREQ2 waiter stranded asleep)
 *   - LKC_EXREQ2 is not cleared
 *   - LKC_CANCEL is not cleared
 *   - LKC_SHARED is not pre-set
 *
 * A thread parked in lockmgr_exclusive()'s EXREQ2 tsleep then sleeps on a
 * lock that is now completely FREE.  It stays asleep until unrelated
 * traffic on the same lock happens to issue a wakeup.
 *
 * This module stages the exact precondition deterministically:
 *   lk_count = LKC_XMASK(2) | LKC_EXREQ2   (parked exclusive waiter)
 *   lk_lockholder = LK_KERNTHREAD          (any cpu may release)
 * and then races two lwkt threads pinned to cpu 0 / cpu 1 calling
 * lockmgr(&lk, LK_RELEASE) simultaneously.
 *
 * HIT condition observed by the driver:
 *   after both releases complete, lk_count == 0x08000000 (LKC_EXREQ2 only,
 *   zero XMASK/SMASK) and the waiter is still asleep (only a manual
 *   wakeup() from the driver revives it).
 *
 * Harness discipline (v3):
 *   - the three worker threads are created ONCE at module load and persist,
 *     parking in a 1-tick polling tsleep between invocations (immune to
 *     lost schedule IPIs on idle vCPUs);
 *   - every in-round wait is a BOUNDED spin with a diagnostic bail-out, so
 *     the module can never wedge the guest;
 *   - on bail-out the driver drains leftover lock counts (legal because the
 *     holder is LK_KERNTHREAD) so a waiter parked inside lockmgr can
 *     always finish; MOD_UNLOAD refuses unless threads have exited.
 *
 * Build:  see build.sh   (kernel module against in-guest /usr/src)
 * Run:    sysctl -w debug.lkmc=<rounds>     (as root; prints result via kprintf)
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/lock.h>
#include <sys/sysctl.h>
#include <sys/thread.h>
#include <sys/thread2.h>
#include <sys/types.h>
#include <machine/cpufunc.h>

#define SPIN_CAP	2000000000LL	/* ~1-4s hard cap: bail out, don't wedge */

#define PH_PARKED	0		/* threads idle in polling tsleep */
#define PH_ACTIVE	1		/* running rounds */
#define PH_STOP		2		/* exit */

static struct lock lkmc_lk;

static volatile int  phase;
static volatile int  upcnt;		/* threads in active phase */
static volatile int  parkedcnt;		/* threads back in park */
static volatile int  nexited;

static volatile int  rls_go;	/* 0=idle 1=go (spin barrier) */
static volatile int  rls_done;	/* releasers completed this round */
static volatile int  rls_idle;	/* releasers back at barrier */
static volatile int  t3_cmd;	/* waiter command: 0=idle >0=round */
static volatile int  t3_got;	/* waiter completed acquire+release */

static volatile int  lkmc_hits;
static volatile int  lkmc_rounds_run;
static volatile uint64_t lkmc_bad;
static volatile int  lkmc_healed;
static volatile int  lkmc_stuck;	/* nonzero = harness bail-out point */
static volatile int  lkmc_busy;
static volatile int  lkmc_loaded;

#define BAIL(code)	do { lkmc_stuck = (code); goto out; } while (0)
#define WAIT_FOR(cond, code)	do {			\
	int64_t _n = 0;					\
	while (!(cond)) {				\
		if (++_n > SPIN_CAP)			\
			BAIL(code);			\
		cpu_ccfence();				\
	}						\
} while (0)

/*
 * Exclusive-request waiter: enters lockmgr_exclusive(), blocks on EXREQ2.
 * The driver polls for LKC_EXREQ2 to appear - which by construction means
 * this thread already executed tsleep_interlock() (kern_lock.c:369) before
 * its fcmpset (:370), so any wakeup() on the lock ident is guaranteed to
 * reach it.
 */
static void
lkmc_waiter(void *arg __unused)
{
	int64_t n;
	int cmd;

	atomic_add_int(&nexited, 0);
	for (;;) {
		/* park between invocations (1-tick poll: IPI-loss safe) */
		while (phase == PH_PARKED)
			tsleep(&lkmc_lk, 0, "lkpark", 1);
		if (phase == PH_STOP)
			break;
		atomic_add_int(&upcnt, 1);

		for (;;) {
			n = 0;
			while ((cmd = t3_cmd) == 0) {
				if (phase != PH_ACTIVE)
					goto repark;
				if (++n > SPIN_CAP)
					goto repark;
				cpu_ccfence();
			}
			atomic_swap_int(&t3_cmd, 0);

			if (lockmgr(&lkmc_lk, LK_EXCLUSIVE) == 0)
				lockmgr(&lkmc_lk, LK_RELEASE);
			atomic_swap_int(&t3_got, 1);
			/* continue; driver re-kicks next round */
		}
repark:
		atomic_add_int(&parkedcnt, 1);
	}
	atomic_add_int(&nexited, 1);
	lwkt_exit();
}

/*
 * Releaser: pinned to a specific cpu; both instances are released from a
 * shared spin barrier so they enter lockmgr_release() within nanoseconds
 * of each other, each reading the stale count (XMASK==2) at function entry.
 */
static void
lkmc_releaser(void *arg __unused)
{
	int64_t n;
	int go;

	for (;;) {
		/* park between invocations (1-tick poll: IPI-loss safe) */
		while (phase == PH_PARKED)
			tsleep(&lkmc_lk, 0, "lkpark", 1);
		if (phase == PH_STOP)
			break;
		atomic_add_int(&upcnt, 1);

		for (;;) {
			n = 0;
			while ((go = rls_go) == 0) {
				if (phase != PH_ACTIVE)
					goto repark;
				if (++n > SPIN_CAP)
					goto repark;
				cpu_ccfence();
			}

			lockmgr(&lkmc_lk, LK_RELEASE);
			atomic_add_int(&rls_done, 1);

			n = 0;
			while (rls_go == 1) {
				if (phase != PH_ACTIVE)
					goto repark;
				if (++n > SPIN_CAP)
					break;	/* driver reset late */
				cpu_ccfence();
			}
			atomic_add_int(&rls_idle, 1);
		}
repark:
		atomic_add_int(&parkedcnt, 1);
	}
	atomic_add_int(&nexited, 1);
	lwkt_exit();
}

static int
sysctl_lkmc(SYSCTL_HANDLER_ARGS)
{
	int error;
	int rounds = 0;
	int i;
	uint64_t c;

	if (req->newptr == NULL) {
		int r = lkmc_rounds_run;
		return (sysctl_handle_int(oidp, &r, 0, req));
	}
	error = sysctl_handle_int(oidp, &rounds, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);
	if (rounds <= 0 || rounds > 1000000)
		return (EINVAL);
	if (!lkmc_loaded)
		return (ENXIO);
	if (atomic_swap_int(&lkmc_busy, 1) != 0)
		return (EBUSY);

	lkmc_hits = 0;
	lkmc_healed = 0;
	lkmc_bad = 0;
	lkmc_stuck = 0;
	lkmc_rounds_run = 0;

	/* activate persistent threads */
	rls_go = 0;
	rls_done = 0;
	rls_idle = 0;
	t3_cmd = 0;
	t3_got = 0;
	upcnt = 0;
	parkedcnt = 0;
	cpu_ccfence();
	phase = PH_ACTIVE;
	wakeup(&lkmc_lk);

	tsleep(&lkmc_lk, 0, "lkmcs", 2);	/* settle + wake halted cpus */
	WAIT_FOR(upcnt >= 3, 10);

	for (i = 0; i < rounds; i++) {
		/*
		 * Stage: XMASK==2 (recursive exclusive), KERNTHREAD owner,
		 * and a parked EXREQ2 waiter.
		 */
		lockinit(&lkmc_lk, "lkmc", 0, LK_CANRECURSE);
		lockmgr(&lkmc_lk, LK_EXCLUSIVE);
		lockmgr(&lkmc_lk, LK_EXCLUSIVE);		/* XMASK = 2 */
		lockmgr_kernproc(&lkmc_lk);		/* any cpu may release */

		t3_got = 0;
		t3_cmd = i + 1;				/* kick waiter */
		WAIT_FOR((lkmc_lk.lk_count & LKC_EXREQ2) != 0, 1);

		/* race the two pinned releasers */
		rls_done = 0;
		rls_idle = 0;
		cpu_ccfence();
		rls_go = 1;
		WAIT_FOR(rls_done >= 2, 2);

		c = lkmc_lk.lk_count;
		if ((c & (LKC_XMASK | LKC_SMASK)) == 0 &&
		    (c & LKC_EXREQ2) && t3_got == 0) {
			/*
			 * LOST WAKEUP: lock is FREE (no counts) yet the
			 * waiter is still asleep and EXREQ2 is stranded.
			 */
			lkmc_hits++;
			lkmc_bad = c;
			wakeup(&lkmc_lk);		/* manual heal */
			WAIT_FOR(t3_got != 0, 3);
			lkmc_healed++;
		} else {
			/* normal path: case-1 release woke the waiter */
			WAIT_FOR(t3_got != 0, 4);
		}
		lkmc_rounds_run = i + 1;

		rls_go = 0;				/* reset barrier */
		WAIT_FOR(rls_idle >= 2, 5);
	}

out:
	/*
	 * Drain leftover exclusive counts so a waiter parked inside
	 * lockmgr_exclusive() can always complete.  The lock holder is
	 * LK_KERNTHREAD, which by design allows any thread to release.
	 */
	{
		int k;
		for (k = 0; k < 1000; k++) {
			if ((lkmc_lk.lk_count & (LKC_SMASK | LKC_XMASK)) == 0)
				break;
			lockmgr(&lkmc_lk, LK_RELEASE);
			wakeup(&lkmc_lk);
		}
		wakeup(&lkmc_lk);
	}

	kprintf("lkmc: rounds=%d hits=%d healed=%d bad_count=%016jx stuck=%d\n",
	    lkmc_rounds_run, lkmc_hits, lkmc_healed,
	    (uintmax_t)lkmc_bad, lkmc_stuck);
	if (lkmc_hits)
		kprintf("lkmc: DEFECT REPRODUCED: concurrent KERNTHREAD "
			"releases drove XMASK 2->0, no wakeup, waiter slept "
			"on a free lock\n");
	if (lkmc_stuck)
		kprintf("lkmc: HARNESS BAIL-OUT at wait #%d\n", lkmc_stuck);

	/* send threads back to park */
	parkedcnt = 0;
	phase = PH_PARKED;
	wakeup(&lkmc_lk);
	{
		int64_t n = 0;
		while (parkedcnt < 3 && n < SPIN_CAP) {
			++n;
			cpu_ccfence();
		}
	}

	atomic_swap_int(&lkmc_busy, 0);
	return (0);
}

SYSCTL_PROC(_debug, OID_AUTO, lkmc, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, sysctl_lkmc, "I", "lockmgr concurrent-release lost-wakeup test");

static int
lkmc_modevent(module_t mod __unused, int type, void *data __unused)
{
	switch (type) {
	case MOD_LOAD:
		nexited = 0;
		phase = PH_PARKED;
		if (lwkt_create(lkmc_releaser, NULL, NULL, NULL,
				0, 0, "lkrel0") ||
		    lwkt_create(lkmc_releaser, NULL, NULL, NULL,
				0, 1, "lkrel1") ||
		    lwkt_create(lkmc_waiter, NULL, NULL, NULL,
				0, 2, "lkwait")) {
			kprintf("lkmc: lwkt_create failed\n");
			return (ENOMEM);
		}
		lkmc_loaded = 1;
		kprintf("lkmc: loaded (DF-2750 PoC, persistent threads)\n");
		return (0);
	case MOD_UNLOAD:
		if (atomic_swap_int(&lkmc_busy, 1) != 0)
			return (EBUSY);
		phase = PH_STOP;
		wakeup(&lkmc_lk);
		{
			int64_t n = 0;
			while (nexited < 3 && n < SPIN_CAP) {
				++n;
				cpu_ccfence();
			}
			if (nexited < 3) {
				phase = PH_PARKED;	/* abort unload */
				atomic_swap_int(&lkmc_busy, 0);
				return (EBUSY);
			}
		}
		kprintf("lkmc: unloaded\n");
		return (0);
	default:
		return (EOPNOTSUPP);
	}
}

static moduledata_t lkmc_mod = { "lkmc", lkmc_modevent, 0 };
DECLARE_MODULE(lkmc, lkmc_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);