/*
 * DF-5839 trigger module for sys/kern/lwkt_thread.c (DragonFly 6.5-DEVELOPMENT)
 *
 * The lwkt_exit() -> gd_freetd publication window is only protected against
 * the OWNER cpu (a per-cpu critical section, lwkt_thread.c:1707 crit_enter_quick
 * .. :1742 gd->gd_freetd = td).  lwkt_hold()/lwkt_rele() are cross-cpu atomic
 * operations (lwkt_thread.c:497-507) that a critical section on cpu A cannot
 * exclude: a lwkt_hold() issued by a *foreign* cpu that lands after the exit
 * path's final td_refs==0 check (lwkt_thread.c:1709 loop / :1735 KKASSERT)
 * publishes a corpse into gd_freetd with td_refs == 1.
 *
 * The corpse is then silently reborn: lwkt_alloc_thread() grabs gd_freetd
 * (lwkt_thread.c:365-368) checking ONLY TDF_RUNNING|TDF_PREEMPT_LOCK|TDF_RUNQ
 * -- never td_refs -- and lwkt_init_thread() bzero()s the whole struct
 * (lwkt_thread.c:445), wiping the live refcount to 0.  The foreign cpu's
 * later lwkt_rele() then decrements the REBORN thread's td_refs to -1:
 *
 *   INVARIANTS: panic at lwkt_rele() KKASSERT(td->td_refs > 0) (lwkt_thread.c:505)
 *               -- or, if the hold landed inside the check->publish window,
 *               panic at lwkt_exit() KKASSERT(td->td_refs == 0) (:1735).
 *   stock:      silent -1 refcount; when the reborn thread later exits,
 *               lwkt_exit()'s drain loop ("if (td->td_refs)") spins in
 *               tsleep("tdreap", 1 tick) forever  ==>  permanent thread-exit
 *               livelock + the rele wrote to recycled-then-reused memory.
 *
 * Mode 0 (control): hold/rele the victim from its OWNER cpu (migrate first,
 *   exactly like sysctl kern.proc's tdallq walker, kern_proc.c:1804-1818):
 *   the exit drain loop observes the ref and waits; clean exit; clean rebirth.
 *   Must NOT panic -- proves the owner-cpu discipline is what makes it safe.
 *
 * Mode 1 (deterministic): hold the *published corpse* from a foreign cpu
 *   (watch cpu1's gd_freetd until it equals the victim and TDF_RUNNING is
 *   clear), force cpu1 to consume the corpse with kthread_create() (grabs
 *   gd_freetd on the allocating cpu), then lwkt_rele() the reborn thread.
 *   Expected: panic "td->td_refs > 0" in lwkt_rele.
 *
 * Mode 2 (racy window): tight-spin on (TDF_RUNQ|TDF_TSLEEPQ)==0 -- the state
 *   first reached between lwkt_deschedule_self() (:1733) and the publication
 *   (:1742) -- and lwkt_hold() immediately.  Landing before :1735's KKASSERT
 *   panics in lwkt_exit; landing after publication is converted to the
 *   mode-1 chain.  N attempts with fresh victims.
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/malloc.h>
#include <sys/thread.h>
#include <sys/kthread.h>
#include <sys/globaldata.h>
#include <machine/smp.h>
#include <machine/cpu.h>
#include <sys/lock.h>

static struct thread *watcher_td;
static struct thread *spawner_td;
static struct thread *victim_td;
static struct thread *child_td;

static volatile int victim_go;
static volatile int spawner_go;
static volatile int test_mode;
static volatile int test_done;	/* 1 = finished, 2 = child reborn */
static volatile int last_result;

#define RES_MISS_PUBLISH	-1
#define RES_CONTROL_OK		0
#define RES_REBORN_RELE		1
#define RES_WINDOW_HELD		2
#define RES_CONSUMED_ELSEWHERE	3

#define TDF_POLL	(TDF_RUNQ | TDF_TSLEEPQ)

static void
victim_func(void *arg)
{
	kprintf("df5839: victim %p running on cpu%d\n",
		curthread, mycpu->gd_cpuid);
	while (!victim_go)
		tsleep(&victim_go, 0, "df5839v", 1);
	kprintf("df5839: victim %p exiting on cpu%d\n",
		curthread, mycpu->gd_cpuid);
	kthread_exit();
}

static void
child_func(void *arg)
{
	kprintf("df5839: child %p (reborn corpse) running on cpu%d refs=%d\n",
		curthread, mycpu->gd_cpuid, curthread->td_refs);
	/* stay parked; do not exit during the test */
	for (;;)
		tsleep(child_td, 0, "df5839c", 0);
}

static void
spawner_func(void *arg)
{
	for (;;) {
		while (!spawner_go)
			tsleep(&spawner_go, 0, "df5839s", 1);
		spawner_go = 0;
		kprintf("df5839: spawner on cpu%d consuming gd_freetd=%p\n",
			mycpu->gd_cpuid, mycpu->gd_freetd);
		kthread_create(child_func, NULL, &child_td, "df5839c");
		kprintf("df5839: child td=%p  (== victim corpse: %s) "
			"victim td_refs now=%d\n",
			child_td, (child_td == victim_td) ? "YES" : "no",
			victim_td->td_refs);
		test_done = 2;
		wakeup(&test_done);
	}
}

/*
 * Wait for the corpse to be fully switched out, hand the ref to the
 * spawner, and release the stale ref on the reborn thread.
 */
static int
rebirth_and_rele(globaldata_t gd1)
{
	/* publication observed; wait for TDF_RUNNING to clear (switch_return) */
	while (victim_td->td_flags & TDF_RUNNING) {
		if (tsleep(&test_done, 0, "df5839w1", 1))
			break;
	}
	kprintf("df5839: foreign hold on cached corpse %p from cpu%d "
		"(refs=%d freetd=%p)\n",
		victim_td, mycpu->gd_cpuid, victim_td->td_refs,
		gd1->gd_freetd);
	lwkt_hold(victim_td);		/* corpse refs 0 -> 1 */

	spawner_go = 1;
	wakeup(&spawner_go);
	while (test_done != 2) {
		if (tsleep(&test_done, 0, "df5839w2", 1))
			break;
	}
	if (child_td != victim_td) {
		kprintf("df5839: corpse consumed elsewhere; benign\n");
		lwkt_rele(victim_td);
		return (RES_CONSUMED_ELSEWHERE);
	}
	kprintf("df5839: corpse REBORN as %p, td_refs wiped to %d; "
		"releasing stale foreign ref\n", child_td, victim_td->td_refs);
	lwkt_rele(victim_td);		/* reborn refs 0 -> -1  *** panic *** */
	kprintf("df5839: NOT PANICKED? victim td_refs=%d\n",
		victim_td->td_refs);
	return (RES_REBORN_RELE);
}

static void
watcher_func(void *arg)
{
	globaldata_t gd1 = globaldata_find(1);
	int mode;
	int i;

	kprintf("df5839: watcher %p on cpu%d\n", curthread, mycpu->gd_cpuid);
	for (;;) {
		while (!test_mode)
			tsleep(&test_mode, 0, "df5839w", 1);
		mode = test_mode;
		test_mode = 0;
		test_done = 0;
		last_result = -99;

		if (mode == 0) {
			/*
			 * CONTROL: owner-cpu hold/rele (kern_proc walker
			 * discipline).  Migrate to cpu1 FIRST.
			 */
			kprintf("df5839: mode0 control: migrating watcher "
				"to victim's cpu\n");
			lwkt_setcpu_self(gd1);
			victim_go = 0;
			kthread_create_cpu(victim_func, NULL, &victim_td, 1,
					   "df5839v0");
			/* wait until victim is asleep */
			while ((victim_td->td_flags & TDF_TSLEEPQ) == 0)
				tsleep(&test_done, 0, "df5839c0", 1);
			kprintf("df5839: mode0: owner-cpu hold of live "
				"victim %p (refs=%d)\n",
				victim_td, victim_td->td_refs);
			lwkt_hold(victim_td);
			victim_go = 1;
			wakeup(&victim_go);
			/* exit drain loop must wait for us */
			tsleep(&test_done, 0, "df5839c1", 2);
			kprintf("df5839: mode0: rele after drain wait "
				"(refs=%d)\n", victim_td->td_refs);
			lwkt_rele(victim_td);
			for (i = 0; i < 10; ++i) {
				if (gd1->gd_freetd == victim_td)
					break;
				tsleep(&test_done, 0, "df5839c2", 1);
			}
			kprintf("df5839: mode0: victim drained & exited "
				"cleanly (freetd==victim: %d, refs=%d)\n",
				gd1->gd_freetd == victim_td,
				victim_td->td_refs);
			/* rebirth via spawner to prove the cycle is clean */
			test_done = 0;
			spawner_go = 1;
			wakeup(&spawner_go);
			while (test_done != 2)
				tsleep(&test_done, 0, "df5839c3", 1);
			kprintf("df5839: mode0: clean rebirth child=%p "
				"refs=%d -- control OK\n",
				child_td, child_td->td_refs);
			last_result = RES_CONTROL_OK;
		} else if (mode == 1) {
			victim_go = 0;
			kthread_create_cpu(victim_func, NULL, &victim_td, 1,
					   "df5839v1");
			while ((victim_td->td_flags & TDF_TSLEEPQ) == 0)
				tsleep(&test_done, 0, "df5839m1", 1);
			victim_go = 1;
			wakeup(&victim_go);
			for (i = 0; i < 50; ++i) {
				if (gd1->gd_freetd == victim_td)
					break;
				tsleep(&test_done, 0, "df5839m2", 1);
			}
			if (gd1->gd_freetd != victim_td) {
				kprintf("df5839: mode1: publication not "
					"observed (miss)\n");
				last_result = RES_MISS_PUBLISH;
			} else {
				last_result = rebirth_and_rele(gd1);
			}
		} else if (mode == 2) {
			int attempt;

			for (attempt = 0; attempt < 30; ++attempt) {
				victim_go = 0;
				child_td = NULL;
				test_done = 0;
				kthread_create_cpu(victim_func, NULL,
						   &victim_td, 1,
						   "df5839w%d", attempt);
				while ((victim_td->td_flags & TDF_TSLEEPQ) == 0)
					tsleep(&test_done, 0, "df5839r0", 1);
				victim_go = 1;
				wakeup(&victim_go);

				/*
				 * Tight spin: (RUNQ|TSLEEPQ)==0 is first
				 * reached between lwkt_deschedule_self()
				 * and the gd_freetd publication.
				 */
				cpu_ccfence();
				while (victim_td->td_flags & TDF_POLL)
					cpu_ccfence();
				lwkt_hold(victim_td);
				kprintf("df5839: mode2 attempt %d: held at "
					"refs=%d freetd=%p RUNNING=%d\n",
					attempt, victim_td->td_refs,
					gd1->gd_freetd,
					!!(victim_td->td_flags &
					   TDF_RUNNING));
				if (victim_td->td_refs != 1) {
					/* landed post-rebirth (live thread) */
					lwkt_rele(victim_td);
					continue;
				}
				for (i = 0; i < 20; ++i) {
					if (gd1->gd_freetd == victim_td)
						break;
					tsleep(&test_done, 0, "df5839r1", 1);
				}
				if (gd1->gd_freetd != victim_td) {
					/*
					 * hold landed inside the
					 * check->publish window: lwkt_exit
					 * KKASSERT(refs==0) fires (or the
					 * exit already drained elsewhere)
					 */
					kprintf("df5839: mode2: window hold; "
						"waiting exit outcome\n");
					for (i = 0; i < 20; ++i)
						tsleep(&test_done, 0,
						       "df5839r2", 1);
					lwkt_rele(victim_td);
					last_result = RES_WINDOW_HELD;
					break;
				}
				last_result = rebirth_and_rele(gd1);
				break;
			}
			if (last_result == -99)
				last_result = RES_MISS_PUBLISH;
		}

		test_done = 1;
		wakeup(&test_done);
	}
}

static int
sysctl_df5839(SYSCTL_HANDLER_ARGS)
{
	int mode;
	int error;
	int i;

	error = sysctl_handle_int(oidp, &mode, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	if (mode < 0 || mode > 2)
		return (EINVAL);

	kprintf("df5839: triggering mode %d\n", mode);
	test_done = 0;
	test_mode = mode;
	wakeup(&test_mode);

	for (i = 0; i < 100; ++i) {
		if (test_done)
			break;
		/* tsleep returns EWOULDBLOCK on timeout: keep waiting */
		tsleep(&test_done, 0, "df5839t", 1);
	}
	kprintf("df5839: mode %d finished, last_result=%d refs(victim)=%d\n",
		mode, last_result, victim_td ? victim_td->td_refs : -9);
	return (0);
}

SYSCTL_NODE(_debug, OID_AUTO, df5839, CTLFLAG_RW, 0, "DF-5839 trigger");
SYSCTL_PROC(_debug_df5839, OID_AUTO, run, CTLTYPE_INT | CTLFLAG_RW,
	    0, 0, sysctl_df5839, "I", "0=control 1=deterministic 2=window-race");

static void
df5839_load(void *arg)
{
	int error;

	error = kthread_create_cpu(watcher_func, NULL, &watcher_td, 0,
				   "df5839w");
	if (error) {
		kprintf("df5839: watcher create failed %d\n", error);
		return;
	}
	error = kthread_create_cpu(spawner_func, NULL, &spawner_td, 1,
				   "df5839s");
	if (error) {
		kprintf("df5839: spawner create failed %d\n", error);
		return;
	}
	kprintf("df5839: loaded (watcher=cpu0 %p, spawner=cpu1 %p)\n",
		watcher_td, spawner_td);
}

SYSINIT(df5839_load, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, df5839_load, NULL);