/*
 * ochang.c - KLD driver proving the objcache_get() M_WAITOK lost-wakeup
 * (remote per-cpu magazine stranding) in sys/kern/kern_objcache.c.
 *
 * Finding DF-2813.
 *
 * Scenario (deterministic, private objcache):
 *   ARM     - drain the cache with M_NOWAIT gets until exhausted
 *             (nheld == ncpus*2*magcap + cluster_limit objects held live;
 *              this also *derives* magcap at runtime: (nheld-limit)/(2*ncpus)).
 *   SLEEPER - kernel thread pinned to cpu5 calls objcache_get(oc, M_WAITOK);
 *             cache exhausted, depot empty, own mags empty -> ssleep(depot)
 *             forever (flags=0 => no PCATCH, timo=0 => no timeout).
 *   STRAND  - free exactly magcap objects on cpu0 (via IPI): they land in
 *             cpu0's loaded magazine and NEVER reach the depot.  The hot-path
 *             wakeup check is cpucache-of-the-PUTTER -> wakeup_mycpu, which
 *             cannot see the remote sleeper.  Objects are available, the
 *             sleeper never wakes.  Observe at +1s/+5s/+10s.
 *   PROBE   - behavioral proof of stranding: run one objcache_get(M_NOWAIT)
 *             + objcache_put() pair on the chosen cpu via IPI.  On the
 *             strand cpu it returns an object (from the per-cpu magazine,
 *             before any exhaustion check); on the sleeper cpu it returns
 *             NULL (exhausted).  Same cache, same instant, different cpus.
 *   RESCUE  - positive control: free enough more objects on cpu0 to fill
 *             both cpu0 magazines; the depot cycle (objcache_put depot path)
 *             deposits a full magazine and does wakeup(depot) -> the sleeper
 *             wakes within milliseconds.
 *
 * sysctls:
 *   kern.ochang.cmd        (write) 1=ARM 2=SLEEPER 3=STRAND 4=RESCUE 5=DRAIN 6=DESTROY 7=PROBE(strand cpu) 8=PROBE(sleeper cpu)
 *   kern.ochang.status     (read)  counters + sleeper state
 *   kern.ochang.strand_cpu (int) default 0
 *   kern.ochang.sleeper_cpu(int) default 5
 *   kern.ochang.nstrand    (int) default 0 = auto (derived magcap)
 *   kern.ochang.nrescue    (int) default 0 = auto (2*magcap)
 *   kern.ochang.last_probe (read) result of last PROBE: 1 obj, 0 NULL
 *
 * Build: make ; load with kldload ./ochang.ko
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/malloc.h>
#include <sys/objcache.h>
#include <sys/sysctl.h>
#include <sys/thread.h>
#include <sys/thread2.h>
#include <sys/globaldata.h>
#include <sys/types.h>

MALLOC_DEFINE(M_OCHANG, "ochang", "objcache hang test");

#define CLUSTER_LIMIT	8
#define MAXOBJ		512

static struct objcache *oc;
static struct objcache_malloc_args margs = { 16, M_OCHANG };

static void *held[MAXOBJ];
static int nheld;
static int consumed;			/* held[0..consumed-1] already put */

static struct thread *sleeper_td;
static volatile int sleeper_started;
static volatile int sleeper_got;	/* 0 = still inside objcache_get */
static volatile int sleeper_start_ticks;
static volatile int sleeper_end_ticks;
static void *sleeper_obj;

static int strand_cpu = 0;
static int sleeper_cpu = 5;
static int nstrand;			/* 0 => derived from ARM */
static int nrescue;			/* 0 => 2 * nstrand */
static int magcap;
static int last_probe = -1;

static struct ipisync {
	volatile int done;
} ipisync;

static void
sleeper_thread(void *arg)
{
	void *obj;

	sleeper_started = 1;
	sleeper_start_ticks = ticks;
	obj = objcache_get(oc, M_WAITOK);	/* expected: sleeps forever */
	sleeper_end_ticks = ticks;
	sleeper_obj = obj;
	if (obj)
		sleeper_got = 1;
	else
		sleeper_got = -1;
	/* lwkt exit */
}

static void
ipi_puts_cb(void *arg1, int arg2, struct intrframe *frame __unused)
{
	struct ipisync *s = arg1;
	int i;

	for (i = 0; i < arg2; i++)
		objcache_put(oc, held[consumed + i]);
	s->done = 1;
	wakeup(s);
}

static void
ipi_probe_cb(void *arg1, int arg2, struct intrframe *frame __unused)
{
	struct ipisync *s = arg1;
	void *obj;

	obj = objcache_get(oc, M_NOWAIT);
	if (obj != NULL) {
		last_probe = 1;
		objcache_put(oc, obj);	/* return it to the same cpu mag */
	} else {
		last_probe = 0;
	}
	s->done = 1;
	wakeup(s);
}

static int
puts_on_cpu(int cpu, int n)
{
	if (consumed + n > nheld)
		return (EINVAL);
	ipisync.done = 0;
	lwkt_send_ipiq3(globaldata_find(cpu), ipi_puts_cb, &ipisync, n);
	tsleep(&ipisync, 0, "oipi", 5 * hz);
	if (!ipisync.done)
		return (ETIMEDOUT);
	consumed += n;
	return (0);
}

static int
probe_cpu(int cpu)
{
	ipisync.done = 0;
	lwkt_send_ipiq3(globaldata_find(cpu), ipi_probe_cb, &ipisync, 0);
	tsleep(&ipisync, 0, "oprobe", 5 * hz);
	if (!ipisync.done)
		return (-1);
	return (last_probe);
}

static int
ochang_cmd_sysctl(SYSCTL_HANDLER_ARGS)
{
	int cmd, error;

	cmd = 0;
	error = sysctl_handle_int(oidp, &cmd, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);

	switch (cmd) {
	case 1:				/* ARM: exhaust the cache */
		if (oc == NULL)
			return (ENXIO);
		nheld = 0;
		consumed = 0;
		while (nheld < MAXOBJ) {
			void *p = objcache_get(oc, M_NOWAIT);
			if (p == NULL)
				break;
			held[nheld++] = p;
		}
		magcap = (nheld - CLUSTER_LIMIT) / (2 * ncpus);
		if (nstrand == 0)
			nstrand = magcap;
		if (nrescue == 0)
			nrescue = 2 * magcap;
		kprintf("ochang: ARM: drained %d objects, magcap=%d "
		    "(expect %d)\n", nheld, magcap,
		    ncpus * 2 * magcap + CLUSTER_LIMIT);
		break;
	case 2:				/* SLEEPER: thread on sleeper_cpu */
		if (oc == NULL || sleeper_td != NULL)
			return (EBUSY);
		sleeper_started = 0;
		sleeper_got = 0;
		lwkt_create(sleeper_thread, NULL, &sleeper_td, NULL, 0,
			    sleeper_cpu, "ochang_slp");
		break;
	case 3:				/* STRAND: nstrand puts on strand_cpu */
		if (oc == NULL)
			return (ENXIO);
		error = puts_on_cpu(strand_cpu, nstrand);
		kprintf("ochang: STRAND: %d puts on cpu%d -> %d\n",
		    nstrand, strand_cpu, error);
		break;
	case 4:				/* RESCUE: nrescue more puts */
		if (oc == NULL)
			return (ENXIO);
		error = puts_on_cpu(strand_cpu, nrescue);
		kprintf("ochang: RESCUE: %d puts on cpu%d -> %d\n",
		    nrescue, strand_cpu, error);
		break;
	case 5:				/* DRAIN: return the rest */
		if (oc == NULL)
			return (ENXIO);
		if (sleeper_got == 1 && sleeper_obj != NULL) {
			objcache_put(oc, sleeper_obj);
			sleeper_obj = NULL;
		}
		while (consumed < nheld) {
			objcache_put(oc, held[consumed]);
			consumed++;
		}
		kprintf("ochang: DRAIN: all objects returned\n");
		break;
	case 6:				/* DESTROY */
		if (oc == NULL)
			return (ENXIO);
		if (sleeper_td != NULL && sleeper_got == 0)
			return (EBUSY);	/* still hung */
		objcache_destroy(oc);
		oc = NULL;
		sleeper_td = NULL;
		kprintf("ochang: cache destroyed\n");
		break;
	case 7:				/* PROBE strand cpu */
		if (oc == NULL)
			return (ENXIO);
		last_probe = probe_cpu(strand_cpu);
		kprintf("ochang: PROBE cpu%d -> %s\n", strand_cpu,
		    last_probe == 1 ? "OBJECT AVAILABLE" :
		    last_probe == 0 ? "NULL (exhausted)" : "timeout");
		break;
	case 8:				/* PROBE sleeper cpu */
		if (oc == NULL)
			return (ENXIO);
		last_probe = probe_cpu(sleeper_cpu);
		kprintf("ochang: PROBE cpu%d -> %s\n", sleeper_cpu,
		    last_probe == 1 ? "OBJECT AVAILABLE" :
		    last_probe == 0 ? "NULL (exhausted)" : "timeout");
		break;
	default:
		return (EINVAL);
	}
	return (0);
}

static int
ochang_status_sysctl(SYSCTL_HANDLER_ARGS)
{
	char buf[1024];
	int len = 0;

	len += ksnprintf(buf + len, sizeof(buf) - len,
	    "nheld=%d consumed=%d magcap=%d nstrand=%d nrescue=%d "
	    "last_probe=%d\n",
	    nheld, consumed, magcap, nstrand, nrescue, last_probe);
	len += ksnprintf(buf + len, sizeof(buf) - len,
	    "sleeper: started=%d got=%d", sleeper_started, sleeper_got);
	if (sleeper_started && sleeper_got == 1)
		len += ksnprintf(buf + len, sizeof(buf) - len,
		    " latency=%d ticks", sleeper_end_ticks - sleeper_start_ticks);
	else if (sleeper_started && sleeper_got == 0 && sleeper_td != NULL) {
		int asleep_ticks = ticks - sleeper_start_ticks;
		len += ksnprintf(buf + len, sizeof(buf) - len,
		    " STUCK for %d ticks (%d s), TDF_TSLEEPQ=%d wmesg=\"%s\"",
		    asleep_ticks, asleep_ticks / hz,
		    (sleeper_td->td_flags & TDF_TSLEEPQ) ? 1 : 0,
		    sleeper_td->td_wmesg ? sleeper_td->td_wmesg : "?");
	}
	len += ksnprintf(buf + len, sizeof(buf) - len, "\n");

	return (SYSCTL_OUT(req, buf, len));
}

SYSCTL_NODE(_kern, OID_AUTO, ochang, CTLFLAG_RW, 0, "ochang");
SYSCTL_PROC(_kern_ochang, OID_AUTO, cmd, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, ochang_cmd_sysctl, "I", "ochang command");
SYSCTL_PROC(_kern_ochang, OID_AUTO, status, CTLTYPE_STRING | CTLFLAG_RD,
    0, 0, ochang_status_sysctl, "A", "ochang status");
SYSCTL_INT(_kern_ochang, OID_AUTO, strand_cpu, CTLFLAG_RW,
    &strand_cpu, 0, "cpu where stranded frees happen");
SYSCTL_INT(_kern_ochang, OID_AUTO, sleeper_cpu, CTLFLAG_RW,
    &sleeper_cpu, 0, "cpu of the sleeping getter");
SYSCTL_INT(_kern_ochang, OID_AUTO, nstrand, CTLFLAG_RW,
    &nstrand, 0, "objects freed on strand cpu (0=auto)");
SYSCTL_INT(_kern_ochang, OID_AUTO, nrescue, CTLFLAG_RW,
    &nrescue, 0, "objects freed for rescue (0=auto)");
SYSCTL_INT(_kern_ochang, OID_AUTO, last_probe, CTLFLAG_RD,
    &last_probe, 0, "result of last probe");

static int
ochang_modevent(module_t mod, int type, void *data)
{
	switch (type) {
	case MOD_LOAD:
		oc = objcache_create("ochang", CLUSTER_LIMIT, 0,
				     NULL, NULL, NULL,
				     objcache_malloc_alloc,
				     objcache_malloc_free, &margs);
		kprintf("ochang: cache created, ncpus=%d\n", ncpus);
		break;
	case MOD_UNLOAD:
		if (oc != NULL) {
			kprintf("ochang: cache still alive, refusing "
			    "(cmd 4 then 5 6 first)\n");
			return (EBUSY);
		}
		break;
	default:
		break;
	}
	return (0);
}

static moduledata_t ochang_mod = {
	"ochang",
	ochang_modevent,
	NULL
};
DECLARE_MODULE(ochang, ochang_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);