/*
 * slabstrand.c - KLD driver to exercise the kern_slaballoc.c remote-free
 * (cross-cpu kfree) path and detect stranded slab zones.
 *
 * Finding: DF-2721 - lost-wakeup race in the z_RSignal interlock.
 *
 * Exposes:
 *	kern.slabstrand.size   - chunk size used by push (default 2048)
 *	kern.slabstrand.push   - (write n) kmalloc up to n chunks into the ring
 *				 (caller should be pinned to cpu0)
 *	kern.slabstrand.pop    - (write n) kfree up to n chunks from the ring
 *				 (caller should be pinned to another cpu ->
 *				 exercises _kfree() z_CpuGd != gd remote path)
 *	kern.slabstrand.drain  - (write 1) free everything left in the ring
 *	kern.slabstrand.stats  - (read)  ops counters
 *	kern.slabstrand.census - (read)  kernel_map census:
 *				 #VM_SUBSYS_KMALLOC entries of exactly
 *				 ZoneSize bytes (128K on this guest) == the
 *				 number of live slab zones, plus totals.
 *				 A stranded zone is one that appears here
 *				 and never goes away.
 *
 * Build: make -f /tmp/Makefile.slabstrand
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <sys/errno.h>
#include <sys/types.h>
#include <sys/thread.h>
#include <sys/thread2.h>

#include <vm/vm.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>

MALLOC_DEFINE(M_STRAND, "slabstrand", "slab strand test");

#define RING_IDX	1024
#define RING_MASK	(RING_IDX - 1)

#define NRING	2			/* one SPSC ring per consumer	*/
struct sring {
	void		*slots[RING_IDX];
	volatile u_int head;		/* consumer index		*/
	volatile u_int tail;		/* producer index		*/
};
static struct sring rings[NRING];
static int slabstrand_size = 2048;	/* chunk size			*/
static long op_alloc;
static long op_free;
static long op_full;			/* ring-full backoffs		*/
static long op_empty;			/* ring-empty backoffs		*/

/*
 * In-kernel hammer: lwkt threads bound to specific cpus hammering
 * kmalloc (producer, cpu0) and kfree (consumer, cpu1[/cpu2]) so the
 * kfree() remote path (z_CpuGd != gd) is exercised at memory speed.
 */
static volatile int krun_secs;		/* seconds remaining (approx)	*/
static volatile int krun_end;
static volatile long k_alloc;
static volatile long k_free;
static int krun_which = 7;		/* bitmask of consumer cpus	*/

static void
producer_loop(void *arg)
{
	int cpuid = (int)(intptr_t)arg;

	{
		int ridx = 0;
		long burst = 0;

		while (!krun_end && cpuid == mycpu->gd_cpuid) {
			struct sring *sr;
			if (++burst >= 2048) {
				burst = 0;
				tsleep(&burst, 0, "sspb", 1);
				continue;
			}
			sr = &rings[ridx];
			u_int h = sr->head;
			u_int t = sr->tail;

			ridx = (ridx + 1) % NRING;
			if (t - h >= RING_IDX - 1) {
				++op_full;
				tsleep(&burst, 0, "sspf", 1);
				continue;
			}
			sr->slots[t & RING_MASK] =
				kmalloc(slabstrand_size, M_STRAND, M_WAITOK);
			cpu_sfence();
			sr->tail = t + 1;
			++k_alloc;
		}
	}
	atomic_add_int(&krun_end, 0);
	++krun_secs;			/* bogus, keeps var live */
	lwkt_exit();
}

static void
consumer_loop(void *arg)
{
	int cpuid = (int)(intptr_t)arg;

	{
		struct sring *sr = &rings[cpuid - 1];
		long burst = 0;

		while (!krun_end && cpuid == mycpu->gd_cpuid) {
			u_int h = sr->head;
			u_int t = sr->tail;
			void *p;

			if (++burst >= 2048) {
				burst = 0;
				tsleep(&burst, 0, "sscb", 1);
				continue;
			}
			if (h == t) {
				++op_empty;
				tsleep(&burst, 0, "sscf", 1);
				continue;
			}
			p = sr->slots[h & RING_MASK];
			cpu_sfence();
			sr->head = h + 1;
			kfree(p, M_STRAND);	/* remote free if cpu != 0 */
			++k_free;
		}
	}
	++krun_secs;
	lwkt_exit();
}

static int
sysctl_krun(SYSCTL_HANDLER_ARGS)
{
	int error, n = 0;
	struct thread *tprod, *tcons1, *tcons2;
	int started = 0;

	error = sysctl_handle_int(oidp, &n, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	if (n <= 0 || n > 300)
		return (EINVAL);

	/* drain the rings first so the run starts from a known state */
	{
		int r;

		for (r = 0; r < NRING; ++r) {
			struct sring *sr = &rings[r];
			while (sr->head != sr->tail) {
				u_int h = sr->head;
				void *p = sr->slots[h & RING_MASK];

				cpu_sfence();
				sr->head = h + 1;
				kfree(p, M_STRAND);
				++op_free;
			}
		}
	}
	k_alloc = k_free = op_full = op_empty = 0;
	krun_end = 0;

	kprintf("slabstrand: run %ds size=%d which=%d\n",
		n, slabstrand_size, krun_which);

	lwkt_create(producer_loop, (void *)(intptr_t)0, &tprod, NULL,
		    TDF_NOSTART | TDF_FIXEDCPU, 0, "sstrand_p");
	++started;
	if (krun_which & 2) {
		lwkt_create(consumer_loop, (void *)(intptr_t)1, &tcons1, NULL,
			    TDF_NOSTART | TDF_FIXEDCPU, 1, "sstrand_c1");
		++started;
	}
	if (krun_which & 4) {
		lwkt_create(consumer_loop, (void *)(intptr_t)2, &tcons2, NULL,
			    TDF_NOSTART | TDF_FIXEDCPU, 2, "sstrand_c2");
		++started;
	}
	lwkt_schedule(tprod);
	if (krun_which & 2)
		lwkt_schedule(tcons1);
	if (krun_which & 4)
		lwkt_schedule(tcons2);

	/* wait out the run duration */
	tsleep(&krun_secs, 0, "sstrand", n * hz);
	krun_end = 1;
	/* let threads notice and exit */
	tsleep(&krun_secs, 0, "sstrand2", 2 * hz);
	{
		vm_map_entry_t entry;
		int zones = 0, kz = 0;

		vm_map_lock(kernel_map);
		RB_FOREACH(entry, vm_map_rb_tree, &kernel_map->rb_root) {
			if (entry->id == VM_SUBSYS_KMALLOC) {
				++kz;
				if (entry->ba.end - entry->ba.start == 131072)
					++zones;
			}
		}
		vm_map_unlock(kernel_map);
		kprintf("slabstrand: done k_alloc=%ld k_free=%ld full=%ld "
			"empty=%ld zones128k=%d kentries=%d wire=%ld\n",
			k_alloc, k_free, op_full, op_empty, zones, kz,
			(long)vmstats.v_wire_count);
	}
	return (0);
}

static int
sysctl_push(SYSCTL_HANDLER_ARGS)
{
	int error, n = 0;

	error = sysctl_handle_int(oidp, &n, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	while (n-- > 0) {
		static int ridx;
		struct sring *sr = &rings[ridx % NRING];
		u_int h = sr->head;
		u_int t = sr->tail;

		++ridx;
		if (t - h >= RING_IDX - 1) {
			++op_full;
			break;
		}
		sr->slots[t & RING_MASK] = kmalloc(slabstrand_size, M_STRAND,
					      M_WAITOK);
		cpu_sfence();
		sr->tail = t + 1;
		++op_alloc;
	}
	return (0);
}

static int
sysctl_pop(SYSCTL_HANDLER_ARGS)
{
	int error, n = 0;

	error = sysctl_handle_int(oidp, &n, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	while (n-- > 0) {
		struct sring *sr = &rings[mycpuid % NRING];
		u_int h = sr->head;
		u_int t = sr->tail;
		void *p;

		if (h == t) {
			++op_empty;
			break;
		}
		p = sr->slots[h & RING_MASK];
		cpu_sfence();
		sr->head = h + 1;
		kfree(p, M_STRAND);
		++op_free;
	}
	return (0);
}

static int
sysctl_drain(SYSCTL_HANDLER_ARGS)
{
	int error, n = 0;

	error = sysctl_handle_int(oidp, &n, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	{
		int r;

		for (r = 0; r < NRING; ++r) {
			struct sring *sr = &rings[r];
			while (sr->head != sr->tail) {
				u_int h = sr->head;
				void *p = sr->slots[h & RING_MASK];

				cpu_sfence();
				sr->head = h + 1;
				kfree(p, M_STRAND);
				++op_free;
			}
		}
	}
	return (0);
}

/*
 * Census of kernel_map: count slab zones.  Zones are exactly one
 * VM_SUBSYS_KMALLOC map entry of ZoneSize bytes (128K on this guest).
 */
static int
sysctl_census(SYSCTL_HANDLER_ARGS)
{
	vm_map_entry_t entry;
	int zones = 0;		/* entries of exactly 128K		*/
	int kz = 0;		/* all KMALLOC entries			*/
	int kvabytes = 0;
	char buf[160];
	int error, len;

	vm_map_lock(kernel_map);
	RB_FOREACH(entry, vm_map_rb_tree, &kernel_map->rb_root) {
		if (entry->id == VM_SUBSYS_KMALLOC) {
			++kz;
			kvabytes += (entry->ba.end - entry->ba.start) / 1024;
			if (entry->ba.end - entry->ba.start == 131072)
				++zones;
		}
	}
	vm_map_unlock(kernel_map);

	len = ksnprintf(buf, sizeof(buf),
			"zones128k=%d kmalloc_entries=%d kmalloc_kva_kb=%d "
			"map_kb=%ld wire=%ld alloc=%ld free=%ld kalloc=%ld kfree=%ld",
			zones, kz, kvabytes, (long)(kernel_map->size / 1024),
			(long)vmstats.v_wire_count, op_alloc, op_free,
			k_alloc, k_free);
	error = SYSCTL_OUT(req, buf, len + 1);
	return (error);
}

static int
sysctl_stats(SYSCTL_HANDLER_ARGS)
{
	char buf[128];
	int error, len;

	len = ksnprintf(buf, sizeof(buf),
			"alloc=%ld free=%ld full=%ld empty=%ld "
			"h0=%u t0=%u h1=%u t1=%u",
			op_alloc, op_free, op_full, op_empty,
			rings[0].head, rings[0].tail,
			rings[1].head, rings[1].tail);
	error = SYSCTL_OUT(req, buf, len + 1);
	return (error);
}

static int
sysctl_size(SYSCTL_HANDLER_ARGS)
{
	int error, s = slabstrand_size;

	error = sysctl_handle_int(oidp, &s, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	if (rings[0].head != rings[0].tail ||
	    rings[NRING-1].head != rings[NRING-1].tail)
		return (EBUSY);
	if (s < 16 || s > 16384)
		return (EINVAL);
	slabstrand_size = s;
	return (0);
}

static SYSCTL_NODE(_kern, OID_AUTO, slabstrand, CTLFLAG_RW, 0, "slab strand");
SYSCTL_PROC(_kern_slabstrand, OID_AUTO, push,
	    CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_push, "I", "push n chunks");
SYSCTL_PROC(_kern_slabstrand, OID_AUTO, pop,
	    CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_pop, "I", "pop n chunks");
SYSCTL_PROC(_kern_slabstrand, OID_AUTO, drain,
	    CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_drain, "I", "drain ring");
SYSCTL_PROC(_kern_slabstrand, OID_AUTO, size,
	    CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_size, "I", "chunk size");
SYSCTL_PROC(_kern_slabstrand, OID_AUTO, census,
	    CTLTYPE_STRING | CTLFLAG_RD, 0, 0, sysctl_census, "A", "census");
SYSCTL_PROC(_kern_slabstrand, OID_AUTO, stats,
	    CTLTYPE_STRING | CTLFLAG_RD, 0, 0, sysctl_stats, "A", "stats");
SYSCTL_PROC(_kern_slabstrand, OID_AUTO, krun,
	    CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_krun, "I",
	    "in-kernel hammer for N seconds");
SYSCTL_INT(_kern_slabstrand, OID_AUTO, which,
	    CTLFLAG_RW, &krun_which, 0, "consumer cpu bitmask");

static int
slabstrand_modevent(module_t mod, int type, void *data)
{
	switch (type) {
	case MOD_LOAD:
	case MOD_UNLOAD:
		break;
	default:
		return (EOPNOTSUPP);
	}
	return (0);
}

static moduledata_t slabstrand_mod = { "slabstrand", slabstrand_modevent, 0 };
DECLARE_MODULE(slabstrand, slabstrand_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);