/*
 * DF-5442 - per-CPU variable-MTRR snapshot module.
 *
 * debug.mtrr_snap collects MSR 0x200..0x221 (SNAP_SLOTS variable MTRR
 * base/mask pairs) plus MTRRdefType from EVERY cpu using
 * lwkt_cpusync_simple() - a full rendezvous barrier.  Because the
 * interlock IPI is queued behind any pending IPI FIFO entries, a read
 * of this sysctl drains all in-flight amd64_mrstoreone() work before
 * sampling: cross-CPU divergence in one snapshot therefore proves a
 * persistent torn MTRR (the desync the race leaves behind), not merely
 * lag.
 *
 * The compare step reports slots whose (base,mask) pair differs across
 * CPUs, plus a summary line:
 *   "MTRR-SNAP: cpus=%d divergent_slots=%d"
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/malloc.h>
#include <sys/thread.h>
#include <sys/thread2.h>

#include <machine/cpufunc.h>
#include <machine/clock.h>
#include <machine/smp.h>
#include <machine/specialreg.h>

#define	SNAP_SLOTS	8			/* var MTRR pairs 0x200..0x20F (KVM vcnt=8) */
#define	SNAP_MAXCPU	MAXCPU

struct mtrr_snap {
	u_int64_t	base[SNAP_SLOTS];
	u_int64_t	mask[SNAP_SLOTS];
	u_int64_t	deftype;
	int		base_err[SNAP_SLOTS];	/* msr unreadable on this cpu */
	int		mask_err[SNAP_SLOTS];
	int		deftype_err;
	int		touched;
};

static struct mtrr_snap snaps[SNAP_MAXCPU];
static int snap_ncpu;

static void
snapfun(void *arg)
{
	struct globaldata *gd = mycpu;
	struct mtrr_snap *s = &snaps[gd->gd_cpuid];
	int i;

	if (gd->gd_cpuid < 0 || gd->gd_cpuid >= SNAP_MAXCPU)
		return;
	for (i = 0; i < SNAP_SLOTS; i++) {
		s->base_err[i] = rdmsr_safe(MSR_MTRRVarBase + 2 * i,
		    &s->base[i]);
		s->mask_err[i] = rdmsr_safe(MSR_MTRRVarBase + 2 * i + 1,
		    &s->mask[i]);
	}
	s->deftype_err = rdmsr_safe(MSR_MTRRdefType, &s->deftype);
	s->touched = 1;
}

static int
sysctl_mtrr_snap(SYSCTL_HANDLER_ARGS)
{
	char *buf, *p, *end;
	size_t bufsz = 65536;
	int cpu, i, j, len, divergent = 0, first;
	int error;

	buf = kmalloc(bufsz, M_TEMP, M_WAITOK | M_ZERO);
	p = buf;
	end = buf + bufsz - 1;

	memset(snaps, 0, sizeof(snaps));

	/* full rendezvous: all cpus in smp_active_mask run snapfun(), and
	 * this returns only after every one of them has completed it */
	lwkt_cpusync_simple(smp_active_mask, snapfun, NULL);

	snap_ncpu = 0;
	for (cpu = 0; cpu < SNAP_MAXCPU; cpu++) {
		if (!snaps[cpu].touched)
			continue;
		snap_ncpu++;
	}

	for (i = 0; i < SNAP_SLOTS; i++) {
		int skip = 0, first;

		/* slots unreadable on any cpu cannot be compared */
		for (cpu = 0; cpu < SNAP_MAXCPU; cpu++) {
			if (!snaps[cpu].touched)
				continue;
			if (snaps[cpu].base_err[i] || snaps[cpu].mask_err[i])
				skip = 1;
		}
		if (skip)
			continue;
		first = -1;
		for (cpu = 0; cpu < SNAP_MAXCPU; cpu++) {
			if (!snaps[cpu].touched)
				continue;
			if (first < 0) {
				first = cpu;
				continue;
			}
			if (snaps[cpu].base[i] != snaps[first].base[i] ||
			    snaps[cpu].mask[i] != snaps[first].mask[i]) {
				if (!divergent)
					p += ksnprintf(p, end - p,
					    "DIVERGENT SLOTS:\n");
				divergent++;
				p += ksnprintf(p, end - p,
				    "  slot %d (MSR 0x%x/0x%x):\n", i,
				    MSR_MTRRVarBase + 2 * i,
				    MSR_MTRRVarBase + 2 * i + 1);
				for (j = 0; j < SNAP_MAXCPU; j++) {
					if (!snaps[j].touched)
						continue;
					p += ksnprintf(p, end - p,
					    "    cpu%-3d base=0x%016llx mask=0x%016llx\n",
					    j,
					    (unsigned long long)snaps[j].base[i],
					    (unsigned long long)snaps[j].mask[i]);
				}
				break;
			}
		}
	}

	p += ksnprintf(p, end - p, "MTRR-SNAP: cpus=%d divergent_slots=%d",
	    snap_ncpu, divergent);
	/* deftype line: readable on all cpus? */
	for (cpu = 0; cpu < SNAP_MAXCPU; cpu++) {
		if (snaps[cpu].touched && snaps[cpu].deftype_err) {
			p += ksnprintf(p, end - p, " deftype_err(cpu%d)", cpu);
			break;
		}
	}
	{
		int dt_first = -1, dt_div = 0;

		for (cpu = 0; cpu < SNAP_MAXCPU; cpu++) {
			if (!snaps[cpu].touched || snaps[cpu].deftype_err)
				continue;
			if (dt_first < 0) {
				dt_first = cpu;
				continue;
			}
			if (snaps[cpu].deftype != snaps[dt_first].deftype)
				dt_div = 1;
		}
		p += ksnprintf(p, end - p, " deftype_divergent=%d deftype=0x%016llx\n",
		    dt_div, (unsigned long long)(dt_first >= 0 ?
		    snaps[dt_first].deftype : 0));
	}

	len = p - buf;
	error = SYSCTL_OUT(req, buf, len);
	kfree(buf, M_TEMP);
	return (error);
}

SYSCTL_PROC(_debug, OID_AUTO, mtrr_snap,
    CTLTYPE_STRING | CTLFLAG_RD, 0, 0, sysctl_mtrr_snap, "A",
    "per-cpu variable MTRR snapshot (post-rendezvous)");

/*
 * DF-5442 instrumentation: busy-IPI injector.  A write of N (usec) fires
 * an IPI to every cpu in smp_active_mask whose handler spins for N usec.
 * This emulates ordinary kernel IPI load (TLB shootdowns, cpusyncs): a
 * remote cpu busy in one IPI entry cannot start the MTRR storeone IPI
 * queued behind it, which is the realistic arrival-delay distribution
 * the amd64_mrstore()/amd64_mrfetch() race needs.  It does NOT touch
 * the memrange code path itself.
 */
static void
busyfun(void *arg, int usec, struct intrframe *frame)
{
	tsc_uclock_t delta;
	tsc_uclock_t target = rdtsc() + (tsc_uclock_t)usec *
	    tsc_frequency / 1000000U;

	while ((delta = rdtsc()) < target)
		cpu_pause();
}

static int
sysctl_mtrr_busy(SYSCTL_HANDLER_ARGS)
{
	int error, usec = 0;

	error = SYSCTL_IN(req, &usec, sizeof(usec));
	if (error == 0 && usec > 0 && usec <= 10000)
		lwkt_send_ipiq3_mask(smp_active_mask,
		    (ipifunc3_t)busyfun, NULL, usec);
	if (error == 0)
		error = SYSCTL_OUT(req, &usec, sizeof(usec));
	return (error);
}

SYSCTL_PROC(_debug, OID_AUTO, mtrr_busy,
    CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_mtrr_busy, "I",
    "fire a usec-spin IPI at all cpus (load injector)");

/*
 * DF-5442 measurement: time the three cost centers of the SET path
 * (variable-MTRR wrmsr sweep, wbinvd, rdmsr sweep) in usec on this cpu,
 * to size the amd64_mrstore/amd64_mrfetch phase windows.
 */
static int
sysctl_mtrr_time(SYSCTL_HANDLER_ARGS)
{
	char buf[256];
	int error, i, len;
	u_int64_t t0, t1, t2, t3, t4;
	u_int64_t v[SNAP_SLOTS][2];
	char *p = buf;

	for (i = 0; i < SNAP_SLOTS; i++) {
		v[i][0] = rdmsr(MSR_MTRRVarBase + 2 * i);
		v[i][1] = rdmsr(MSR_MTRRVarBase + 2 * i + 1);
	}
	t0 = rdtsc();
	for (i = 0; i < SNAP_SLOTS; i++) {
		wrmsr(MSR_MTRRVarBase + 2 * i, v[i][0]);
		wrmsr(MSR_MTRRVarBase + 2 * i + 1, v[i][1]);
	}
	t1 = rdtsc();
	wbinvd();
	t2 = rdtsc();
	for (i = 0; i < SNAP_SLOTS; i++) {
		v[i][0] = rdmsr(MSR_MTRRVarBase + 2 * i);
		v[i][1] = rdmsr(MSR_MTRRVarBase + 2 * i + 1);
	}
	t3 = rdtsc();
	cpu_invltlb();
	t4 = rdtsc();

	p += ksnprintf(p, sizeof(buf) - (p - buf),
	    "var_wrmsr=%llu wbinvd=%llu var_rdmsr=%llu invltlb=%llu usec "
	    "(tsc_frequency=%llu)\n",
	    (unsigned long long)((t1 - t0) * 1000000ULL / tsc_frequency),
	    (unsigned long long)((t2 - t1) * 1000000ULL / tsc_frequency),
	    (unsigned long long)((t3 - t2) * 1000000ULL / tsc_frequency),
	    (unsigned long long)((t4 - t3) * 1000000ULL / tsc_frequency),
	    (unsigned long long)tsc_frequency);
	len = p - buf;
	error = SYSCTL_OUT(req, buf, len);
	return (error);
}

SYSCTL_PROC(_debug, OID_AUTO, mtrr_time,
    CTLTYPE_STRING | CTLFLAG_RD, 0, 0, sysctl_mtrr_time, "A",
    "time the MTRR store/fetch cost centers on this cpu");

static int
mtrrdump_modevent(module_t mod, int type, void *data)
{
	switch (type) {
	case MOD_LOAD:
		kprintf("mtrrdump: loaded (debug.mtrr_snap)\n");
		return (0);
	case MOD_UNLOAD:
		kprintf("mtrrdump: unloaded\n");
		return (0);
	}
	return (EOPNOTSUPP);
}

static moduledata_t mtrrdump_mod = { "mtrrdump", mtrrdump_modevent, 0 };
DECLARE_MODULE(mtrrdump, mtrrdump_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
MODULE_VERSION(mtrrdump, 1);