/*
 * nvmm_drive.c -- DragonFly NVMM (VMX backend, sys/dev/virtual/nvmm/x86/
 * nvmm_x86_vmx.c) guest->host boundary exerciser.  Pass-2 live audit aid.
 *
 * Drives a real (unrestricted-guest, 16-bit real mode) vCPU through the
 * in-kernel exit handlers of nvmm_x86_vmx.c: CPUID, CR0/CR4/CR8, IO,
 * RDMSR/WRMSR, VMX-instruction (#UD injection), and then hammers the
 * state-set / event-injection entry points with hostile values.
 *
 * Build:  cc -O2 -o nvmm_drive nvmm_drive.c -I/usr/src/sys -lpthread
 * Usage:  nvmm_drive run      -- sequential sub-tests (expect: no panic,
 *                               sane exit codes, HALTED at the end)
 *         nvmm_drive spin SEC [NVCPUS] -- keep vCPUs resident (NMI test)
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <time.h>
#include <pthread.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <dev/virtual/nvmm/nvmm.h>
#include <dev/virtual/nvmm/nvmm_ioctl.h>

#define HVA_BASE	0x10000000UL
#define RAM_SIZE	(2UL*1024*1024)
#define G_CODE		0x1000
#define G_STACK		0x7000

static int fd;
static nvmm_machid_t machid;
static unsigned long g_hva = HVA_BASE;
static size_t g_ramsize = RAM_SIZE;

#define X(nv)		__CONCAT(NVMM_X64_, nv)

static void
die(const char *m)
{
	fprintf(stderr, "DIE: %s: %s\n", m, strerror(errno));
	exit(1);
}

static void
ioc(unsigned long cmd, void *arg, const char *what)
{
	int r = ioctl(fd, cmd, arg);
	if (r == -1) {
		unsigned char *p = arg;
		size_t n = ((cmd >> 16) & 0x1FF);
		if (n > 48) n = 48;
		printf("[XX] %s: cmd=%#lx sz=%zu errno=%d(%s) bytes:", what,
		    (unsigned long)cmd, n, errno, strerror(errno));
		for (size_t i = 0; i < n; i++)
			printf(" %02x", p[i]);
		printf("\n");
		fflush(stdout);
		die(what);
	}
}

static void
ioc_ok(unsigned long cmd, void *arg, const char *what, int expect_err, const char *ctx)
{
	if (ioctl(fd, cmd, arg) == -1) {
		int e = errno;
		if (expect_err && e == expect_err) {
			printf("[ok] %s: rejected with %s (as expected) [%s]\n",
			    what, strerror(e), ctx);
			return;
		}
		printf("[!!] %s: unexpected errno %d (%s) [%s]\n",
		    what, e, strerror(e), ctx);
		return;
	}
	printf("[ok] %s: succeeded [%s]\n", what, ctx);
}

static void
full_state_reset(struct nvmm_comm_page *comm)
{
	struct nvmm_x64_state *st = &comm->state;
	memset(st, 0, sizeof(*st));
	for (int i = 0; i < NVMM_X64_NSEG; i++) {
		st->segs[i].selector = 0;
		st->segs[i].base = 0;
		st->segs[i].limit = 0xFFFF;
		st->segs[i].attrib.type = 3;
		st->segs[i].attrib.s = (i == NVMM_X64_SEG_LDT ||
					i == NVMM_X64_SEG_TR) ? 0 : 1;
		st->segs[i].attrib.p = 1;
	}
	st->segs[NVMM_X64_SEG_GDT].attrib.type = 2;
	st->segs[NVMM_X64_SEG_IDT].attrib.type = 2;
	st->segs[NVMM_X64_SEG_LDT].attrib.type = 2;	/* LDT must be type 2 */
	st->segs[NVMM_X64_SEG_TR].attrib.type = 3;
	st->gprs[NVMM_X64_GPR_RFLAGS] = 0x2;
	st->gprs[NVMM_X64_GPR_RIP] = G_CODE;
	st->gprs[NVMM_X64_GPR_RSP] = G_STACK;
	st->crs[NVMM_X64_CR_CR0] = 0x10;	/* ET only: real mode */
	st->crs[NVMM_X64_CR_XCR0] = 0x1;
	st->drs[NVMM_X64_DR_DR6] = 0xFFFF0FF0;
	st->drs[NVMM_X64_DR_DR7] = 0x400;
}

static void
vcpu_setstate(nvmm_machid_t mach, nvmm_cpuid_t cpu,
    struct nvmm_comm_page *comm, uint64_t flags)
{
	struct nvmm_ioc_vcpu_setstate args;
	memset(&args, 0, sizeof(args));
	args.machid = mach;
	args.cpuid = cpu;
	comm->state_wanted = flags;
	ioc(NVMM_IOC_VCPU_SETSTATE, &args, "SETSTATE");
}

static void
vcpu_getstate(nvmm_machid_t mach, nvmm_cpuid_t cpu,
    struct nvmm_comm_page *comm, uint64_t flags)
{
	struct nvmm_ioc_vcpu_getstate args;
	memset(&args, 0, sizeof(args));
	args.machid = mach;
	args.cpuid = cpu;
	comm->state_wanted = flags;
	ioc(NVMM_IOC_VCPU_GETSTATE, &args, "GETSTATE");
}

static void
vcpu_inject(nvmm_machid_t mach, nvmm_cpuid_t cpu,
    struct nvmm_comm_page *comm, u_int type, uint8_t vector, uint64_t error)
{
	struct nvmm_ioc_vcpu_inject args;
	memset(&args, 0, sizeof(args));
	args.machid = mach;
	args.cpuid = cpu;
	comm->event.type = type;
	comm->event.vector = vector;
	comm->event.u.excp.error = error;
	ioc(NVMM_IOC_VCPU_INJECT, &args, "INJECT");
}

static const char *
exit_reason_name(uint64_t r)
{
	switch (r) {
	case NVMM_VCPU_EXIT_NONE: return "NONE";
	case NVMM_VCPU_EXIT_INVALID: return "INVALID";
	case NVMM_VCPU_EXIT_MEMORY: return "MEMORY(EPT)";
	case NVMM_VCPU_EXIT_IO: return "IO";
	case NVMM_VCPU_EXIT_RDMSR: return "RDMSR";
	case NVMM_VCPU_EXIT_WRMSR: return "WRMSR";
	case NVMM_VCPU_EXIT_CPUID: return "CPUID";
	case NVMM_VCPU_EXIT_HALTED: return "HALTED";
	case NVMM_VCPU_EXIT_SHUTDOWN: return "SHUTDOWN";
	case NVMM_VCPU_EXIT_INT_READY: return "INT_READY";
	case NVMM_VCPU_EXIT_NMI_READY: return "NMI_READY";
	case NVMM_VCPU_EXIT_TPR_CHANGED: return "TPR_CHANGED";
	default: return "?";
	}
}

/* run a vcpu and emulate enough to keep it going; returns last exit */
static uint64_t
run_loop(nvmm_machid_t mach, nvmm_cpuid_t cpu, struct nvmm_comm_page *comm,
    int max_exits, const char *ctx)
{
	struct nvmm_ioc_vcpu_run args;
	uint64_t last = ~0ULL;
	int n = 0;

	memset(&args, 0, sizeof(args));
	args.machid = mach;
	args.cpuid = cpu;

	while (n++ < max_exits) {
		if (ioctl(fd, NVMM_IOC_VCPU_RUN, &args) == -1) {
			printf("[--] RUN returned errno %d (%s) after %d exits "
			    "[%s]\n", errno, strerror(errno), n - 1, ctx);
			return ~0ULL;
		}
		last = args.exit.reason;
		switch (last) {
		case NVMM_VCPU_EXIT_NONE:
			continue;
		case NVMM_VCPU_EXIT_IO:
			printf("[--] IO exit: %s port 0x%x size %u str=%u "
			    "rep=%u seg=%u npc=0x%llx [%s]\n",
			    args.exit.u.io.in ? "in" : "out",
			    args.exit.u.io.port, args.exit.u.io.operand_size,
			    args.exit.u.io.str, args.exit.u.io.rep,
			    args.exit.u.io.seg,
			    (unsigned long long)args.exit.u.io.npc, ctx);
			vcpu_getstate(mach, cpu, comm, NVMM_X64_STATE_GPRS);
			comm->state.gprs[NVMM_X64_GPR_RIP] =
			    args.exit.u.io.npc;
			if (args.exit.u.io.in)
				comm->state.gprs[NVMM_X64_GPR_RAX] = 0;
			vcpu_setstate(mach, cpu, comm,
			    NVMM_X64_STATE_GPRS);
			continue;
		case NVMM_VCPU_EXIT_RDMSR:
			printf("[--] RDMSR exit: msr=0x%x npc=0x%llx [%s]\n",
			    args.exit.u.rdmsr.msr,
			    (unsigned long long)args.exit.u.rdmsr.npc, ctx);
			vcpu_getstate(mach, cpu, comm, NVMM_X64_STATE_GPRS);
			comm->state.gprs[NVMM_X64_GPR_RAX] = 0;
			comm->state.gprs[NVMM_X64_GPR_RDX] = 0;
			comm->state.gprs[NVMM_X64_GPR_RIP] =
			    args.exit.u.rdmsr.npc;
			vcpu_setstate(mach, cpu, comm, NVMM_X64_STATE_GPRS);
			continue;
		case NVMM_VCPU_EXIT_WRMSR:
			printf("[--] WRMSR exit: msr=0x%x val=0x%llx [%s]\n",
			    args.exit.u.wrmsr.msr,
			    (unsigned long long)args.exit.u.wrmsr.val, ctx);
			vcpu_getstate(mach, cpu, comm, NVMM_X64_STATE_GPRS);
			comm->state.gprs[NVMM_X64_GPR_RIP] =
			    args.exit.u.wrmsr.npc;
			vcpu_setstate(mach, cpu, comm, NVMM_X64_STATE_GPRS);
			continue;
		case NVMM_VCPU_EXIT_MEMORY:
			printf("[--] EPT fault: gpa=0x%llx prot=%d "
			    "(kernel faults it in) [%s]\n",
			    (unsigned long long)args.exit.u.mem.gpa,
			    args.exit.u.mem.prot, ctx);
			continue;	/* nvmm_do_vcpu_run retries */
		case NVMM_VCPU_EXIT_INVALID:
			printf("[--] INVALID exit: hwcode=%lld [%s]\n",
			    (long long)args.exit.u.inv.hwcode, ctx);
			return last;
		case NVMM_VCPU_EXIT_SHUTDOWN:
			printf("[--] SHUTDOWN exit [%s]\n", ctx);
			return last;
		case NVMM_VCPU_EXIT_HALTED:
			printf("[--] HALTED exit [%s]\n", ctx);
			return last;
		default:
			printf("[--] exit reason %llu (%s) [%s]\n",
			    (unsigned long long)last,
			    exit_reason_name(last), ctx);
			return last;
		}
	}
	return last;
}

struct spin_ctx {
	nvmm_cpuid_t cpu;
	struct nvmm_comm_page *comm;
	int sec;
};

static void *
spin_thread(void *v)
{
	struct spin_ctx *c = v;
	struct nvmm_ioc_vcpu_run args;
	time_t t0 = time(NULL);
	int it = 0;

	memset(&args, 0, sizeof(args));
	args.machid = machid;
	args.cpuid = c->cpu;

	while (time(NULL) - t0 < c->sec) {
		if (ioctl(fd, NVMM_IOC_VCPU_RUN, &args) == -1) {
			if (errno == EINTR)
				continue;
			fprintf(stderr, "spin vcpu %u: RUN errno %d\n",
			    c->cpu, errno);
			return NULL;
		}
		if (args.exit.reason != NVMM_VCPU_EXIT_NONE)
			it++;
	}
	printf("[--] spin vcpu %u done (%d exits)\n", c->cpu, it);
	return NULL;
}

static int g_hammer_secs;

/* hammer SETSTATE/GETSTATE/INJECT on vcpu0..N concurrently with RUN */
static void *
hammer_thread(void *v)
{
	struct spin_ctx *base = v;
	time_t t0 = time(NULL);
	uint64_t n = 0, einval = 0, ok = 0;
	unsigned rnd = (unsigned)(uintptr_t)v | 1;

	while (time(NULL) - t0 < g_hammer_secs) {
		struct spin_ctx *c = &base[(rnd >> 7) & 1];
		struct nvmm_ioc_vcpu_setstate ss;
		struct nvmm_ioc_vcpu_getstate gs;
		struct nvmm_ioc_vcpu_inject inj;
		memset(&ss, 0, sizeof(ss));
		memset(&gs, 0, sizeof(gs));
		memset(&inj, 0, sizeof(inj));
		rnd = rnd * 1103515245 + 12345;

		gs.machid = machid; gs.cpuid = c->cpu;
		c->comm->state_wanted = NVMM_X64_STATE_GPRS;
		if (ioctl(fd, NVMM_IOC_VCPU_GETSTATE, &gs) == 0)
			ok++;

		c->comm->state.gprs[NVMM_X64_GPR_RAX] = rnd;
		ss.machid = machid; ss.cpuid = c->cpu;
		c->comm->state_wanted = NVMM_X64_STATE_GPRS;
		if (ioctl(fd, NVMM_IOC_VCPU_SETSTATE, &ss) == 0)
			ok++;

		inj.machid = machid; inj.cpuid = c->cpu;
		c->comm->event.type = NVMM_VCPU_EVENT_EXCP;
		c->comm->event.vector = 13;
		c->comm->event.u.excp.error = rnd;
		if (ioctl(fd, NVMM_IOC_VCPU_INJECT, &inj) == 0)
			ok++;
		else if (errno == EINVAL)
			einval++;
		else
			printf("[!!] hammer: inject errno %d\n", errno);
		n += 3;
	}
	printf("[--] hammer(vcpu%u..): %llu ops, %llu ok, %llu EINVAL\n",
	    base[0].cpu, (unsigned long long)n, (unsigned long long)ok,
	    (unsigned long long)einval);
	return NULL;
}

/* ---------------------------------------------------------------- */
/* guest code blobs (16-bit real mode, CS base 0, RIP=0x1000)        */
/* ---------------------------------------------------------------- */

/* test A: 16-bit real-mode: cpuid variants, CR0(CD|NW) write, out,
 * rdmsr(0x1A0 handled), wrmsr(MSR_TSC handled), rdmsr(unknown)->exit,
 * hlt.  32-bit operands need the 0x66 prefix in 16-bit mode; CR8 needs
 * REX (64-bit only), so it is not tested from a 16-bit guest. */
static const uint8_t code_a[] = {
	0x66,0x31,0xC0,                   /* 0000: xor eax,eax          */
	0x0F,0xA2,                        /* 0003: cpuid (leaf 0)       */
	0x66,0xB8,0x17,0x00,0x00,0x00,    /* 0005: mov eax,0x17         */
	0x0F,0xA2,                        /* 000B: cpuid (>max basic)   */
	0x66,0xB8,0x01,0x00,0x00,0x40,    /* 000D: mov eax,0x40000001   */
	0x0F,0xA2,                        /* 0013: cpuid (hypervisor)   */
	0x66,0x0F,0x20,0xC0,              /* 0015: mov eax,cr0          */
	0x66,0x0D,0x00,0x00,0x00,0x60,    /* 0019: or eax,CD|NW         */
	0x66,0x0F,0x22,0xC0,              /* 001F: mov cr0,eax -> EXIT  */
	0xE6,0xF8,                        /* 0023: out 0xf8,al -> EXIT  */
	0x66,0xB9,0xA0,0x01,0x00,0x00,    /* 0025: mov ecx,0x1A0        */
	0x0F,0x32,                        /* 002B: rdmsr (in-kernel)    */
	0x66,0xB9,0x10,0x00,0x00,0x00,    /* 002D: mov ecx,0x10 (TSC)   */
	0x66,0xBA,0x00,0x00,0x00,0x00,    /* 0033: mov edx,0            */
	0x66,0xB8,0x11,0x11,0x00,0x00,    /* 0039: mov eax,0x1111       */
	0x0F,0x30,                        /* 003F: wrmsr (in-kernel)    */
	0x66,0xB9,0xEF,0xBE,0xAD,0xDE,    /* 0041: mov ecx,0xDEADBEEF   */
	0x0F,0x32,                        /* 0047: rdmsr -> EXIT (user) */
	0xF4,                             /* 0049: hlt                  */
};

/* test B: mov cr4 with VMXE (invalid bit) -> kernel GP injection;
 * no usable IDT -> shutdown expected */
static const uint8_t code_b[] = {
	0x66,0x0F,0x20,0xE0,               /* 0000: mov eax,cr4            */
	0x66,0x0D,0x00,0x20,0x00,0x00,     /* 0004: or eax,0x2000 (VMXE)   */
	0x66,0x0F,0x22,0xE0,               /* 000A: mov cr4,eax -> GP exit */
	0xF4,                              /* 000E: hlt                    */
};

/* test C: xsetbv with ecx=1 -> kernel GP injection */
static const uint8_t code_c[] = {
	0x66,0xB9,0x01,0x00,0x00,0x00,     /* 0000: mov ecx,1              */
	0x0F,0x01,0xD1,                    /* 0006: xsetbv -> GP exit      */
	0xF4,                              /* 0009: hlt                    */
};

/* test D: vmxon -> kernel #UD injection */
static const uint8_t code_d[] = {
	0x66,0x31,0xC0,                    /* 0000: xor eax,eax            */
	0x0F,0xC7,0x30,                    /* 0003: vmxon [eax] -> UD exit */
	0xF4,                              /* 0006: hlt                    */
};

/* spin: jmp $ */
static const uint8_t code_spin[] = { 0xEB, 0xFE };

/* ---------------------------------------------------------------- */

static struct nvmm_comm_page *
create_vcpu(nvmm_cpuid_t cpuid)
{
	struct nvmm_ioc_vcpu_create args;
	memset(&args, 0, sizeof(args));
	args.machid = machid;
	args.cpuid = cpuid;
	args.comm = (void *)0x30000000UL;
	{
		int r = ioctl(fd, NVMM_IOC_VCPU_CREATE, &args);
		if (r == -1)
			printf("[--] VCPU_CREATE(%u,%u,%p) -> errno %d (%s)\n",
			    args.machid, args.cpuid, args.comm, errno,
			    strerror(errno));
		if (r == -1 && errno == EINVAL && args.comm) {
			/* retry with NULL hint */
			args.comm = NULL;
			r = ioctl(fd, NVMM_IOC_VCPU_CREATE, &args);
			printf("[--] VCPU_CREATE retry(NULL hint) -> r=%d "
			    "errno=%d\n", r, errno);
		}
		if (r == -1)
			die("VCPU_CREATE");
		return args.comm;
	}
	ioc(NVMM_IOC_VCPU_CREATE, &args, "VCPU_CREATE");
	return args.comm;
}

static void
load_code(uint8_t *ram, const uint8_t *code, size_t len)
{
	memcpy(ram + G_CODE, code, len);
}

static uint64_t
one_test(const char *name, const uint8_t *code, size_t len, nvmm_cpuid_t cpu,
    struct nvmm_comm_page *comm, uint8_t *ram)
{
	full_state_reset(comm);
	load_code(ram, code, len);
	vcpu_setstate(machid, cpu, comm, NVMM_X64_STATE_ALL);
	return run_loop(machid, cpu, comm, 64, name);
}

int
main(int argc, char **argv)
{
	struct nvmm_ioc_capability cap;
	struct nvmm_ioc_machine_create mc;
	struct nvmm_ioc_hva_map hm;
	struct nvmm_ioc_gpa_map gm;
	uint8_t *ram;

	if (argc < 2) {
		fprintf(stderr, "usage: %s run|spin SEC [NVCPUS]\n", argv[0]);
		return 2;
	}

	fd = open("/dev/nvmm", O_RDONLY | O_CLOEXEC);
	if (fd == -1)
		die("open /dev/nvmm");
	memset(&cap, 0, sizeof(cap));
	ioc(NVMM_IOC_CAPABILITY, &cap, "CAPABILITY");
	printf("[--] nvmm v%u: state=%u comm=%u maxmach=%u maxvcpus=%u "
	    "maxram=%llu xcr0_mask=0x%llx conf_cpuid_maxops=%u\n",
	    cap.cap.version, cap.cap.state_size, cap.cap.comm_size,
	    cap.cap.max_machines, cap.cap.max_vcpus,
	    (unsigned long long)cap.cap.max_ram,
	    (unsigned long long)cap.cap.arch.xcr0_mask,
	    cap.cap.arch.conf_cpuid_maxops);

	memset(&mc, 0, sizeof(mc));
	if (ioctl(fd, NVMM_IOC_MACHINE_CREATE, &mc) == -1)
		die("MACHINE_CREATE");
	machid = mc.machid;
	printf("[--] machine %u created\n", machid);

	memset(&hm, 0, sizeof(hm));
	hm.machid = machid;
	hm.hva = HVA_BASE;
	hm.size = RAM_SIZE;
	hm.flags = 0;
	if (ioctl(fd, NVMM_IOC_HVA_MAP, &hm) == -1) {
		int e = errno;
		printf("[--] HVA_MAP 0x%lx/0x%lx failed: %d (%s), retrying "
		    "variants...\n", (unsigned long)HVA_BASE,
		    (unsigned long)RAM_SIZE, e, strerror(e));
		unsigned long bases[] = { 0x20000000UL, 0x30000000UL,
					  0x60000000UL };
		size_t sizes[] = { RAM_SIZE, 1024*1024, 4096 };
		int ok = 0;
		for (unsigned i = 0; i < 3 && !ok; i++)
		for (unsigned j = 0; j < 3 && !ok; j++) {
			memset(&hm, 0, sizeof(hm));
			hm.machid = machid;
			hm.hva = bases[i];
			hm.size = sizes[j];
			if (ioctl(fd, NVMM_IOC_HVA_MAP, &hm) == 0) {
				printf("[--] HVA_MAP retry ok: hva=0x%lx "
				    "size=0x%zx\n", bases[i], sizes[j]);
				ok = 1;
			}
		}
		if (!ok)
			die("HVA_MAP (all variants)");
		g_hva = hm.hva;
		g_ramsize = hm.size;
	}

	memset(&gm, 0, sizeof(gm));
	gm.machid = machid;
	gm.hva = g_hva;
	gm.gpa = 0;
	gm.size = g_ramsize;
	gm.prot = PROT_READ | PROT_WRITE | PROT_EXEC;
	if (ioctl(fd, NVMM_IOC_GPA_MAP, &gm) == -1) {
		struct nvmm_ioc_ctl ctl;
		struct nvmm_ctl_mach_info info;
		int e = errno;
		memset(&ctl, 0, sizeof(ctl));
		memset(&info, 0, sizeof(info));
		info.machid = machid;
		ctl.op = NVMM_CTL_MACH_INFO;
		ctl.data = &info;
		ctl.size = sizeof(info);
		printf("[--] GPA_MAP failed: %d (%s); gm bytes:", e,
		    strerror(e));
		unsigned char *p = (unsigned char *)&gm;
		for (size_t i = 0; i < sizeof(gm); i++)
			printf(" %02x", p[i]);
		printf("\n");
		if (ioctl(fd, NVMM_IOC_CTL, &ctl) == 0)
			printf("[--] mach %u: nvcpus=%u nram=%llu pid=%d\n",
			    info.machid, info.nvcpus,
			    (unsigned long long)info.nram, info.pid);
		die("GPA_MAP");
	}

	ram = (uint8_t *)g_hva;

	if (strcmp(argv[1], "spin") == 0 || strcmp(argv[1], "stress") == 0) {
		int stress = (strcmp(argv[1], "stress") == 0);
		int sec = (argc > 2) ? atoi(argv[2]) : 5;
		int nv = (argc > 3) ? atoi(argv[3]) : 6;
		struct spin_ctx ctxs[16];
		pthread_t th[16], hm[2];
		if (nv > 16) nv = 16;
		printf("[--] %s: %d vcpus x %d sec\n",
		    stress ? "stress" : "spin", nv, sec);
		for (int i = 0; i < nv; i++) {
			ctxs[i].cpu = i;
			ctxs[i].comm = create_vcpu(i);
			ctxs[i].sec = sec;
			full_state_reset(ctxs[i].comm);
			load_code(ram, code_spin, sizeof(code_spin));
			vcpu_setstate(machid, i, ctxs[i].comm,
			    NVMM_X64_STATE_ALL);
		}
		for (int i = 0; i < nv; i++)
			pthread_create(&th[i], NULL, spin_thread, &ctxs[i]);
		if (stress) {
			g_hammer_secs = sec;
			pthread_create(&hm[0], NULL, hammer_thread, &ctxs[0]);
			pthread_create(&hm[1], NULL, hammer_thread,
			    &ctxs[0]);
			pthread_join(hm[0], NULL);
			pthread_join(hm[1], NULL);
		}
		for (int i = 0; i < nv; i++)
			pthread_join(th[i], NULL);
		printf("[--] %s: ALL DONE, system alive\n",
		    stress ? "stress" : "spin");
		return 0;
	}

	/* ---- sequential sub-tests, fresh vCPU each (fresh VMCS/vmlaunch;
	 * under nested KVM a vmresume after a first exit failValid's) ---- */
	struct nvmm_comm_page *comms[8];
	for (int i = 0; i < 8; i++)
		comms[i] = create_vcpu(i);
	struct nvmm_comm_page *comm = comms[0];
	printf("[--] comm=%p ram=%p machid=%u\n", comm, ram, machid);
	fflush(stdout);
	uint64_t r;

	printf("\n=== TEST A: cpuid/CR0/CR8/IO/MSR happy paths ===\n");
	r = one_test("A", code_a, sizeof(code_a), 0, comms[0], ram);
	printf("[%c] TEST A end reason=%s (want HALTED)\n",
	    r == NVMM_VCPU_EXIT_HALTED ? '+' : '-', exit_reason_name(r));

	printf("\n=== TEST B: guest CR4 write w/ reserved VMXE bit ===\n");
	r = one_test("B", code_b, sizeof(code_b), 1, comms[1], ram);
	printf("[%c] TEST B end reason=%s (want SHUTDOWN via injected #GP; "
	    "PASS = no panic)\n", (r == NVMM_VCPU_EXIT_SHUTDOWN ||
	    r == NVMM_VCPU_EXIT_HALTED) ? '+' : '-', exit_reason_name(r));

	printf("\n=== TEST C: guest XSETBV with ECX=1 ===\n");
	r = one_test("C", code_c, sizeof(code_c), 2, comms[2], ram);
	printf("[%c] TEST C end reason=%s (PASS = no panic)\n",
	    r != ~0ULL ? '+' : '-', exit_reason_name(r));

	printf("\n=== TEST D: guest VMXON -> kernel #UD injection ===\n");
	r = one_test("D", code_d, sizeof(code_d), 3, comms[3], ram);
	printf("[%c] TEST D end reason=%s (PASS = no panic)\n",
	    r != ~0ULL ? '+' : '-', exit_reason_name(r));

	printf("\n=== TEST E: hostile setstate (PAT/EFER/CR garbage) ===\n");
	comm = comms[4];
	full_state_reset(comm);
	load_code(ram, code_a, sizeof(code_a));
	comm->state.msrs[NVMM_X64_MSR_PAT] = 0xFFFFFFFFFFFFFFFFULL;
	comm->state.msrs[NVMM_X64_MSR_EFER] = 0xD;	/* LMA|LME w/o NX */
	comm->state.crs[NVMM_X64_CR_CR3] = 0xDEAD000;
	comm->state.drs[NVMM_X64_DR_DR7] = 0xFFFFFFFFFFFFFFFFULL;
	vcpu_setstate(machid, 0, comm, NVMM_X64_STATE_ALL);
	r = run_loop(machid, 4, comm, 8, "E");
	printf("[%c] TEST E end reason=%s (PASS = clean VMentry-failure or "
	    "progress, no panic)\n", r != ~0ULL ? '+' : '-',
	    exit_reason_name(r));

	printf("\n=== TEST F: hostile FPU/MXCSR state ===\n");
	comm = comms[5];
	full_state_reset(comm);
	load_code(ram, code_a, sizeof(code_a));
	memset(&comm->state.fpu, 0xFF, sizeof(comm->state.fpu));
	vcpu_setstate(machid, 5, comm, NVMM_X64_STATE_ALL);
	r = run_loop(machid, 5, comm, 8, "F");
	printf("[%c] TEST F end reason=%s (PASS = no panic)\n",
	    r != ~0ULL ? '+' : '-', exit_reason_name(r));

	printf("\n=== TEST G: hostile event injection ===\n");
	comm = comms[6];
	full_state_reset(comm);
	load_code(ram, code_a, sizeof(code_a));
	vcpu_setstate(machid, 0, comm, NVMM_X64_STATE_ALL);
	{
		struct nvmm_ioc_vcpu_inject inj;
		memset(&inj, 0, sizeof(inj));
		inj.machid = machid;
		inj.cpuid = 6;
		/* G1: EXCP vector 0xFF -> must be rejected EINVAL */
		comm->event.type = NVMM_VCPU_EVENT_EXCP;
		comm->event.vector = 0xFF;
		comm->event.u.excp.error = 0;
		ioc_ok(NVMM_IOC_VCPU_INJECT, &inj, "INJECT EXCP vec=0xFF",
		    EINVAL, "G1");
		/* G2: EXCP vector 13, error with high 32 bits set */
		vcpu_inject(machid, 6, comm, NVMM_VCPU_EVENT_EXCP, 13,
		    0xFFFFFFFF00000000ULL);
		printf("[ok] INJECT EXCP vec=13 err=0xFFFFFFFF00000000 "
		    "accepted (G2)\n");
		r = run_loop(machid, 6, comm, 8, "G2");
		printf("[%c] G2 run end reason=%s (PASS = no panic)\n",
		    r != ~0ULL ? '+' : '-', exit_reason_name(r));
		/* G3: INTR vector 0xFF */
		full_state_reset(comm);
		load_code(ram, code_a, sizeof(code_a));
		vcpu_setstate(machid, 0, comm, NVMM_X64_STATE_ALL);
		vcpu_inject(machid, 6, comm, NVMM_VCPU_EVENT_INTR, 0xFF, 0);
		printf("[ok] INJECT INTR vec=0xFF accepted (G3)\n");
		r = run_loop(machid, 6, comm, 8, "G3");
		printf("[%c] G3 run end reason=%s (PASS = no panic)\n",
		    r != ~0ULL ? '+' : '-', exit_reason_name(r));
	}

	printf("\n=== TEST H: cpuid config (user-driven masks) ===\n");
	{
		struct nvmm_ioc_vcpu_configure vc;
		struct nvmm_vcpu_conf_cpuid conf;
		memset(&vc, 0, sizeof(vc));
		vc.machid = machid;
		vc.cpuid = 7;
		vc.op = NVMM_VCPU_CONF_CPUID;
		vc.conf = &conf;
		memset(&conf, 0, sizeof(conf));
		conf.leaf = 0x40000000;
		conf.exit = 1;
		ioc_ok(NVMM_IOC_VCPU_CONFIGURE, &vc,
		    "CONFIGURE cpuid exit leaf 0x40000000", 0, "H1");
		comm = comms[7];
		full_state_reset(comm);
		load_code(ram, code_a, sizeof(code_a));
		vcpu_setstate(machid, 7, comm, NVMM_X64_STATE_ALL);
		r = run_loop(machid, 7, comm, 64, "H");
		printf("[%c] TEST H end reason=%s (want CPUID exit delivered)\n",
		    r == NVMM_VCPU_EXIT_CPUID ? '+' : '-',
		    exit_reason_name(r));
	}

	printf("\n=== ALL SEQUENTIAL TESTS COMPLETE, system alive ===\n");
	return 0;
}