/*
 * dfbt_rig.c -- production-handler dispatch rig for the DragonFly bt(4)
 * (BusLogic MultiMaster) driver, used to verify findings DF-2429,
 * DF-4097, DF-4098, DF-4099, DF-4100 and DF-4101.
 *
 * The bt driver is compiled into X86_64_GENERIC (nm shows btaction,
 * btdone, btreset, bttimeout, bt_intr_locked), but this QEMU guest has
 * no BusLogic PCI HBA, so bt_attach() never runs.  This rig stands in
 * ONLY for the missing PCI hardware:
 *
 *   - resolves the REAL static production handlers from the running
 *     kernel image (no KASLR: file vaddr == runtime vaddr); the host
 *     run script byte-verifies their prologues via sysctl before use;
 *   - registers a REAL cam_sim whose sim_action/sim_poll ARE the
 *     production btaction/btpoll, over a fabricated post-attach
 *     bt_softc (real lockmgr lock, real bus_dma tags/maps, real DMA
 *     memory for the CCB array and mailboxes);
 *   - drives the REAL CAM XPT layer via xpt_action() with real
 *     xpt_alloc_ccb'd CCBs on a real registered bus;
 *   - plays the role of the host adapter firmware: writes completion
 *     status into the CCB DMA memory and delivers mailbox completions
 *     to the production btdone()/bt_intr_locked() exactly as
 *     bt.c:1363-1370 does, and scripts a virtual 4-byte register file
 *     (bt_inb/bt_outb are plain volatile memory accesses when the
 *     resource tag is X86_64_BUS_SPACE_MEM, see
 *     sys/cpu/x86_64/include/bus_dma.h:134-142).
 *
 * Everything from dev_ops dispatch through xpt_action, the XPT device
 * queues, btaction, btexecuteccb, btdone, bttimeout and btreset is
 * production code executing from the shipped kernel image.
 *
 * Root-only device /dev/dfbt0.  Unload-safe on the clean-run boot.
 */
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/conf.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/sysctl.h>
#include <sys/proc.h>
#include <sys/thread.h>
#include <sys/kthread.h>
#include <sys/callout.h>
#include <sys/bus.h>
#include <sys/device.h>
#include <sys/rman.h>
#include <sys/types.h>
#include <sys/ioccom.h>

#include <vm/vm.h>
#include <vm/pmap.h>

#include <bus/cam/cam.h>
#include <bus/cam/cam_ccb.h>
#include <bus/cam/cam_sim.h>
#include <bus/cam/cam_xpt.h>
#include <bus/cam/cam_xpt_sim.h>
#include <bus/cam/cam_periph.h>
#include <bus/cam/cam_xpt_periph.h>

#include <dev/disk/buslogic/btreg.h>

#include <sys/bus_private.h>	/* struct device (nameunit) */

/* ------------------------------------------------------------------ */
/* Production handler addresses, primed from userland (no KASLR).     */
/* ------------------------------------------------------------------ */
static unsigned long a_btaction, a_btpoll, a_btdone, a_bttimeout,
		     a_btreset, a_bt_intr_locked;

SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btaction, CTLFLAG_RW, &a_btaction, 0,
    "runtime addr of production btaction");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btpoll, CTLFLAG_RW, &a_btpoll, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btdone, CTLFLAG_RW, &a_btdone, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_bttimeout, CTLFLAG_RW, &a_bttimeout, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btreset, CTLFLAG_RW, &a_btreset, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btintr, CTLFLAG_RW, &a_bt_intr_locked, 0, "");

/* first 8 bytes at each production address (host script verifies) */
static uint64_t m_btaction, m_btpoll, m_btdone, m_bttimeout, m_btreset,
		m_btintr;
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btaction, CTLFLAG_RD, &m_btaction, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btpoll, CTLFLAG_RD, &m_btpoll, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btdone, CTLFLAG_RD, &m_btdone, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_bttimeout, CTLFLAG_RD, &m_bttimeout, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btreset, CTLFLAG_RD, &m_btreset, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btintr, CTLFLAG_RD, &m_btintr, 0, "");

static int vhba_enable = 1;	/* virtual HBA reset state machine */
SYSCTL_INT(_kern, OID_AUTO, dfbt_vhba_enable, CTLFLAG_RW, &vhba_enable, 0,
    "virtual HBA scripts DIAG dance on hard/soft reset writes");

/* virtual HBA observation of production register writes */
static u_int vhba_hard_resets, vhba_soft_resets, vhba_mbox_starts;
static u_int vhba_last_ctl_write;
SYSCTL_UINT(_kern, OID_AUTO, dfbt_vhba_hard_resets, CTLFLAG_RD,
    &vhba_hard_resets, 0, "HARD_RESET(0x80) writes the driver made");
SYSCTL_UINT(_kern, OID_AUTO, dfbt_vhba_soft_resets, CTLFLAG_RD,
    &vhba_soft_resets, 0, "SOFT_RESET(0x40) writes the driver made");
SYSCTL_UINT(_kern, OID_AUTO, dfbt_vhba_mbox_starts, CTLFLAG_RD,
    &vhba_mbox_starts, 0, "BOP_START_MBOX writes the driver made");
SYSCTL_UINT(_kern, OID_AUTO, dfbt_vhba_last_ctl, CTLFLAG_RD,
    &vhba_last_ctl_write, 0, "last CONTROL_REG value the driver wrote");

typedef void (*btaction_fn)(struct cam_sim *, union ccb *);
typedef void (*btdone_fn)(struct bt_softc *, struct bt_ccb *, bt_mbi_comp_code_t);
typedef void (*bttimeout_fn)(void *);
typedef int  (*btreset_fn)(struct bt_softc *, int);
typedef void (*btintr_fn)(struct bt_softc *);

/* ------------------------------------------------------------------ */
/* Fabricated post-attach state                                        */
/* ------------------------------------------------------------------ */
#define RIG_MAX_CCBS	16
#define RIG_UNIT	9

static struct bt_softc *bt;
static struct bsd_device rig_dev;	/* fabricated device_t */
static struct resource vport;		/* fabricated port resource (MEM tag) */
static uint8_t *regpage;		/* virtual HBA register file */
static struct cam_sim *rig_sim;
static struct cam_path *io_path;	/* target 3 lun 0 on our bus */
static struct cam_periph rig_periph;	/* fabricated periph: type only */
static cdev_t rig_cdev;
static struct thread *vhba_td;

/* async event bookkeeping */
static u_int async_bdr_count, async_busreset_count;

/* CCB registry */
struct rig_cctx {
	union ccb	*ccb;
	struct bt_ccb	*bccb;
	uint32_t	 bccb_phys;
	u_int		 comp_count;
	uint32_t	 last_status;
};
static struct rig_cctx cctx[32];
static int ncctx;

static d_open_t		rig_open;
static d_close_t	rig_close;
static d_ioctl_t	rig_ioctl;

static struct dev_ops rig_ops = {
	{ "dfbt", 0, D_MPSAFE, NULL, 0, 0 },
	.d_open =	rig_open,
	.d_close =	rig_close,
	.d_ioctl =	rig_ioctl,
};

/* ioctls */
struct rig_queue_args {
	uint32_t	func_code;	/* XPT_SCSI_IO | XPT_RESET_DEV */
	uint32_t	timeout_ms;
	uint32_t	dxfer_len;	/* 0 => CAM_DIR_NONE */
	uint32_t	dir_in;		/* 1 => CAM_DIR_IN */
};
struct rig_calcgeom_args {
	uint32_t	block_size;
	uint32_t	volume_size;
	uint32_t	heads, secs_per_track;
	uint32_t	cylinders;
	uint32_t	status;
};
struct rig_comp_args {
	uint32_t	use_recovery;	/* target the recovery BCCB */
	uint32_t	ccb_id;
	uint8_t		comp_code;
	uint8_t		btstat;
	uint8_t		sdstat;
	uint8_t		_pad;
};
struct rig_state {
	int32_t		 err;
	uint32_t	 ccb_id;
	uint64_t	 bccb_kva;
	uint64_t	 recovery_kva;
	uint32_t	 bccb_flags;
	uint32_t	 recovery_flags;
	uint32_t	 hccb_opcode, hccb_cmd_len, hccb_sense_len;
	uint32_t	 hccb_datain, hccb_dataout;
	uint32_t	 hccb_btstat, hccb_sdstat;
	uint32_t	 ccb_status;
	uint32_t	 timer_pending;
	uint32_t	 dmamap_nsegs;
	uint32_t	 on_pending_list;
	uint32_t	 comp_count;
	uint32_t	 last_status;
	uint32_t	 async_bdr, async_busreset;
	uint32_t	 vregs[4];
	uint32_t	 outbox_ccbaddr, outbox_action;
	uint32_t	 active_ccbs, free_list_walk, pending_walk;
};
struct rig_vreg_args { uint32_t off, val; };

#define RIGIOC_QUEUE	_IOWR('B', 1, struct rig_queue_args)
#define RIGIOC_SNAP	_IOWR('B', 2, struct rig_state)
#define RIGIOC_COMP	_IOWR('B', 3, struct rig_comp_args)
#define RIGIOC_TIMEOUT	_IOWR('B', 4, struct rig_state) /* + ccb_id in st */
#define RIGIOC_CALCGEOM	_IOWR('B', 5, struct rig_calcgeom_args)
#define RIGIOC_BTRESET	_IOWR('B', 6, struct rig_state)
#define RIGIOC_VREG	_IOWR('B', 7, struct rig_vreg_args)
#define RIGIOC_HOLD	_IOWR('B', 8, struct rig_state)
#define RIGIOC_REL	_IOWR('B', 9, struct rig_state)
#define RIGIOC_RACE_FIRE _IOWR('B', 10, struct rig_state)
#define RIGIOC_SIMQ_REL	_IOWR('B', 11, struct rig_state)

static void	rig_done(struct cam_periph *periph, union ccb *done_ccb);
static void	rig_async(void *softc, u_int32_t code,
			    struct cam_path *path, void *args);

static __inline uint32_t
rig_ccbvtop(struct bt_ccb *bccb)
{
	return (bt->bt_ccb_physbase +
	    (uint32_t)((caddr_t)bccb - (caddr_t)bt->bt_ccb_array));
}

static __inline struct bt_ccb *
rig_ccbptov(uint32_t ccb_addr)
{
	return (bt->bt_ccb_array +
	    ((struct bt_ccb *)(uintptr_t)ccb_addr -
	     (struct bt_ccb *)(uintptr_t)bt->bt_ccb_physbase));
}

static void
rig_capture(struct rig_state *st, struct rig_cctx *cx)
{
	struct ccb_hdr *ch;
	struct bt_ccb *b;
	int i;

	bzero(st, sizeof(*st));
	st->err = 0;
	st->async_bdr = async_bdr_count;
	st->async_busreset = async_busreset_count;
	for (i = 0; i < 4; i++)
		st->vregs[i] = regpage[i];
	st->outbox_ccbaddr = bt->cur_outbox->ccb_addr;
	st->outbox_action = bt->cur_outbox->action_code;
	st->recovery_kva = (uint64_t)(uintptr_t)bt->recovery_bccb;
	st->recovery_flags = bt->recovery_bccb ? bt->recovery_bccb->flags : ~0u;
	st->active_ccbs = bt->active_ccbs;
	st->free_list_walk = 0;
	SLIST_FOREACH(b, &bt->free_bt_ccbs, links)
		st->free_list_walk++;
	st->pending_walk = 0;
	LIST_FOREACH(ch, &bt->pending_ccbs, sim_links.le)
		st->pending_walk++;

	if (cx == NULL || cx->bccb == NULL)
		return;
	st->ccb_id = cx - cctx;
	st->bccb_kva = (uint64_t)(uintptr_t)cx->bccb;
	st->bccb_flags = cx->bccb->flags;
	st->hccb_opcode = cx->bccb->hccb.opcode;
	st->hccb_cmd_len = cx->bccb->hccb.cmd_len;
	st->hccb_sense_len = cx->bccb->hccb.sense_len;
	st->hccb_datain = cx->bccb->hccb.datain;
	st->hccb_dataout = cx->bccb->hccb.dataout;
	st->hccb_btstat = cx->bccb->hccb.btstat;
	st->hccb_sdstat = cx->bccb->hccb.sdstat;
	st->ccb_status = cx->ccb ? cx->ccb->ccb_h.status : ~0u;
	st->timer_pending = callout_pending(&cx->bccb->timer);
	st->dmamap_nsegs = cx->bccb->dmamap ? 1 : 0;	/* map exists */
	st->on_pending_list = 0;
	LIST_FOREACH(ch, &bt->pending_ccbs, sim_links.le) {
		if ((struct bt_ccb *)(uintptr_t)ch->spriv_ptr0 == cx->bccb) {
			st->on_pending_list = 1;
			break;
		}
	}
	st->comp_count = cx->comp_count;
	st->last_status = cx->last_status;
}

static void
rig_done(struct cam_periph *periph, union ccb *done_ccb)
{
	int i;

	for (i = 0; i < ncctx; i++) {
		if (cctx[i].ccb == done_ccb) {
			cctx[i].comp_count++;
			cctx[i].last_status = done_ccb->ccb_h.status;
			return;
		}
	}
	kprintf("[rig] rig_done: unknown ccb %p\n", done_ccb);
}

static void
rig_async(void *softc, u_int32_t code, struct cam_path *path, void *args)
{
	if (code == AC_SENT_BDR)
		async_bdr_count++;
	if (code == AC_BUS_RESET)
		async_busreset_count++;
	kprintf("[rig] async event 0x%x fired (bdr=%u busrst=%u)\n",
	    code, async_bdr_count, async_busreset_count);
}

/* ------------------------------------------------------------------ */
/* busdma map callbacks (mirror btmapmboxes/btmapccbs)                 */
/* ------------------------------------------------------------------ */
static void
rig_mapmboxes(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
	bt->mailbox_physbase = segs->ds_addr;
}
static void
rig_mapccbs(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
	bt->bt_ccb_physbase = segs->ds_addr;
}

/* ------------------------------------------------------------------ */
/* Virtual HBA state machine: on a HARD/SOFT reset write to the        */
/* CONTROL_REG, assert DIAG_ACTIVE briefly, then come ready -- so the  */
/* production btreset() succeeds against scripted registers.          */
/* Disable via kern.dfbt.vhba_enable=0 to keep DIAG_ACTIVE stuck       */
/* (DF-4098 btreset panic variant).                                    */
/* ------------------------------------------------------------------ */
static void
rig_vhba_thread(void *arg)
{
	uint8_t prev = 0;
	u_int prev_cmd = 0;

	while (vhba_td != NULL) {
		uint8_t ctl = regpage[CONTROL_REG];
		uint8_t cmd = regpage[COMMAND_REG];

		if (cmd != prev_cmd) {
			if (cmd == BOP_START_MBOX)
				vhba_mbox_starts++;
			prev_cmd = cmd;
		}
		if (vhba_enable && prev == 0 &&
		    (ctl & (HARD_RESET | SOFT_RESET)) != 0) {
			vhba_last_ctl_write = ctl;
			if (ctl & HARD_RESET)
				vhba_hard_resets++;
			if (ctl & SOFT_RESET)
				vhba_soft_resets++;
			/* reset request seen: run the "diagnostics" */
			regpage[STATUS_REG] = DIAG_ACTIVE;
			tsleep(&regpage, 0, "vhba", hz / 8);
			regpage[STATUS_REG] = HA_READY | INIT_REQUIRED;
			regpage[CONTROL_REG] = 0;
		}
		prev = ctl & (HARD_RESET | SOFT_RESET);
		tsleep(&vhba_td, 0, "vhbaidle", hz / 20);
	}
	kthread_exit();
}

/* ------------------------------------------------------------------ */
/* Setup: fabricate the post-attach softc and register the real SIM    */
/* ------------------------------------------------------------------ */
static int
rig_setup(void)
{
	struct cam_devq *devq;
	u_int boxes;
	int error;

	if (bt != NULL)
		return (0);

	regpage = kmalloc(PAGE_SIZE, M_DEVBUF, M_INTWAIT | M_ZERO);
	bzero(&vport, sizeof(vport));
	vport.r_bustag = 1;	/* X86_64_BUS_SPACE_MEM */
	vport.r_bushandle = (bus_space_handle_t)(uintptr_t)regpage;
	regpage[STATUS_REG] = HA_READY;

	bzero(&rig_dev, sizeof(rig_dev));
	rig_dev.nameunit = "bt9";

	bt = kmalloc(sizeof(*bt), M_DEVBUF, M_INTWAIT | M_ZERO);
	SLIST_INIT(&bt->free_bt_ccbs);
	LIST_INIT(&bt->pending_ccbs);
	SLIST_INIT(&bt->sg_maps);
	bt->dev = &rig_dev;
	bt->port = &vport;
	lockinit(&bt->lock, "bt", 0, LK_CANRECURSE);	/* mirrors bt_init_softc */
	bt->max_ccbs = RIG_MAX_CCBS;
	boxes = RIG_MAX_CCBS;
	bt->num_boxes = boxes;
	bt->scsi_id = 7;
	ksnprintf(bt->firmware_ver, sizeof(bt->firmware_ver), "5.07");
	ksnprintf(bt->model, sizeof(bt->model), "958");
	bt->extended_trans = 1;
	bt->strict_rr = 1;
	bt->init_level = 4;	/* mailboxes not re-inited by btreset */

	/* real busdma tags/maps exactly as bt_init() builds them */
	error = bus_dma_tag_create(/*parent*/ NULL, /*alignment*/ 1,
	    /*boundary*/ 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
	    MAXBSIZE, BT_NSEG, BUS_SPACE_MAXSIZE_32BIT, BUS_DMA_ALLOCNOW,
	    &bt->buffer_dmat);
	if (error) { kprintf("[rig] buffer_dmat tag fail %d\n", error); return (error); }

	error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT,
	    BUS_SPACE_MAXADDR, boxes * (sizeof(bt_mbox_in_t) + sizeof(bt_mbox_out_t)),
	    1, BUS_SPACE_MAXSIZE_32BIT, 0, &bt->mailbox_dmat);
	if (error) { kprintf("[rig] mailbox_dmat tag fail\n"); return (error); }

	error = bus_dmamem_alloc(bt->mailbox_dmat, (void **)&bt->out_boxes,
	    BUS_DMA_NOWAIT, &bt->mailbox_dmamap);
	if (error) { kprintf("[rig] mailbox alloc fail\n"); return (error); }
	bus_dmamap_load(bt->mailbox_dmat, bt->mailbox_dmamap, bt->out_boxes,
	    boxes * (sizeof(bt_mbox_in_t) + sizeof(bt_mbox_out_t)),
	    rig_mapmboxes, NULL, 0);
	bt->in_boxes = (bt_mbox_in_t *)&bt->out_boxes[boxes];

	error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT,
	    BUS_SPACE_MAXADDR, RIG_MAX_CCBS * sizeof(struct bt_ccb),
	    1, BUS_SPACE_MAXSIZE_32BIT, 0, &bt->ccb_dmat);
	if (error) { kprintf("[rig] ccb_dmat tag fail\n"); return (error); }

	error = bus_dmamem_alloc(bt->ccb_dmat, (void **)&bt->bt_ccb_array,
	    BUS_DMA_NOWAIT, &bt->ccb_dmamap);
	if (error) { kprintf("[rig] ccb array alloc fail\n"); return (error); }
	bus_dmamap_load(bt->ccb_dmat, bt->ccb_dmamap, bt->bt_ccb_array,
	    RIG_MAX_CCBS * sizeof(struct bt_ccb), rig_mapccbs, NULL, 0);
	bzero(bt->bt_ccb_array, RIG_MAX_CCBS * sizeof(struct bt_ccb));

	error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT,
	    BUS_SPACE_MAXADDR, PAGE_SIZE, 1, BUS_SPACE_MAXSIZE_32BIT, 0,
	    &bt->sg_dmat);
	if (error) { kprintf("[rig] sg_dmat tag fail\n"); return (error); }

	/* mailbox pointers exactly as btinitmboxes() leaves them */
	bzero(bt->in_boxes, sizeof(bt_mbox_in_t) * boxes);
	bzero(bt->out_boxes, sizeof(bt_mbox_out_t) * boxes);
	bt->cur_inbox = bt->in_boxes;
	bt->last_inbox = bt->in_boxes + boxes - 1;
	bt->cur_outbox = bt->out_boxes;
	bt->last_outbox = bt->out_boxes + boxes - 1;

	/* register the REAL sim: production btaction/btpoll over bt */
	devq = cam_simq_alloc(RIG_MAX_CCBS);
	if (devq == NULL)
		return (ENOMEM);
	rig_sim = cam_sim_alloc((sim_action_func)a_btaction,
	    (sim_poll_func)a_btpoll, "bt", bt, RIG_UNIT, &bt->lock,
	    RIG_MAX_CCBS, RIG_MAX_CCBS, devq);
	cam_simq_release(devq);
	if (rig_sim == NULL)
		return (ENOMEM);
	/* bt_attach holds bt->lock across registration (bt.c:769-784) */
	lockmgr(&bt->lock, LK_EXCLUSIVE);
	bt->sim = rig_sim;	/* bt_attach:922 keeps its sim in the softc */
	if (xpt_bus_register(rig_sim, 0) != CAM_SUCCESS) {
		lockmgr(&bt->lock, LK_RELEASE);
		cam_sim_free(rig_sim);
		rig_sim = NULL;
		return (ENXIO);
	}
	if (xpt_create_path(&bt->path, /*periph*/ NULL, cam_sim_path(rig_sim),
	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
		lockmgr(&bt->lock, LK_RELEASE);
		return (ENXIO);
	}

	if (xpt_create_path(&io_path, /*periph*/ NULL, cam_sim_path(rig_sim),
	    3, 0) != CAM_REQ_CMP) {
		lockmgr(&bt->lock, LK_RELEASE);
		return (ENXIO);
	}
	/* fabricated periph: xpt_done reads path->periph->type only
	 * (cam_xpt.c:4903), must be CAM_PERIPH_BIO.  struct cam_path is
	 * private to cam_xpt.c; periph is its first field (cam_xpt.c:202),
	 * so write it via offset 0. */
	bzero(&rig_periph, sizeof(rig_periph));
	rig_periph.type = CAM_PERIPH_BIO;
	rig_periph.path = io_path;
	rig_periph.periph_name = "dfbt";
	*(struct cam_periph **)(void *)io_path = &rig_periph;

	xpt_register_async(AC_SENT_BDR | AC_BUS_RESET, rig_async, NULL, io_path);
	lockmgr(&bt->lock, LK_RELEASE);

	error = kthread_create(rig_vhba_thread, NULL, &vhba_td,
	    "dfbt_vhba");
	if (error)
		vhba_td = NULL;

	/* prime magic prologue words for host-side verification */
	m_btaction = *(volatile uint64_t *)a_btaction;
	m_btpoll   = *(volatile uint64_t *)a_btpoll;
	m_btdone   = *(volatile uint64_t *)a_btdone;
	m_bttimeout= *(volatile uint64_t *)a_bttimeout;
	m_btreset  = *(volatile uint64_t *)a_btreset;
	m_btintr   = *(volatile uint64_t *)a_bt_intr_locked;

	kprintf("[rig] dfbt rig up: softc=%p sim=%p path_id=%d "
	    "ccb_physbase=0x%08x mbox_physbase=0x%08x\n",
	    bt, rig_sim, cam_sim_path(rig_sim), bt->bt_ccb_physbase,
	    (unsigned)bt->mailbox_physbase);
	return (0);
}

/* ------------------------------------------------------------------ */
/* stale-handler wrapper for DF-4101: production bttimeout on a thread */
/* ------------------------------------------------------------------ */
static struct bt_ccb *race_bccb;
static volatile int race_entered, race_done;

static void
rig_race_thread(void *arg)
{
	kprintf("[rig] stale handler thread: calling production bttimeout("
	    "%p), will block on bt->lock\n", race_bccb);
	race_entered = 1;
	((bttimeout_fn)a_bttimeout)(race_bccb);	/* blocks on bt->lock */
	race_done = 1;
	kprintf("[rig] stale handler thread: bttimeout returned\n");
	kthread_exit();
}

/* ------------------------------------------------------------------ */
/* ioctl dispatch                                                      */
/* ------------------------------------------------------------------ */
static int
rig_open(struct dev_open_args *ap)
{
	return (0);
}

static int
rig_close(struct dev_close_args *ap)
{
	return (0);
}

static int
rig_ioctl(struct dev_ioctl_args *ap)
{
	u_long cmd = ap->a_cmd;
	caddr_t data = ap->a_data;
	struct rig_state *st;
	int error = 0;

	if (bt == NULL) {
		if (a_btaction == 0 || a_btdone == 0 || a_bttimeout == 0 ||
		    a_btreset == 0 || a_bt_intr_locked == 0 || a_btpoll == 0) {
			kprintf("[rig] addresses not primed\n");
			return (ENXIO);
		}
		error = rig_setup();
		if (error)
			return (error);
	}

	switch (cmd) {
	case RIGIOC_QUEUE:
	{
		struct rig_queue_args *qa = (struct rig_queue_args *)data;
		union ccb *ccb;
		struct rig_cctx *cx;

		if (ncctx >= 30)
			return (ENOMEM);
		ccb = xpt_alloc_ccb();
		xpt_setup_ccb(&ccb->ccb_h, io_path, /*priority*/ 1);
		ccb->ccb_h.func_code = qa->func_code;
		ccb->ccb_h.cbfcnp = rig_done;
		ccb->ccb_h.timeout = qa->timeout_ms;
		if (qa->func_code == XPT_SCSI_IO) {
			ccb->csio.cdb_len = 6;
			ccb->csio.cdb_io.cdb_bytes[0] = 0x00;	/* TUR */
			ccb->csio.sense_len = 0;
			if (qa->dir_in && qa->dxfer_len) {
				ccb->ccb_h.flags = CAM_DIR_IN;
				ccb->csio.dxfer_len = qa->dxfer_len;
				ccb->csio.data_ptr = kmalloc(qa->dxfer_len,
				    M_DEVBUF, M_INTWAIT | M_ZERO);
			} else {
				ccb->ccb_h.flags = CAM_DIR_NONE;
				ccb->csio.dxfer_len = 0;
				ccb->csio.data_ptr = NULL;
			}
		} else {
			ccb->ccb_h.flags = CAM_DIR_NONE;
		}
		cx = &cctx[ncctx];
		cx->ccb = ccb;
		cx->bccb = NULL;
		cx->bccb_phys = 0;
		cx->comp_count = 0;
		cx->last_status = 0;
		ncctx++;

		kprintf("[rig] QUEUE: xpt_action(func=%x) id=%d\n",
		    qa->func_code, ncctx - 1);
		lockmgr(&bt->lock, LK_EXCLUSIVE);
		xpt_action(ccb);	/* production: XPT -> btaction */
		kprintf("[rig] QUEUE post: pinfo.index=%d devq.entries=%u "
		    "send_openings=%u send_active=%u sim_openings=%d\n",
		    ccb->ccb_h.pinfo.index,
		    rig_sim->devq->send_queue.entries,
		    rig_sim->devq->send_openings,
		    rig_sim->devq->send_active,
		    rig_sim->max_dev_openings);
		lockmgr(&bt->lock, LK_RELEASE);

		/* btaction stored the bccb pointer for us */
		cx->bccb = cx->bccb ? cx->bccb :
		    (struct bt_ccb *)(uintptr_t)ccb->ccb_h.spriv_ptr0;
		if (cx->bccb)
			cx->bccb_phys = rig_ccbvtop(cx->bccb);
		st = (struct rig_state *)data;
		rig_capture(st, cx);
		st->ccb_id = ncctx - 1;
		kprintf("[rig] QUEUE done: id=%d bccb=%p opcode=0x%02x "
		    "cmd_len=%u sense_len=%u bccb_phys=0x%08x\n",
		    ncctx - 1, cx->bccb, st->hccb_opcode, st->hccb_cmd_len,
		    st->hccb_sense_len, cx->bccb_phys);
		break;
	}
	case RIGIOC_SNAP:
	{
		st = (struct rig_state *)data;
		lockmgr(&bt->lock, LK_EXCLUSIVE);
		rig_capture(st, st->ccb_id < ncctx ? &cctx[st->ccb_id] : NULL);
		lockmgr(&bt->lock, LK_RELEASE);
		break;
	}
	case RIGIOC_COMP:
	{
		struct rig_comp_args *ca = (struct rig_comp_args *)data;
		struct bt_ccb *bccb;
		struct rig_cctx *cx = NULL;

		if (ca->use_recovery) {
			bccb = bt->recovery_bccb;
		} else {
			if (ca->ccb_id >= ncctx)
				return (EINVAL);
			cx = &cctx[ca->ccb_id];
			bccb = cx->bccb;
		}
		if (bccb == NULL) {
			kprintf("[rig] COMP: target bccb is NULL (command was "
			    "never queued -- REQUEUE'd?); refusing\n");
			return (EINVAL);
		}
		kprintf("[rig] COMP: firmware writes completion: bccb=%p "
		    "comp_code=0x%02x btstat=0x%02x sdstat=0x%02x -> "
		    "production btdone()\n", bccb, ca->comp_code,
		    ca->btstat, ca->sdstat);
		/* the host adapter writes status into CCB DMA memory */
		if (ca->btstat || ca->sdstat) {
			bccb->hccb.btstat = ca->btstat;
			bccb->hccb.sdstat = ca->sdstat;
		}
		lockmgr(&bt->lock, LK_EXCLUSIVE);
		((btdone_fn)a_btdone)(bt, bccb, ca->comp_code);
		lockmgr(&bt->lock, LK_RELEASE);
		st = (struct rig_state *)data;
		bzero(st, sizeof(*st));
		if (cx) {
			rig_capture(st, cx);
		} else {
			rig_capture(st, NULL);
			st->bccb_kva = (uint64_t)(uintptr_t)bccb;
			st->bccb_flags = bccb->flags;
			st->hccb_opcode = bccb->hccb.opcode;
			st->hccb_btstat = bccb->hccb.btstat;
		}
		kprintf("[rig] COMP done: flags=0x%x pending_list=%u "
		    "timer=%u nsegs=%u ccb_status=0x%x\n",
		    st->bccb_flags, st->on_pending_list, st->timer_pending,
		    st->dmamap_nsegs, st->ccb_status);
		break;
	}
	case RIGIOC_TIMEOUT:
	{
		st = (struct rig_state *)data;
		if (st->ccb_id >= (uint32_t)ncctx ||
		    cctx[st->ccb_id].bccb == NULL)
			return (EINVAL);
		kprintf("[rig] TIMEOUT: production bttimeout(%p)\n",
		    cctx[st->ccb_id].bccb);
		((bttimeout_fn)a_bttimeout)(cctx[st->ccb_id].bccb);
		lockmgr(&bt->lock, LK_EXCLUSIVE);
		rig_capture(st, &cctx[st->ccb_id]);
		lockmgr(&bt->lock, LK_RELEASE);
		kprintf("[rig] TIMEOUT done: flags=0x%x recovery_flags=0x%x "
		    "ctlreg=0x%02x outbox_ccb=0x%08x outbox_act=0x%02x\n",
		    st->bccb_flags, st->recovery_flags, st->vregs[0],
		    st->outbox_ccbaddr, st->outbox_action);
		break;
	}
	case RIGIOC_RACE_FIRE:
	{
		st = (struct rig_state *)data;
		if (st->ccb_id >= (uint32_t)ncctx ||
		    cctx[st->ccb_id].bccb == NULL)
			return (EINVAL);
		race_bccb = cctx[st->ccb_id].bccb;
		race_entered = race_done = 0;
		error = kthread_create(rig_race_thread, NULL, NULL,
		    "dfbt_stale");
		if (error)
			return (error);
		/* wait for the thread to *enter* bttimeout (before lock) */
		int n;
		for (n = 0; n < 100 && !race_entered; n++)
			tsleep(&race_entered, 0, "rigent", hz / 20);
		kprintf("[rig] RACE_FIRE: stale bttimeout entered=%d "
		    "(it is now blocked acquiring bt->lock)\n", race_entered);
		break;
	}
	case RIGIOC_CALCGEOM:
	{
		struct rig_calcgeom_args *ga = (struct rig_calcgeom_args *)data;
		union ccb *ccb = xpt_alloc_ccb();

		xpt_setup_ccb(&ccb->ccb_h, io_path, /*priority*/ 1);
		ccb->ccb_h.func_code = XPT_CALC_GEOMETRY;
		ccb->ccb_h.cbfcnp = rig_done;
		ccb->ccg.block_size = ga->block_size;
		ccb->ccg.volume_size = ga->volume_size;
		ccb->ccg.heads = 0;
		ccb->ccg.secs_per_track = 0;
		ccb->ccg.cylinders = 0;
		kprintf("[rig] CALCGEOM: xpt_action(block_size=%u "
		    "volume_size=%u)\n", ga->block_size, ga->volume_size);
		lockmgr(&bt->lock, LK_EXCLUSIVE);
		xpt_action(ccb);	/* production XPT -> btaction */
		lockmgr(&bt->lock, LK_RELEASE);
		/* if we get here the geometry completed without a trap */
		ga->heads = ccb->ccg.heads;
		ga->secs_per_track = ccb->ccg.secs_per_track;
		ga->cylinders = ccb->ccg.cylinders;
		ga->status = ccb->ccb_h.status;
		kprintf("[rig] CALCGEOM survived: heads=%u spt=%u cyl=%u "
		    "status=0x%x\n", ga->heads, ga->secs_per_track,
		    ga->cylinders, ga->status);
		break;
	}
	case RIGIOC_BTRESET:
	{
		st = (struct rig_state *)data;
		kprintf("[rig] BTRESET: production btreset(hard=%d)\n",
		    st->hccb_opcode);
		lockmgr(&bt->lock, LK_EXCLUSIVE);
		st->err = ((btreset_fn)a_btreset)(bt, st->hccb_opcode ? 1 : 0);
		lockmgr(&bt->lock, LK_RELEASE);
		rig_capture(st, NULL);
		kprintf("[rig] BTRESET done: err=%d status_reg=0x%02x\n",
		    st->err, st->vregs[0]);
		break;
	}
	case RIGIOC_VREG:
	{
		struct rig_vreg_args *va = (struct rig_vreg_args *)data;
		if (va->off > 3)
			return (EINVAL);
		regpage[va->off] = va->val;
		break;
	}
	case RIGIOC_HOLD:
		lockmgr(&bt->lock, LK_EXCLUSIVE);
		kprintf("[rig] HOLD: bt->lock held by ioctl thread\n");
		break;
	case RIGIOC_REL:
		lockmgr(&bt->lock, LK_RELEASE);
		kprintf("[rig] REL: bt->lock released\n");
		break;
	case RIGIOC_SIMQ_REL:
		lockmgr(&bt->lock, LK_EXCLUSIVE);
		xpt_release_simq(rig_sim, /*run_queue*/ TRUE);
		lockmgr(&bt->lock, LK_RELEASE);
		kprintf("[rig] SIMQ released\n");
		break;
	default:
		return (ENOTTY);
	}
	return (0);
}

/* ------------------------------------------------------------------ */
/* module boilerplate                                                  */
/* ------------------------------------------------------------------ */
static int
dfbt_modevent(module_t mod, int type, void *data)
{
	switch (type) {
	case MOD_LOAD:
		rig_cdev = make_dev(&rig_ops, 0, UID_ROOT, GID_WHEEL,
		    0600, "dfbt%d", 0);
		kprintf("[rig] dfbt rig loaded (prime kern.dfbt.a_* then "
		    "open /dev/dfbt0)\n");
		return (0);
	case MOD_UNLOAD:
		if (bt != NULL) {
			kprintf("[rig] refusing unload while rig is set up "
			    "(reboot instead)\n");
			return (EBUSY);
		}
		if (rig_cdev != NULL)
			destroy_dev(rig_cdev);
		return (0);
	default:
		return (EOPNOTSUPP);
	}
}

DEV_MODULE(dfbt_rig, dfbt_modevent, NULL);