/*
 * DF-2263 / DF-2264 / DF-2265 verification harness -- POSTAMBLE.
 *
 * Appended after the verbatim scsi_sa.c. Implements the interposed
 * external shims, the fabricated CAM/tape state, the test drivers and
 * the sysctl triggers (debug.df226x.*).
 *
 * Tests:
 *   t2263a  DF-2263 double-biodone of a physio buffer (pbuf +
 *           biodone_sync, exactly what physio() queues for /dev/sa0):
 *           real biodone twice on the same bio -> bpdone() twice ->
 *           KASSERT panic on INVARIANTS kernels ("bpdone: already done").
 *   t2263b  DF-2263 EOF-drain loop non-termination + queue_count
 *           underflow + starvation of the remaining queued bios
 *           (counting completion callback; harness-side rescue after
 *           N iterations because the real loop has no exit).
 *   t2264a  DF-2264 saioctl(control dev, unknown ioctl) returns EINVAL
 *           while still holding the SIM lock; wedge probe.
 *   t2264b  DF-2264 saioctl(MTIOCSETEOTMODEL) with cam_periph_hold()
 *           interrupted by a signal returns while holding the SIM lock.
 *   t2265   DF-2265 sasetparams(compression-only) on a drive whose
 *           DATA COMPRESSION page reports no DCC bit returns ENODEV
 *           and leaks the ccomp allocation (M_SCSISA accounting).
 */

#undef biodone
#undef xpt_release_ccb
#undef xpt_schedule
#undef cam_periph_getccb
#undef cam_periph_runccb
#undef xpt_print

/* The real declarations were consumed from the system headers in the
 * preamble BEFORE the interposition macros were defined; they remain
 * in scope for the whole translation unit, so the #undef'd names below
 * call the REAL kernel functions. */

/* ================= shim implementations ================= */

static int		dfh_xpt_rel_count;
static int		dfh_xpt_sched_count;
static union ccb	dfh_ccb;

static void
dfh_xpt_release_ccb(union ccb *ccb)
{
	(void)ccb;
	dfh_xpt_rel_count++;
}

static void
dfh_xpt_schedule(struct cam_periph *periph, u_int32_t pri)
{
	(void)periph;
	(void)pri;
	dfh_xpt_sched_count++;
}

static union ccb *
dfh_cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
{
	(void)periph;
	(void)priority;
	bzero(&dfh_ccb, sizeof(dfh_ccb));
	return (&dfh_ccb);
}

static void
dfh_xpt_print(struct cam_path *path, const char *fmt, ...)
{
	__va_list ap;

	(void)path;
	kprintf("DFH xpt_print: ");
	__va_start(ap, fmt);
	kvprintf(fmt, ap);
	__va_end(ap);
	kprintf("\n");
}

/*
 * Fake transport for DF-2265.  Answers MODE SENSE(6) the way a real
 * drive that "supports the page but is not compression capable" does:
 * a DATA COMPRESSION page (0x0f) whose dce_and_dcc byte has the DCC
 * bit (0x40) clear.  sagetparams() copies this page into the caller's
 * sa_comp_t; sasetparams() then falls through to the ENODEV path at
 * scsi_sa.c:2924-2941 (leaking ccomp, the bug under test).
 */
static int
dfh_cam_periph_runccb(union ccb *ccb,
		      int (*error_routine)(union ccb *, cam_flags,
		      u_int32_t),
		      cam_flags camflags, u_int32_t sense_flags,
		      struct devstat *stats)
{
	u_int8_t *cdb;
	u_int8_t *d;
	u_int32_t len;

	(void)error_routine;
	(void)camflags;
	(void)sense_flags;
	(void)stats;

	cdb = ccb->csio.cdb_io.cdb_bytes;
	d = ccb->csio.data_ptr;
	len = ccb->csio.dxfer_len;

	if (cdb[0] == 0x1a && d != NULL && len >= 28) {	/* MODE SENSE(6) */
		u_int8_t page = cdb[2] & 0x3f;

		bzero(d, len);
		d[0] = (u_int8_t)(len - 1);	/* mode data length */
		d[3] = 8;			/* block descriptor length */
		/* block descriptor, block length = 512 */
		d[4 + 4] = 0;
		d[4 + 5] = 0;
		d[4 + 6] = 2;
		d[4 + 7] = 0;
		if (page == 0x0f) {	/* SMS_DATA_COMPRESSION_PAGE */
			struct scsi_data_compression_page *pg;

			pg = (struct scsi_data_compression_page *)(d + 12);
			pg->page_code = 0x0f;
			pg->page_length = 0x0e;
			/*
			 * dce_and_dcc left 0: the drive reports the
			 * compression page but WITHOUT the DCC
			 * ("compression capable") bit.
			 */
			scsi_ulto4b(0, pg->comp_algorithm);
			scsi_ulto4b(0, pg->decomp_algorithm);
		}
	}
	ccb->ccb_h.status = CAM_REQ_CMP;
	return (0);
}

/* ================= biodone observation shim ================= */

#define	DFH_MAXBIOS	8
struct dfh_bio_count {
	struct bio	*bio;
	int		count;
};
static struct dfh_bio_count	dfh_bc[DFH_MAXBIOS];
static int			dfh_nbc;
static int			dfh_biodone_calls;

/* scenario selectors */
static int			dfh_mode;	/* 0=off 1=2263a 2=2263b */
static int			dfh_rescue_at = 2000;
static int			dfh_rescued;
static struct sa_softc		*dfh_sa_watch;	/* 2263b softc */

static int
dfh_bio_tick(struct bio *bio)
{
	int i;

	for (i = 0; i < dfh_nbc; i++) {
		if (dfh_bc[i].bio == bio)
			break;
	}
	if (i == dfh_nbc) {
		if (i == DFH_MAXBIOS)
			return (-1);
		dfh_bc[i].bio = bio;
		dfh_bc[i].count = 0;
		dfh_nbc++;
	}
	return (++dfh_bc[i].count);
}

/*
 * Completion callback for the fabricated bios of scenario 2263b.
 * Real biodone() clears bio_done before calling us; we re-arm so the
 * next (buggy) biodone of the same bio also terminates here instead
 * of falling through to bpdone() on a fabricated struct buf.
 *
 * The rescue (after dfh_rescue_at re-completions of the first bio)
 * runs in the context of the thread spinning inside sastart(), so it
 * races with nothing: it neutralizes the stale bio_act link of bio #1
 * and empties the queue head, which makes the real loop's
 * `bioq_first() != NULL' test fail and lets sastart return.
 */
static void
dfh_bio_done_cb(struct bio *bio)
{
	int c;

	c = dfh_bio_tick(bio);
	if (c == 1) {
		kprintf("DFH2263B: first completion of bio %p\n", bio);
	} else if (c == dfh_rescue_at && !dfh_rescued &&
		   bio == dfh_bc[0].bio) {
		dfh_rescued = 1;
		kprintf("DFH2263B: *** LOOP DOES NOT TERMINATE: bio %p "
			"completed %d times (queue still %sEMPTY, "
			"queue_count=%d). Applying harness rescue.\n",
			bio, c,
			(bioq_first(&dfh_sa_watch->bio_queue) != NULL) ?
			"" : "NOT ", dfh_sa_watch->queue_count);
		/*
		 * Neutralize the stale forward link only; leave tqe_prev
		 * pointing at the queue head so the next stale
		 * TAILQ_REMOVE writes head.first = NULL (benign) instead
		 * of dereferencing NULL.
		 */
		bio->bio_act.tqe_next = NULL;
		dfh_sa_watch->bio_queue.queue.tqh_first = NULL;
		dfh_sa_watch->bio_queue.queue.tqh_last =
		    &dfh_sa_watch->bio_queue.queue.tqh_first;
	} else if (c > 1 && (c <= 3 || (c % 1000) == 0)) {
		kprintf("DFH2263B: bio %p completed %d times "
			"(queue_count=%d)\n", bio, c,
			dfh_sa_watch->queue_count);
	}
	bio->bio_done = dfh_bio_done_cb;	/* re-arm */
}

static void
dfh_biodone(struct bio *bio)
{
	int c;

	dfh_biodone_calls++;

	if (dfh_mode == 2) {
		/* scenario B: the completion callback owns the counting */
		if (dfh_biodone_calls <= 6 ||
		    (dfh_biodone_calls % 500) == 0)
			kprintf("DFH2263B: biodone call #%d on bio %p "
				"(queue_count=%d bioq_first=%p ra=%p)\n",
				dfh_biodone_calls, bio,
				dfh_sa_watch->queue_count,
				bioq_first(&dfh_sa_watch->bio_queue),
				__builtin_return_address(0));
		biodone(bio);
		if (dfh_biodone_calls <= 6)
			kprintf("DFH2263B: after-biodone: bioq_first=%p "
				"transition=%p B1(n=%p p=%p) "
				"queue_count=%d\n",
				bioq_first(&dfh_sa_watch->bio_queue),
				dfh_sa_watch->bio_queue.transition,
				dfh_bc[0].bio->bio_act.tqe_next,
				dfh_bc[0].bio->bio_act.tqe_prev,
				dfh_sa_watch->queue_count);
		return;
	}

	c = dfh_bio_tick(bio);

	if (dfh_mode == 1 && c >= 2) {
		struct buf *bp = bio->bio_buf;

		/*
		 * Second completion of the SAME bio: the bug.  Print the
		 * evidence, then let the real biodone run -- on this
		 * INVARIANTS kernel bpdone() is expected to KASSERT
		 * ("bp %p already done!").
		 */
		kprintf("DFH2263A: *** RE-COMPLETION: biodone called on "
			"bio %p for the %d. time (bp=%p b_cmd=%d "
			"b_flags=0x%x b_resid=%d b_error=%d)\n",
			bio, c, bp, (int)bp->b_cmd, bp->b_flags,
			bp->b_resid, bp->b_error);
		kprintf("DFH2263A: entering real biodone -> bpdone() "
			"(expect KASSERT panic on INVARIANTS)\n");
	}
	biodone(bio);	/* the real thing */
}

/* ================= fabricated CAM state ================= */

static void
dfh_setup_periph(struct cam_periph *ph, struct cam_sim *sim,
		 struct lock *lk, struct sa_softc *sa, u_int32_t pflags)
{
	bzero(ph, sizeof(*ph));
	bzero(sim, sizeof(*sim));
	bzero(sa, sizeof(*sa));

	lockinit(lk, "dfhsim", 0, 0);
	bioq_init(&sa->bio_queue);
	sim->lock = lk;
	ph->sim = sim;
	ph->softc = sa;
	ph->path = NULL;			/* CAM_DEBUG is inert */
	ph->immediate_priority = CAM_PRIORITY_NONE;
	ph->pinfo.priority = 0;
	ph->refcount = 1;
	ph->flags = pflags;
	sa->state = SA_STATE_NORMAL;
}

/* ================= DF-2263 scenario A ================= */

static struct lock		dfh_lkA;
static struct cam_sim		dfh_simA;
static struct cam_periph		dfh_phA;
static struct sa_softc		dfh_saA;
static struct buf		*dfh_pbufA[3];
static volatile int		dfh_2263a_done;

static void
dfh_thr_2263a(void *arg)
{
	kprintf("DFH2263A: thread calling REAL sastart() with 3 physio "
		"bios queued (pbuf + biodone_sync), "
		"SA_FLAG_ERR_PENDING|SA_FLAG_EOF_PENDING set\n");
	dfh_mode = 1;
	sastart(&dfh_phA, &dfh_ccb);		/* THE REAL FUNCTION */
	dfh_mode = 0;
	kprintf("DFH2263A: sastart RETURNED (no double completion)\n");
	dfh_2263a_done = 1;
	kthread_exit();
}

static int
dfh_run_2263a(void)
{
	int i;

	dfh_setup_periph(&dfh_phA, &dfh_simA, &dfh_lkA, &dfh_saA, 0);
	dfh_nbc = 0;
	dfh_biodone_calls = 0;
	dfh_2263a_done = 0;

	for (i = 0; i < 3; i++) {
		struct buf *bp;

		bp = getpbuf_mem(NULL);		/* what physio() uses */
		bp->b_cmd = BUF_CMD_READ;
		bp->b_bcount = 512;
		bp->b_resid = 0;
		bp->b_error = 0;
		bp->b_bio1.bio_done = biodone_sync;	/* physio() sets this */
		bp->b_bio1.bio_flags |= BIO_SYNC;
		dfh_pbufA[i] = bp;
		bioq_insert_tail(&dfh_saA.bio_queue, &bp->b_bio1);
		dfh_saA.queue_count++;
	}
	dfh_saA.flags = SA_FLAG_ERR_PENDING | SA_FLAG_EOF_PENDING;

	kprintf("DFH2263A: bios queued: %p %p %p queue_count=%d\n",
		&dfh_pbufA[0]->b_bio1, &dfh_pbufA[1]->b_bio1,
		&dfh_pbufA[2]->b_bio1, dfh_saA.queue_count);

	kthread_create(dfh_thr_2263a, NULL, NULL, "dfh263a");

	/*
	 * If the kernel panics inside the thread we never come back --
	 * the serial console captures the panic.  Otherwise (fixed
	 * kernel) wait for completion and report.
	 */
	for (i = 0; i < 50 && !dfh_2263a_done; i++)
		tsleep(&dfh_2263a_done, 0, "dfh263w", hz / 10);

	if (dfh_2263a_done) {
		kprintf("DFH2263A: SUMMARY completions per bio: "
			"%d %d %d (expect 1 1 1), queue_count=%d (expect 0),"
			" xpt_release_ccb calls=%d\n",
			dfh_bc[0].count, dfh_bc[1].count, dfh_bc[2].count,
			dfh_saA.queue_count, dfh_xpt_rel_count);
		for (i = 0; i < 3; i++)
			relpbuf(dfh_pbufA[i], NULL);
		return (0);
	}
	kprintf("DFH2263A: thread still running after 5s\n");
	return (0);
}

/* ================= DF-2263 scenario B ================= */

static struct lock		dfh_lkB;
static struct cam_sim		dfh_simB;
static struct cam_periph		dfh_phB;
static struct sa_softc		dfh_saB;
static struct bio		dfh_bioB[3];
static struct buf		dfh_bufB[3];
static volatile int		dfh_2263b_done;

static void
dfh_thr_2263b(void *arg)
{
	kprintf("DFH2263B: thread calling REAL sastart() with 3 bios "
		"queued, ERR|EOF pending\n");
	dfh_mode = 2;
	sastart(&dfh_phB, &dfh_ccb);		/* THE REAL FUNCTION */
	dfh_mode = 0;
	dfh_2263b_done = 1;
	kprintf("DFH2263B: sastart RETURNED (loop terminated)\n");
	kthread_exit();
}

static int
dfh_run_2263b(void)
{
	int i;

	dfh_setup_periph(&dfh_phB, &dfh_simB, &dfh_lkB, &dfh_saB, 0);
	dfh_nbc = 0;
	dfh_biodone_calls = 0;
	dfh_rescued = 0;
	dfh_2263b_done = 0;
	dfh_sa_watch = &dfh_saB;

	for (i = 0; i < 3; i++) {
		bzero(&dfh_bioB[i], sizeof(dfh_bioB[i]));
		bzero(&dfh_bufB[i], sizeof(dfh_bufB[i]));
		dfh_bioB[i].bio_buf = &dfh_bufB[i];
		dfh_bioB[i].bio_done = dfh_bio_done_cb;
		dfh_bioB[i].bio_prev = NULL;
		dfh_bioB[i].bio_track = NULL;
		dfh_bufB[i].b_bcount = 512;
		bioq_insert_tail(&dfh_saB.bio_queue, &dfh_bioB[i]);
		dfh_saB.queue_count++;
	}
	dfh_saB.flags = SA_FLAG_ERR_PENDING | SA_FLAG_EOF_PENDING;

	kprintf("DFH2263B: queue links: first=%p lastp=%p; "
		"B1(n=%p p=%p) B2(n=%p p=%p) B3(n=%p p=%p)\n",
		dfh_saB.bio_queue.queue.tqh_first,
		dfh_saB.bio_queue.queue.tqh_last,
		dfh_bioB[0].bio_act.tqe_next, dfh_bioB[0].bio_act.tqe_prev,
		dfh_bioB[1].bio_act.tqe_next, dfh_bioB[1].bio_act.tqe_prev,
		dfh_bioB[2].bio_act.tqe_next, dfh_bioB[2].bio_act.tqe_prev);
	dfh_xpt_rel_count = 0;
	dfh_xpt_sched_count = 0;

	kthread_create(dfh_thr_2263b, NULL, NULL, "dfh263b");

	/*
	 * Watchdog: watch the counts climb.  The decisive evidence is
	 * printed by the completion callback at rescue time (and by the
	 * progress prints below while the loop spins).
	 */
	for (i = 0; i < 25 && !dfh_2263b_done; i++) {
		tsleep(&dfh_2263b_done, 0, "dfh263w", hz / 10);
		if (dfh_nbc > 0 && (i % 5) == 1 && !dfh_rescued)
			kprintf("DFH2263B: t+%dms bio#1 completed %d times,"
				" bio#2 %d, bio#3 %d, queue_count=%d,"
				" queue_len_bio2_pending=%d\n",
				(i + 1) * 100, dfh_bc[0].count,
				dfh_nbc > 1 ? dfh_bc[1].count : 0,
				dfh_nbc > 2 ? dfh_bc[2].count : 0,
				dfh_saB.queue_count,
				bioq_first(&dfh_saB.bio_queue) != NULL);
	}

	kprintf("DFH2263B: SUMMARY bio#1=%d bio#2=%d bio#3=%d "
		"(b2/b3 STARVED: their callers never wake), "
		"final queue_count=%d (started at 3: UNDERFLOW by %d), "
		"rescue_used=%d, xpt_release_ccb=%d, xpt_schedule=%d\n",
		dfh_bc[0].count, dfh_nbc > 1 ? dfh_bc[1].count : 0,
		dfh_nbc > 2 ? dfh_bc[2].count : 0,
		dfh_saB.queue_count,
		3 - dfh_saB.queue_count,
		dfh_rescued, dfh_xpt_rel_count, dfh_xpt_sched_count);
	if (!dfh_2263b_done)
		kprintf("DFH2263B: thread did NOT terminate within 2.5s\n");
	return (0);
}

/* ================= DF-2264 ================= */

struct dfh2264_inst {
	struct lock		lk;
	struct cam_sim		sim;
	struct cam_periph	ph;
	struct sa_softc		sa;
	struct cdev		cd;
	volatile int		acquired;
	int			wedged;
};
static struct dfh2264_inst	dfh_i2264[4];
static int			dfh_n2264;

static void
dfh_thr_2264probe(void *arg)
{
	struct dfh2264_inst *in = arg;

	cam_sim_lock(&in->lk);		/* same as cam_periph_lock */
	in->acquired = 1;
	kprintf("DFH2264: probe thread ACQUIRED the SIM lock\n");
	cam_sim_unlock(&in->lk);
	kthread_exit();
}

/*
 * After saioctl() has returned (leaking the lock), spawn a thread that
 * tries to take the same SIM lock.  If the leak is real it blocks
 * forever -- we give it 1s and report.
 */
static int
dfh_2264_wedge_probe(struct dfh2264_inst *in, const char *tag)
{
	int i;

	in->acquired = 0;
	kthread_create(dfh_thr_2264probe, in, NULL, "dfh264p");
	for (i = 0; i < 10 && !in->acquired; i++)
		tsleep(&in->acquired, 0, "dfh264w", hz / 10);
	if (in->acquired) {
		kprintf("DFH2264: %s: probe acquired the lock (NOT leaked)\n",
			tag);
		return (0);
	}
	in->wedged = 1;
	kprintf("DFH2264: %s: *** probe thread still BLOCKED after 1s: "
		"SIM-wide wedge confirmed (thread parked forever)\n", tag);
	return (1);
}

static int
dfh_run_2264a(void)
{
	struct dfh2264_inst *in;
	struct dev_ioctl_args ap;
	int r, st;
	u_int32_t dummy = 0;

	if (dfh_n2264 >= 4) {
		kprintf("DFH2264: no more instances (4 max)\n");
		return (ENOSPC);
	}
	in = &dfh_i2264[dfh_n2264++];
	dfh_setup_periph(&in->ph, &in->sim, &in->lk, &in->sa, 0);
	bzero(&in->cd, sizeof(in->cd));
	in->cd.si_uminor = (1 << 29);		/* SA_IS_CTRL, unit 0 */
	cam_extend_set(saperiphs, 0, &in->ph);

	bzero(&ap, sizeof(ap));
	ap.a_head.a_dev = &in->cd;
	ap.a_cmd = 0xfffffffeUL;			/* unknown ioctl */
	ap.a_data = (caddr_t)&dummy;

	st = lockstatus(&in->lk, curthread);
	kprintf("DFH2264A: lock status before saioctl: %d\n", st);

	r = saioctl(&ap);			/* THE REAL FUNCTION */

	st = lockstatus(&in->lk, curthread);
	kprintf("DFH2264A: saioctl(unknown ioctl, control device) "
		"returned %d (EINVAL=%d); SIM lock status AFTER return: %d "
		"(LK_EXCLUSIVE=%d)\n",
		r, EINVAL, st, LK_EXCLUSIVE);
	if (r == EINVAL && st == LK_EXCLUSIVE)
		kprintf("DFH2264A: *** LOCK LEAKED: returned to caller with "
			"the SIM-wide CAM lock held (scsi_sa.c:849 vs "
			":920 early return)\n");

	dfh_2264_wedge_probe(in, "2264A");
	return (0);
}

static int
dfh_run_2264b(void)
{
	struct dfh2264_inst *in;
	struct dev_ioctl_args ap;
	int r, st;
	u_int32_t eot = 1;

	if (dfh_n2264 >= 4) {
		kprintf("DFH2264: no more instances (4 max)\n");
		return (ENOSPC);
	}
	in = &dfh_i2264[dfh_n2264++];
	/*
	 * CAM_PERIPH_LOCKED: pretend another thread holds the periph
	 * (as an open/ioctl in flight would).  saioctl() then calls
	 * cam_periph_hold() which sleeps -- interrupt it with a signal
	 * from userland (see run.sh: kill -USR1 the sysctl process).
	 */
	dfh_setup_periph(&in->ph, &in->sim, &in->lk, &in->sa,
	    CAM_PERIPH_LOCKED);
	bzero(&in->cd, sizeof(in->cd));
	in->cd.si_uminor = (1 << 29);
	cam_extend_set(saperiphs, 0, &in->ph);

	bzero(&ap, sizeof(ap));
	ap.a_head.a_dev = &in->cd;
	ap.a_cmd = MTIOCSETEOTMODEL;
	ap.a_data = (caddr_t)&eot;

	kprintf("DFH2264B: calling REAL saioctl(MTIOCSETEOTMODEL); "
		"cam_periph_hold will sleep (send SIGUSR1 now)\n");
	r = saioctl(&ap);			/* THE REAL FUNCTION */

	st = lockstatus(&in->lk, curthread);
	kprintf("DFH2264B: saioctl returned %d after interrupted "
		"cam_periph_hold; SIM lock status AFTER return: %d\n",
		r, st);
	if (st == LK_EXCLUSIVE)
		kprintf("DFH2264B: *** LOCK LEAKED via interrupted hold "
			"(scsi_sa.c:912-914 early return)\n");

	dfh_2264_wedge_probe(in, "2264B");
	return (0);
}

/* ================= DF-2265 ================= */

static struct lock		dfh_lkC;
static struct cam_sim		dfh_simC;
static struct cam_periph		dfh_phC;
static struct sa_softc		dfh_saC;

static u_long
dfh_scsisa_stat(int which)
{
	u_long v = 0;
	int i;

	for (i = 0; i < ncpus; i++) {
		if (which == 0)
			v += M_SCSISA->ks_use[i].memuse;
		else
			v += M_SCSISA->ks_use[i].calls;
	}
	return (v);
}

static int
dfh_run_2265(int iters)
{
	u_long mem0, mem1, cal0, cal1;
	int r, enodev = 0, other = 0, i;

	dfh_setup_periph(&dfh_phC, &dfh_simC, &dfh_lkC, &dfh_saC, 0);
	cam_extend_set(saperiphs, 0, &dfh_phC);

	mem0 = dfh_scsisa_stat(0);
	cal0 = dfh_scsisa_stat(1);
	kprintf("DFH2265: M_SCSISA before: memuse=%lu calls=%lu, "
		"sizeof(sa_comp_t)=%zu, iterations=%d\n",
		mem0, cal0, sizeof(sa_comp_t), iters);

	for (i = 0; i < iters; i++) {
		/*
		 * Exactly what saioctl(MTIOCTOP, MTCOMP) does at
		 * scsi_sa.c:1266 for a compression-only change.
		 */
		r = sasetparams(&dfh_phC, SA_PARAM_COMPRESSION, 0, 0, 1, 0);
		if (r == ENODEV)
			enodev++;
		else
			other++;
	}

	mem1 = dfh_scsisa_stat(0);
	cal1 = dfh_scsisa_stat(1);
	kprintf("DFH2265: M_SCSISA after: memuse=%lu calls=%lu; "
		"returns: ENODEV=%d other=%d\n", mem1, cal1, enodev, other);
	kprintf("DFH2265: *** LEAK: memuse grew by %lu bytes = %ld x "
		"sizeof(sa_comp_t) (%zu); allocation calls grew by %lu "
		"(expect 3 per call: ccomp+mode_buffer+sagetparams buffer; "
		"ccomp is the one never freed on the ENODEV path)\n",
		mem1 - mem0, (long)((mem1 - mem0) / sizeof(sa_comp_t)),
		sizeof(sa_comp_t), cal1 - cal0);
	return (0);
}

/* ================= sysctl triggers ================= */

static int
dfh_sysctl_2263a(SYSCTL_HANDLER_ARGS)
{
	int t, error;

	t = 0;
	error = sysctl_handle_int(oidp, &t, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);
	return (dfh_run_2263a());
}

static int
dfh_sysctl_2263b(SYSCTL_HANDLER_ARGS)
{
	int t, error;

	t = 0;
	error = sysctl_handle_int(oidp, &t, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);
	return (dfh_run_2263b());
}

static int
dfh_sysctl_2264a(SYSCTL_HANDLER_ARGS)
{
	int t, error;

	t = 0;
	error = sysctl_handle_int(oidp, &t, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);
	return (dfh_run_2264a());
}

static int
dfh_sysctl_2264b(SYSCTL_HANDLER_ARGS)
{
	int t, error;

	t = 0;
	error = sysctl_handle_int(oidp, &t, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);
	return (dfh_run_2264b());
}

static int
dfh_sysctl_2265(SYSCTL_HANDLER_ARGS)
{
	int t, error;

	error = sysctl_handle_int(oidp, &t, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);
	return (dfh_run_2265(t > 0 ? t : 1000));
}

SYSCTL_NODE(_debug, OID_AUTO, df226x, CTLFLAG_RW, 0, "DF-2263/4/5 rig");
SYSCTL_PROC(_debug_df226x, OID_AUTO, t2263a, CTLTYPE_INT | CTLFLAG_RW,
	    0, 0, dfh_sysctl_2263a, "I", "DF-2263 double biodone (panics)");
SYSCTL_PROC(_debug_df226x, OID_AUTO, t2263b, CTLTYPE_INT | CTLFLAG_RW,
	    0, 0, dfh_sysctl_2263b, "I", "DF-2263 drain loop non-termination");
SYSCTL_PROC(_debug_df226x, OID_AUTO, t2264a, CTLTYPE_INT | CTLFLAG_RW,
	    0, 0, dfh_sysctl_2264a, "I", "DF-2264 unknown-ioctl lock leak");
SYSCTL_PROC(_debug_df226x, OID_AUTO, t2264b, CTLTYPE_INT | CTLFLAG_RW,
	    0, 0, dfh_sysctl_2264b, "I", "DF-2264 interrupted-hold lock leak");
SYSCTL_PROC(_debug_df226x, OID_AUTO, t2265, CTLTYPE_INT | CTLFLAG_RW,
	    0, 0, dfh_sysctl_2265, "I", "DF-2265 ccomp leak (arg=iterations)");