/*
 * dfrig2491.c -- DF-2491 verification rig
 *                (sys/dev/disk/nata/ata-lowlevel.c:771-790)
 *
 * Finding under test:
 *
 *	The CHS sub-path of ata_tf_write() takes its geometry straight
 *	from the device-supplied ATA IDENTIFY words with NO zero check:
 *
 *	    ata-lowlevel.c:774	if (atadev->param.atavalid & ATA_FLAG_54_58) {
 *	    ata-lowlevel.c:775-776	    heads = atadev->param.current_heads;
 *	                                 sectors = atadev->param.current_sectors;
 *	    ata-lowlevel.c:778-780	} else { heads = param.heads;
 *	                                 sectors = param.sectors; }
 *	    ata-lowlevel.c:783	    ATA_IDX_OUTB(ch, ATA_SECTOR,
 *	                                  (request->u.ata.lba % sectors)+1);
 *	    ata-lowlevel.c:784-787	    CYL_LSB/MSB = lba / (sectors * heads);
 *	    ata-lowlevel.c:788-790	    DRIVE head field =
 *	                                  (lba % (sectors*heads)) / sectors;
 *
 *	`sectors`==0 -> #DE at :783 (mod); `heads`==0 -> #DE at :785/:790
 *	(div).  Contrast the sibling consumer of the SAME words,
 *	ad_get_geometry() (ata-disk.c:469-470), which guards with
 *	`&& atadev->param.current_heads && atadev->param.current_sectors`.
 *
 *	Gate (ata-disk.c:484-486, ad_get_geometry):
 *	    if (!ad_version(param.version_major) || !lbasize)
 *		atadev->flags |= ATA_D_USE_CHS;
 *	ad_version(0) == 0 (ata-disk.c ad_version: no bits set -> 0), so a
 *	device reporting version_major==0 (or lba_size==0) is FORCED into
 *	CHS mode -- then every taskfile write divides by the unvalidated
 *	geometry.
 *
 *	DIVERGENCE that makes DF-2491 independent of DF-2489 (the
 *	ata-disk.c:147/:540 divisions): a device whose word 53 reports
 *	words 54-58 valid (ATA_FLAG_54_58) with current_heads==16,
 *	current_sectors==0 and SANE raw words (heads=16, sectors=63):
 *	  - ad_get_geometry falls to its guarded else-branch
 *	    (current_sectors==0) -> adp->heads=16/adp->sectors=63 ->
 *	    ad_attach's ata-disk.c:147 division is SAFE -> the disk
 *	    ATTACHES and ATA_D_USE_CHS is set;
 *	  - ata_tf_write's UNguarded check takes the current_* branch
 *	    (only the flag is tested) -> sectors = current_sectors = 0 ->
 *	    the FIRST I/O (the geom partition-table probe included) dies
 *	    with "Fatal trap 18: integer divide fault" at :783.
 *
 * Rig design (established nata verbatim-interposition class, cf.
 * DF-2210 ata-national, DF-2489 ata-disk):
 *
 *  1. KERNEL-PATH dispatch (tests 1/2/3/4/10): the RUNNING KERNEL'S
 *     OWN production chain is dispatched end-to-end on a fabricated
 *     channel + ata device:
 *
 *	   rig -> ata_controlcmd()            [kernel, ata-queue.c:158]
 *	       -> ata_queue_request()         [kernel, ata-queue.c:77;
 *	                                       ATA_STALL_QUEUE + ATA_R_CONTROL
 *	                                       -> direct HW call, :98-107]
 *	       -> ch->hw.begin_transaction == the kernel's own
 *	       ata_begin_transaction()        [ata-lowlevel.c:67, wired by
 *	                                       the kernel's own exported
 *	                                       ata_generic_hw(), :54-63]
 *	       -> ch->hw.command == the kernel's own
 *	       ata_generic_command()          [ata-lowlevel.c:648]
 *	       -> ch->hw.tf_write == the kernel's own STATIC
 *	       ata_tf_write()                 [ata-lowlevel.c:750]  <-- trap
 *
 *     The channel's I/O registers are a kernel-memory register file
 *     reached through a fabricated struct resource with
 *     r_bustag = X86_64_BUS_SPACE_MEM (exactly how a real MMIO IDE
 *     register block looks to these macros); the division under test
 *     happens BEFORE any register write of the CHS block.  Completion
 *     of the non-fatal (control) variants is the kernel's own
 *     ata_interrupt() [ata-all.c:314] -> ata_end_transaction() ->
 *     tf_read() -> ata_finish() -> ata_completed() -> wakeup, driven
 *     by a rig "device" thread that plays the drive raising its
 *     interrupt (status register = READY).
 *
 *  2. VERBATIM COPY dispatch (tests 5/6): the production
 *     sys/dev/disk/nata/ata-lowlevel.c is #included UNMODIFIED (only
 *     the five global entry points renamed to avoid duplicate globals
 *     against the built-in kernel; the static ata_tf_write under test
 *     keeps its production name and body).  build.sh sha256-checks
 *     the file against the audited sys/ tree.  Same fabricated
 *     channel, same production queue dispatch -- trap in the rig copy
 *     proves byte-identity.
 *
 *  3. FULL PRODUCTION FUNNEL (tests 7/8/9): the real kernel's own
 *     ata_identify() (ata-all.c:683) on the fabricated channel, with
 *     IDENTIFY served by the hw.begin_transaction stub (the DF-2489
 *     seam; the x86_64 bus_space data-FIFO cannot be emulated in
 *     kernel memory, so IDENTIFY delivery is the stubbed seam): real
 *     ata_getparam() parse, real newbus child, real ad_probe(), real
 *     ad_attach() (which SURVIVES -- the DF-2489 sites divide by the
 *     raw 16/63 fallback), real ATA_D_USE_CHS setting.  Then the hw
 *     seam is swapped to the kernel's OWN real
 *     begin/end/command/tf_write chain (ata_generic_hw) and the FIRST
 *     I/O is fired with ata_controlcmd() on the REAL kernel-created
 *     ad child: the kernel's own static ata_tf_write divides by the
 *     kernel-parsed current_sectors==0 -> trap 18 at :783 (test 7) /
 *     current_heads==0 -> trap at :785 (test 9).  The fabricated
 *     state is ONLY: the IDENTIFY bytes (the attacking device) and
 *     the register file (the device's register interface); nothing
 *     between the parsed IDENTIFY words and the division is rig
 *     code.
 *
 * Tests (sysctl kern.dfrig2491.go=N; read kern.dfrig2491.status):
 *
 *   1  CONTROL, kernel path: CHS mode, sane geometry (word53 54-58
 *      valid, current 16/63) -> NO panic; taskfile registers in the
 *      register file must equal the CHS math done independently by
 *      the rig (SECTOR=(lba%63)+1, CYL=lba/1008, DRIVE head bits).
 *   2  THE FINDING, kernel path: current_sectors=0 -> trap 18 at
 *      ata-lowlevel.c:783 (`lba % sectors').
 *   3  THE FINDING, kernel path: current_heads=0 -> trap 18 at
 *      ata-lowlevel.c:785 (`lba / (sectors*heads)').
 *   4  CONTROL, kernel path: LBA mode (no ATA_D_USE_CHS) -> NO
 *      panic; LBA taskfile registers verified (LBA bit set).
 *   5  THE FINDING, verbatim #included copy: current_sectors=0 ->
 *      identical trap in the rig-compiled production copy.
 *   6  CONTROL, verbatim copy: sane CHS -> registers must equal go1.
 *   7  THE FINDING, FULL FUNNEL: crafted IDENTIFY (word53=1,
 *      current 16/0, raw 16/63, version_major=0) -> kernel's own
 *      ad_attach SURVIVES (gate check printed), ATA_D_USE_CHS set by
 *      the kernel's own ad_get_geometry:485, then first I/O ->
 *      kernel's own ata_tf_write -> trap 18 at :783.
 *   8  CONTROL, FULL FUNNEL: sane CHS IDENTIFY -> attach, then first
 *      I/O completes through the kernel's own begin/interrupt/end
 *      chain; registers equal go1.
 *   9  THE FINDING, FULL FUNNEL, current_heads=0 -> trap :785.
 *   10 THE FINDING, kernel path, RAW-words variant: word53 does NOT
 *      report 54-58 valid, raw sectors=0 -> else-branch at :778-780
 *      -> trap at :783.  (On a stock kernel this device would ALSO
 *      panic earlier in ad_attach at ata-disk.c:147 -- DF-2489; this
 *      test isolates the ata_tf_write site directly.)
 *
 *   2/3/5/7/9/10 are FATAL (one per boot); proof = serial console
 *   panic.  Module loaded AT BOOT via /boot/loader.conf (runtime
 *   kldload GPFs on this INVARIANTS guest); scenarios dispatched
 *   from a dedicated worker thread that holds the MP token (the
 *   boot-attach context), DF-2489 pattern.
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <sys/bus.h>
#include <sys/bus_dma.h>	/* X86_64_BUS_SPACE_MEM */
#include <sys/bus_resource.h>
#include <sys/bus_private.h>	/* struct bsd_device layout */
#include <sys/conf.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/device.h>
#include <sys/callout.h>
#include <sys/lock.h>
#include <sys/kthread.h>
#include <sys/rman.h>
#include <sys/mplock2.h>

#include <sys/nata.h>

/* NOTE: nata headers (ata-all.h) have NO include guards; the
 * production ata-lowlevel.c below includes ata-all.h + ata_if.h
 * itself, so all further nata includes happen AFTER it, exactly
 * once. */

/* ------------------------------------------------------------------ */
/* verbatim production copy: only the five GLOBAL entry points are    */
/* renamed (duplicate globals against the built-in kernel are not     */
/* allowed in a KLD); every static function -- ata_tf_write included  */
/* -- is compiled byte-for-byte under its production name.            */
/* build.sh pins the file by sha256 before compiling.                 */
/* ------------------------------------------------------------------ */
#define ata_generic_hw		rig9v_ata_generic_hw
#define ata_begin_transaction	rig9v_ata_begin_transaction
#define ata_end_transaction	rig9v_ata_end_transaction
#define ata_generic_reset	rig9v_ata_generic_reset
#define ata_generic_command	rig9v_ata_generic_command
#include "ata-lowlevel.c"	/* <-- UNMODIFIED production code */
#undef ata_generic_hw
#undef ata_begin_transaction
#undef ata_end_transaction
#undef ata_generic_reset
#undef ata_generic_command

/* the declarations inside ata-all.h were renamed along with the
 * definitions above; re-declare the KERNEL's own global entry point
 * we dispatch (resolved against the running kernel at kldload) */
void ata_generic_hw(device_t dev);

#include "ata-disk.h"		/* struct ad_softc (attach-state check) */

/* ------------------------------------------------------------------ */
/* register file: what the "device" register block looks like to the  */
/* kernel -- a kernel-memory block behind a fabricated struct         */
/* resource with r_bustag = X86_64_BUS_SPACE_MEM (plain memory ops).  */
/* Every taskfile write lands here and is observable after the call.  */
/* ------------------------------------------------------------------ */
#define RO_DATA		0	/* ATA_DATA (16-bit) */
#define RO_FEATURE	2	/* ATA_FEATURE / ATA_ERROR */
#define RO_COUNT	3	/* ATA_COUNT / ATA_IREASON */
#define RO_SECTOR	4	/* ATA_SECTOR */
#define RO_CYLL		5	/* ATA_CYL_LSB */
#define RO_CYLM		6	/* ATA_CYL_MSB */
#define RO_DRIVE	7	/* ATA_DRIVE */
#define RO_COMMAND	8	/* ATA_COMMAND */
#define RO_STATUS	9	/* ATA_STATUS (drive reports READY) */
#define RO_ALTSTAT	10	/* ATA_ALTSTAT */
#define RO_CONTROL	11	/* ATA_CONTROL */
#define RO_NREG		12

static u_int8_t rig_regs_a[32], rig_regs_b[32];
static struct resource rig_res_a, rig_res_b;

static void
rig_regfile_init(u_int8_t *regs)
{
	bzero(regs, 32);
	regs[RO_STATUS] = ATA_S_READY;	/* not busy, no error, no DRQ */
	regs[RO_ALTSTAT] = ATA_S_READY;
}

static void
rig_channel_regs(struct ata_channel *ch, u_int8_t *regs,
		 struct resource *res)
{
	int i;

	bzero(res, sizeof(*res));
	res->r_bustag = X86_64_BUS_SPACE_MEM;
	res->r_bushandle = (bus_space_handle_t)(uintptr_t)regs;
	for (i = 0; i < ATA_MAX_RES; i++) {
		ch->r_io[i].res = res;
		ch->r_io[i].offset = i;	/* unused idx -> its own slot */
	}
	/* the registers the code under test touches */
	ch->r_io[ATA_DATA].offset = RO_DATA;
	ch->r_io[ATA_FEATURE].offset = RO_FEATURE;
	ch->r_io[ATA_ERROR].offset = RO_FEATURE;
	ch->r_io[ATA_COUNT].offset = RO_COUNT;
	ch->r_io[ATA_IREASON].offset = RO_COUNT;
	ch->r_io[ATA_SECTOR].offset = RO_SECTOR;
	ch->r_io[ATA_CYL_LSB].offset = RO_CYLL;
	ch->r_io[ATA_CYL_MSB].offset = RO_CYLM;
	ch->r_io[ATA_DRIVE].offset = RO_DRIVE;
	ch->r_io[ATA_COMMAND].offset = RO_COMMAND;
	ch->r_io[ATA_STATUS].offset = RO_STATUS;
	ch->r_io[ATA_ALTSTAT].offset = RO_ALTSTAT;
	ch->r_io[ATA_CONTROL].offset = RO_CONTROL;
}

/* ------------------------------------------------------------------ */
/* IDENTIFY-serving hw.begin_transaction stub -- the DF-2489 seam.    */
/* IDENTIFY is served from the crafted image; data READs are served   */
/* as zeros; control commands succeed.  Used ONLY for the attach-time */
/* traffic of the funnel tests; the taskfile math under test always   */
/* runs in the REAL (kernel / verbatim copy) hw chain.                */
/* ------------------------------------------------------------------ */
static struct ata_params rig_params;	/* desired FINAL parsed words */
static u_int8_t rig_identify_img[512];
static int rig_identify_served, rig_reads_served, rig_ctl_served;

static void
rig_swap_str(u_int8_t *dst, const char *src, int len)
{
	int i;

	for (i = 0; i < len; i += 2) {
		if (i + 1 < len) {
			dst[i] = src[i + 1];
			dst[i + 1] = src[i];
		} else {
			dst[i] = src[i];
		}
	}
}

/* funnel variants */
#define FV_SANE		0	/* word53 54-58 valid, current 16/63 */
#define FV_CSECT0	1	/* current_heads=16 current_sectors=0 */
#define FV_CHEAD0	2	/* current_heads=0  current_sectors=63 */

static void
rig_build_params(int variant)
{
	bzero(&rig_params, sizeof(rig_params));
	rig_params.config = 0x0040;		/* non-ATAPI direct disk */
	rig_params.cylinders = 16383;		/* word 1 */
	rig_params.heads = 16;			/* word 3 (raw) */
	rig_params.sectors = 63;		/* word 6 (raw) */
	/* word 53: words 54-58 valid -> the tf_write current_* branch */
	rig_params.atavalid = ATA_FLAG_54_58 | ATA_FLAG_64_70 | ATA_FLAG_88;
	rig_params.current_cylinders = 16383;	/* word 54 */
	rig_params.current_heads = (variant == FV_CHEAD0) ? 0 : 16;
	rig_params.current_sectors = (variant == FV_CSECT0) ? 0 : 63;
	rig_params.current_size_1 = 0x0000;	/* words 57-58 */
	rig_params.current_size_2 = 0x0080;
	rig_params.apiomodes = 0x0002;		/* PIO4 */
	rig_params.sectors_intr = 16;		/* word 47 */
	rig_params.version_major = 0x0000;	/* word 80: INVALID ->
						 * !ad_version(0) ->
						 * ATA_D_USE_CHS gate,
						 * ata-disk.c:485-486 */
	rig_params.lba_size_1 = 0;		/* word 60/61 = 0: the other
						 * half of the CHS gate */
	rig_params.lba_size_2 = 0;
	bcopy("DF2491SERIAL00000001", rig_params.serial, 20);
	bcopy(variant == FV_SANE ? "DF2491 SANE GEOM DISK" :
	      "DF2491 ZEROGEOM DISK", rig_params.model, 20);
	bcopy("FW000001", rig_params.revision, 8);
}

static void
rig_make_identify(void)
{
	size_t off;

	bzero(rig_identify_img, sizeof(rig_identify_img));
	bcopy(&rig_params, rig_identify_img,
	      umin(sizeof(rig_params), sizeof(rig_identify_img)));

	off = __offsetof(struct ata_params, serial);
	rig_swap_str(rig_identify_img + off, "DF2491SERIAL00000001", 20);
	off = __offsetof(struct ata_params, revision);
	rig_swap_str(rig_identify_img + off, "FW000001", 8);
	off = __offsetof(struct ata_params, model);
	rig_swap_str(rig_identify_img + off,
		     (const char *)&rig_params.model, 40);
}

static int
rig_stub_begin(struct ata_request *request)
{
	if (request->flags & ATA_R_READ) {
		if (request->u.ata.command == ATA_ATA_IDENTIFY) {
			bcopy(rig_identify_img, request->data,
			      umin(request->bytecount,
				   sizeof(rig_identify_img)));
			rig_identify_served++;
		} else {
			bzero(request->data, request->bytecount);
			rig_reads_served++;
		}
		request->donecount = request->bytecount;
	} else {
		rig_ctl_served++;
	}
	request->status = ATA_S_READY;
	request->error = 0;
	request->result = 0;
	return ATA_OP_FINISHED;
}

/* ------------------------------------------------------------------ */
/* fabricated channel kobj ops: the ata_if defaults (ata_if.m)        */
/* ------------------------------------------------------------------ */
static int
rig_ata_locking(device_t dev, int mode)
{
	struct ata_channel *ch = device_get_softc(dev);

	return ch->unit;	/* interface DEFAULT ata_null_locking */
}

static void
rig_ata_setmode(device_t parent, device_t dev)
{
	struct ata_device *atadev = device_get_softc(dev);

	atadev->mode = ata_limit_mode(dev, atadev->mode, ATA_PIO_MAX);
}

static struct kobj_method rig_ata_methods[] = {
	{ &ata_setmode_desc,	(kobjop_t)rig_ata_setmode },
	{ &ata_locking_desc,	(kobjop_t)rig_ata_locking },
	{ NULL,			NULL }
};
static struct kobj_class rig_ata_class = {
	"dfrig2491_atach", rig_ata_methods, 0, NULL, 0, NULL
};
static struct kobj_ops rig_ata_ops;

static struct kobj_method rig_null_methods[] = {
	{ NULL, NULL }
};
static struct kobj_class rig_null_class = {
	"dfrig2491_null", rig_null_methods, 0, NULL, 0, NULL
};
static struct kobj_ops rig_null_ops;
static struct kobj_method rig_dummy_method = { NULL, NULL };

/* ------------------------------------------------------------------ */
/* fabricated device graphs                                            */
/*   channel A: kernel's own hw chain (ata_generic_hw)                 */
/*   channel B: verbatim rig copy hw chain                             */
/*   channel A doubles as the funnel channel (IDENTIFY via stub begin, */
/*   swapped to the real chain before the first I/O)                   */
/* ------------------------------------------------------------------ */
static struct bsd_device fake_bus_dev, fake_pci_dev;
static struct bsd_device fake_ch_dev_a, fake_ata_dev_a;
static struct bsd_device fake_ch_dev_b, fake_ata_dev_b;
static struct ata_channel fake_ch_a, fake_ch_b;
static struct ata_device fake_atadev_a, fake_atadev_b;
static char a_nameunit[] = "ata9a", b_nameunit[] = "ata9b";
static char ai_nameunit[] = "ad9a", bi_nameunit[] = "ad9b";
static char pci_nameunit[] = "pcid9";

static int
rig_kobj_init(void)
{
	int i;
	devclass_t ata_dc = devclass_find("ata");

	if (ata_dc == NULL) {
		kprintf("dfrig2491: devclass \"ata\" not found\n");
		return ENXIO;
	}
	for (i = 0; i < KOBJ_CACHE_SIZE; i++) {
		rig_ata_ops.cache[i] = &rig_dummy_method;
		rig_null_ops.cache[i] = &rig_dummy_method;
	}
	rig_ata_ops.cls = &rig_ata_class;
	rig_null_ops.cls = &rig_null_class;
	return 0;
}

/* freshen channel A (locks preserved across bzero), graph + hw */
static void
rig_prepare_channel_a(int stub_hw)
{
	struct lock saved_state = fake_ch_a.state_mtx;
	struct lock saved_queue = fake_ch_a.queue_mtx;

	bzero(&fake_ch_a, sizeof(fake_ch_a));
	fake_ch_a.state_mtx = saved_state;
	fake_ch_a.queue_mtx = saved_queue;
	bzero(&fake_bus_dev, sizeof(fake_bus_dev));
	bzero(&fake_pci_dev, sizeof(fake_pci_dev));
	bzero(&fake_ch_dev_a, sizeof(fake_ch_dev_a));
	bzero(&fake_ata_dev_a, sizeof(fake_ata_dev_a));
	bzero(&fake_atadev_a, sizeof(fake_atadev_a));

	fake_bus_dev.ops = (kobj_ops_t)&rig_null_ops;
	fake_pci_dev.ops = (kobj_ops_t)&rig_null_ops;
	fake_ch_dev_a.ops = (kobj_ops_t)&rig_ata_ops;

	fake_pci_dev.parent = &fake_bus_dev;
	fake_pci_dev.devclass = NULL;
	fake_pci_dev.nameunit = pci_nameunit;
	TAILQ_INIT(&fake_pci_dev.children);

	fake_ch_dev_a.parent = &fake_pci_dev;
	fake_ch_dev_a.softc = &fake_ch_a;
	fake_ch_dev_a.devclass = devclass_find("ata");
	fake_ch_dev_a.unit = 0;
	fake_ch_dev_a.nameunit = a_nameunit;
	TAILQ_INIT(&fake_ch_dev_a.children);

	fake_ata_dev_a.parent = &fake_ch_dev_a;
	fake_ata_dev_a.softc = &fake_atadev_a;
	fake_ata_dev_a.nameunit = ai_nameunit;
	TAILQ_INIT(&fake_ata_dev_a.children);

	fake_ch_a.dev = &fake_ch_dev_a;
	fake_ch_a.unit = 0;
	fake_ch_a.devices = ATA_ATA_MASTER;
	fake_ch_a.dma = NULL;			/* PIO-only channel */
	if (stub_hw) {
		/* attach-time seam: IDENTIFY/reads served by the stub */
		fake_ch_a.state = ATA_IDLE;
		fake_ch_a.hw.begin_transaction = rig_stub_begin;
	} else {
		/* the kernel's OWN real chain */
		fake_ch_a.state = ATA_STALL_QUEUE;
		ata_generic_hw(&fake_ch_dev_a);
	}
	ata_queue_init(&fake_ch_a);		/* PRODUCTION */
	rig_regfile_init(rig_regs_a);
	rig_channel_regs(&fake_ch_a, rig_regs_a, &rig_res_a);

	fake_atadev_a.dev = &fake_ata_dev_a;
	fake_atadev_a.unit = ATA_MASTER;
	fake_atadev_a.mode = ATA_PIO_MAX;
	fake_atadev_a.max_iosize = DEV_BSIZE;
}

static void
rig_prepare_channel_b(void)
{
	struct lock saved_state = fake_ch_b.state_mtx;
	struct lock saved_queue = fake_ch_b.queue_mtx;

	bzero(&fake_ch_b, sizeof(fake_ch_b));
	fake_ch_b.state_mtx = saved_state;
	fake_ch_b.queue_mtx = saved_queue;
	bzero(&fake_ch_dev_b, sizeof(fake_ch_dev_b));
	bzero(&fake_ata_dev_b, sizeof(fake_ata_dev_b));
	bzero(&fake_atadev_b, sizeof(fake_atadev_b));

	fake_ch_dev_b.ops = (kobj_ops_t)&rig_ata_ops;
	fake_ch_dev_b.parent = &fake_pci_dev;
	fake_ch_dev_b.softc = &fake_ch_b;
	fake_ch_dev_b.devclass = devclass_find("ata");
	fake_ch_dev_b.unit = 1;
	fake_ch_dev_b.nameunit = b_nameunit;
	TAILQ_INIT(&fake_ch_dev_b.children);

	fake_ata_dev_b.parent = &fake_ch_dev_b;
	fake_ata_dev_b.softc = &fake_atadev_b;
	fake_ata_dev_b.nameunit = bi_nameunit;
	TAILQ_INIT(&fake_ata_dev_b.children);

	fake_ch_b.dev = &fake_ch_dev_b;
	fake_ch_b.unit = 1;
	fake_ch_b.devices = ATA_ATA_MASTER;
	fake_ch_b.dma = NULL;
	fake_ch_b.state = ATA_STALL_QUEUE;
	rig9v_ata_generic_hw(&fake_ch_dev_b);	/* VERBATIM rig copy */
	ata_queue_init(&fake_ch_b);		/* PRODUCTION */
	rig_regfile_init(rig_regs_b);
	rig_channel_regs(&fake_ch_b, rig_regs_b, &rig_res_b);

	fake_atadev_b.dev = &fake_ata_dev_b;
	fake_atadev_b.unit = ATA_MASTER;
	fake_atadev_b.mode = ATA_PIO_MAX;
	fake_atadev_b.max_iosize = DEV_BSIZE;
}

/* ------------------------------------------------------------------ */
/* geometry variants for the direct tests (installed straight into the */
/* fabricated atadev->param, exactly what the kernel's own            */
/* ata_getparam parse of the crafted IDENTIFY leaves)                 */
/* ------------------------------------------------------------------ */
#define GV_SANE		0	/* 54-58 valid, current 16/63 */
#define GV_CSECT0	1	/* 54-58 valid, current 16/0  -> :783 */
#define GV_CHEAD0	2	/* 54-58 valid, current 0/63  -> :785 */
#define GV_LBA		3	/* no CHS mode: LBA path */
#define GV_RAWSECT0	4	/* no 54-58 flag, raw 16/0    -> :783 */

static const char *rig_geom_name[] = {
	"CHS sane (current 16/63)",
	"CHS crafted current_sectors=0 (word 56)",
	"CHS crafted current_heads=0 (word 55)",
	"LBA mode (no ATA_D_USE_CHS)",
	"CHS crafted raw sectors=0 (word 6, no word53 bit)",
};

static void
rig_install_geom(struct ata_device *atadev, int variant)
{
	bzero(&atadev->param, sizeof(atadev->param));
	atadev->param.config = 0x0040;
	atadev->param.cylinders = 16383;
	atadev->param.heads = 16;
	atadev->param.sectors = (variant == GV_RAWSECT0) ? 0 : 63;
	atadev->param.apiomodes = 0x0002;
	if (variant != GV_RAWSECT0)
		atadev->param.atavalid = ATA_FLAG_54_58 |
					 ATA_FLAG_64_70 | ATA_FLAG_88;
	atadev->param.current_cylinders = 16383;
	atadev->param.current_heads = (variant == GV_CHEAD0) ? 0 : 16;
	atadev->param.current_sectors = (variant == GV_CSECT0) ? 0 : 63;
	atadev->param.version_major = 0x0000;	/* CHS gate input */
	atadev->param.lba_size_1 = 0;
	atadev->param.lba_size_2 = 0;

	atadev->flags = 0;
	if (variant != GV_LBA)
		atadev->flags |= ATA_D_USE_CHS;	/* what the kernel's own
						 * ad_get_geometry:485-486
						 * sets for this IDENTIFY */
}

/* ------------------------------------------------------------------ */
/* the "device": completes requests whose command was issued by the   */
/* real begin_transaction (plays the drive raising its interrupt)     */
/* ------------------------------------------------------------------ */
static struct thread *rig_irq_td;
static volatile int rig_irq_armed;
static struct ata_channel *rig_irq_ch;
static char rig_irq_spin;

static void
rig_irq_thread(void *arg)
{
	int spins;

	for (;;) {
		tsleep(&rig_irq_armed, 0, "dfrig2491irq", 0);
		spins = 0;
		while (rig_irq_armed && spins++ < 20000) {
			struct ata_request *rq = rig_irq_ch->running;

			if (rq && (rq->flags & ATA_R_HWCMDQUEUED)) {
				/* drive reports READY, no error/DRQ:
				 * command accepted */
				ata_interrupt(rig_irq_ch);	/* KERNEL */
				continue;
			}
			tsleep(&rig_irq_spin, 0, "dfrig2491iw", 1);
		}
	}
}

static void
rig_irq_arm(struct ata_channel *ch)
{
	rig_irq_ch = ch;
	rig_irq_armed = 1;
	wakeup_one(&rig_irq_armed);
}

static void
rig_irq_disarm(void)
{
	rig_irq_armed = 0;
}

/* ------------------------------------------------------------------ */
/* expected taskfile math (independent recomputation for controls)     */
/* ------------------------------------------------------------------ */
#define TEST_LBA	0x1234
#define TEST_COUNT	1

/* ------------------------------------------------------------------ */
/* status buffer                                                       */
/* ------------------------------------------------------------------ */
static char rig_status_buf[16384];
static int rig_last_rc;

#define P(fmt, ...) \
	ksnprintf(rig_status_buf + strlen(rig_status_buf), \
		  sizeof(rig_status_buf) - strlen(rig_status_buf), \
		  fmt "\n", ## __VA_ARGS__)

static void
rig_dump_taskfile(const char *tag, u_int8_t *r)
{
	P("%s taskfile: FEATURE=0x%02x COUNT=0x%02x SECTOR=0x%02x "
	  "CYL_LSB=0x%02x CYL_MSB=0x%02x DRIVE=0x%02x COMMAND=0x%02x",
	  tag, r[RO_FEATURE], r[RO_COUNT], r[RO_SECTOR], r[RO_CYLL],
	  r[RO_CYLM], r[RO_DRIVE], r[RO_COMMAND]);
}

static int
rig_expect_chs(const char *tag, u_int8_t *r, u_int16_t heads,
	       u_int16_t sectors)
{
	int exp_sect, exp_cyl, exp_head, exp_drive, ok = 1;

	exp_sect = (TEST_LBA % sectors) + 1;
	exp_cyl = TEST_LBA / ((u_int32_t)sectors * heads);
	exp_head = ((TEST_LBA % ((u_int32_t)sectors * heads)) / sectors) & 0xf;
	exp_drive = ATA_D_IBM | ATA_DEV(0) | exp_head;

	rig_dump_taskfile(tag, r);
	P("%s expected (heads=%u sectors=%u): SECTOR=0x%02x CYL_LSB=0x%02x "
	  "CYL_MSB=0x%02x DRIVE=0x%02x", tag, heads, sectors,
	  exp_sect, exp_cyl & 0xff, (exp_cyl >> 8) & 0xff, exp_drive);
	if (r[RO_SECTOR] != exp_sect) { ok = 0;
		P("%s MISMATCH SECTOR 0x%02x != 0x%02x", tag,
		  r[RO_SECTOR], exp_sect); }
	if (r[RO_CYLL] != (exp_cyl & 0xff)) { ok = 0;
		P("%s MISMATCH CYL_LSB", tag); }
	if (r[RO_CYLM] != ((exp_cyl >> 8) & 0xff)) { ok = 0;
		P("%s MISMATCH CYL_MSB", tag); }
	if (r[RO_DRIVE] != exp_drive) { ok = 0;
		P("%s MISMATCH DRIVE 0x%02x != 0x%02x", tag,
		  r[RO_DRIVE], exp_drive); }
	if (r[RO_COUNT] != TEST_COUNT) { ok = 0;
		P("%s MISMATCH COUNT", tag); }
	if (r[RO_COMMAND] != ATA_READ) { ok = 0;
		P("%s MISMATCH COMMAND 0x%02x != ATA_READ", tag,
		  r[RO_COMMAND]); }
	P("%s CHS taskfile %s", tag, ok ? "CORRECT" : "WRONG");
	return ok;
}

static int
rig_expect_lba(const char *tag, u_int8_t *r)
{
	int ok = 1;
	int exp_drive = ATA_D_IBM | ATA_D_LBA | ATA_DEV(0) |
			((TEST_LBA >> 24) & 0x0f);

	rig_dump_taskfile(tag, r);
	P("%s expected (LBA): SECTOR=0x%02x CYL_LSB=0x%02x CYL_MSB=0x%02x "
	  "DRIVE=0x%02x", tag, TEST_LBA & 0xff, (TEST_LBA >> 8) & 0xff,
	  (TEST_LBA >> 16) & 0xff, exp_drive);
	if (r[RO_SECTOR] != (TEST_LBA & 0xff)) { ok = 0;
		P("%s MISMATCH SECTOR", tag); }
	if (r[RO_CYLL] != ((TEST_LBA >> 8) & 0xff)) { ok = 0;
		P("%s MISMATCH CYL_LSB", tag); }
	if (r[RO_CYLM] != ((TEST_LBA >> 16) & 0xff)) { ok = 0;
		P("%s MISMATCH CYL_MSB", tag); }
	if (r[RO_DRIVE] != exp_drive) { ok = 0;
		P("%s MISMATCH DRIVE 0x%02x != 0x%02x", tag,
		  r[RO_DRIVE], exp_drive); }
	if (!(r[RO_DRIVE] & ATA_D_LBA)) { ok = 0;
		P("%s MISMATCH LBA bit not set", tag); }
	P("%s LBA taskfile %s", tag, ok ? "CORRECT" : "WRONG");
	return ok;
}

/* ------------------------------------------------------------------ */
/* direct dispatch tests: kernel's own chain (which=0) or the verbatim */
/* rig copy (which=1)                                                  */
/* ------------------------------------------------------------------ */
static void
rig_test_direct(int which, int variant, const char *tag)
{
	struct ata_channel *ch = which ? &fake_ch_b : &fake_ch_a;
	struct ata_device *atadev = which ? &fake_atadev_b : &fake_atadev_a;
	u_int8_t *regs = which ? rig_regs_b : rig_regs_a;
	int rc;

	if (which)
		rig_prepare_channel_b();
	else
		rig_prepare_channel_a(0);
	rig_install_geom(atadev, variant);

	P("== DF-2491 %s: %s chain, ata_controlcmd(ATA_READ, lba=0x%04x, "
	  "count=1), variant=%d (%s) ==", tag,
	  which ? "VERBATIM COPY" : "KERNEL OWN", TEST_LBA, variant,
	  rig_geom_name[variant]);
	P("atadev->flags=0x%04x (ATA_D_USE_CHS=%d) param.atavalid=0x%04x "
	  "current_heads=%u current_sectors=%u raw heads=%u sectors=%u",
	  atadev->flags, !!(atadev->flags & ATA_D_USE_CHS),
	  atadev->param.atavalid, atadev->param.current_heads,
	  atadev->param.current_sectors, atadev->param.heads,
	  atadev->param.sectors);
	if (variant == GV_CSECT0 || variant == GV_CHEAD0 ||
	    variant == GV_RAWSECT0)
		P("EXPECTING FATAL TRAP 18 (integer divide fault) in "
		  "ata_tf_write at ata-lowlevel.c:%s",
		  variant == GV_CHEAD0 ? "785 (lba/(sectors*heads))" :
		  "783 (lba%%sectors)");
	else
		P("EXPECTING: request completes (result=0), taskfile "
		  "registers written to the register file");

	kprintf("dfrig2491[%s]: firing %s ata_controlcmd() variant=%d "
		"(%s)\n", tag, which ? "verbatim copy" : "kernel own",
		variant, rig_geom_name[variant]);
	rig_irq_arm(ch);
	rc = ata_controlcmd(which ? &fake_ata_dev_b : &fake_ata_dev_a,
			    ATA_READ, 0, TEST_LBA, TEST_COUNT);
	rig_irq_disarm();

	P("ata_controlcmd RETURNED rc=%d (no panic)", rc);
	rig_dump_taskfile(tag, regs);
	if (variant == GV_SANE)
		rig_last_rc = rig_expect_chs(tag, regs, 16, 63) && rc == 0;
	else if (variant == GV_LBA)
		rig_last_rc = rig_expect_lba(tag, regs) && rc == 0;
	else {
		/* on a FIXED kernel the guarded fallback uses the raw
		 * 16/63 words (or the 16/63 sane clamp): the taskfile
		 * must then equal the sane-CHS control values */
		rig_last_rc = rig_expect_chs(tag, regs, 16, 63) && rc == 0;
	}
}

/* ------------------------------------------------------------------ */
/* full production funnel: kernel's own ata_identify -> ad_probe ->   */
/* ad_attach (kernel's own) with the crafted IDENTIFY, then swap the  */
/* hw seam to the kernel's REAL chain and fire the first I/O          */
/* ------------------------------------------------------------------ */
static void
rig_test_funnel(int variant, const char *tag)
{
	device_t child;
	struct ata_device *atadev;
	struct ad_softc *adp;
	int rc;
	u_int16_t eheads, esectors;

	/* attach phase: IDENTIFY served by the stub seam */
	rig_prepare_channel_a(1);
	rig_build_params(variant == FV_SANE ? FV_SANE :
			 variant == FV_CSECT0 ? FV_CSECT0 : FV_CHEAD0);
	rig_make_identify();
	rig_identify_served = rig_reads_served = rig_ctl_served = 0;

	P("== DF-2491 %s: FULL PRODUCTION FUNNEL, IDENTIFY variant=%d =="
	  "(atavalid=0x%04x current_heads=%u current_sectors=%u raw "
	  "%u/%u/%u version_major=0x%04x lba_size=0)==",
	  tag, variant, rig_params.atavalid, rig_params.current_heads,
	  rig_params.current_sectors, rig_params.cylinders,
	  rig_params.heads, rig_params.sectors,
	  rig_params.version_major);
	P("firing the kernel's own ata_identify(): real getparam parse, "
	  "real newbus child, real ad_probe/ad_attach ...");
	P("EXPECTING: ad_attach SURVIVES (ad_get_geometry guard falls to "
	  "raw 16/63, ata-disk.c:469-480), ATA_D_USE_CHS set (:485-486); "
	  "then first I/O%s",
	  variant == FV_SANE ? " completes cleanly" :
	  variant == FV_CSECT0 ?
	  " -> FATAL TRAP 18 at ata-lowlevel.c:783" :
	  " -> FATAL TRAP 18 at ata-lowlevel.c:785");

	kprintf("dfrig2491[%s]: firing PRODUCTION ata_identify() "
		"(funnel variant=%d)\n", tag, variant);
	ata_identify(&fake_ch_dev_a);		/* PRODUCTION */
	kprintf("dfrig2491[%s]: ata_identify RETURNED\n", tag);

	child = TAILQ_FIRST(&fake_ch_dev_a.children);
	if (!child || !(atadev = device_get_softc(child)) ||
	    !(adp = device_get_ivars(child))) {
		P("FUNNEL FAILED: no ad child attached "
		  "(identify=%d reads=%d ctl=%d)", rig_identify_served,
		  rig_reads_served, rig_ctl_served);
		rig_last_rc = -1;
		return;
	}
	P("attach state (kernel's own): child attached, "
	  "atadev->flags=0x%04x ATA_D_USE_CHS=%d "
	  "param.atavalid=0x%04x current_heads=%u current_sectors=%u; "
	  "adp->heads=%u adp->sectors=%u total_secs=%ju",
	  atadev->flags, !!(atadev->flags & ATA_D_USE_CHS),
	  atadev->param.atavalid, atadev->param.current_heads,
	  atadev->param.current_sectors, adp->heads, adp->sectors,
	  (uintmax_t)adp->total_secs);
	P("hw stats: identify=%d reads=%d ctl=%d", rig_identify_served,
	  rig_reads_served, rig_ctl_served);
	if (!(atadev->flags & ATA_D_USE_CHS)) {
		P("GATE CHECK FAILED: kernel did not set ATA_D_USE_CHS");
		rig_last_rc = -1;
		return;
	}
	P("GATE CHECK OK: the kernel's own ad_get_geometry "
	  "(ata-disk.c:485-486) set ATA_D_USE_CHS for this device");

	/* first-I/O phase: swap the seam to the kernel's REAL chain */
	bzero(&fake_ch_a.hw, sizeof(fake_ch_a.hw));
	fake_ch_a.state = ATA_STALL_QUEUE;
	ata_generic_hw(&fake_ch_dev_a);		/* kernel's own wiring */
	rig_regfile_init(rig_regs_a);

	kprintf("dfrig2491[%s]: hw seam swapped to the REAL kernel chain; "
		"firing FIRST I/O (ata_controlcmd ATA_READ lba=0x%04x)\n",
		tag, TEST_LBA);
	rig_irq_arm(&fake_ch_a);
	rc = ata_controlcmd(child, ATA_READ, 0, TEST_LBA, TEST_COUNT);
	rig_irq_disarm();

	P("ata_controlcmd RETURNED rc=%d (no panic)", rc);
	if (variant == FV_SANE) {
		eheads = atadev->param.current_heads ?
			 atadev->param.current_heads :
			 atadev->param.heads;
		esectors = atadev->param.current_sectors ?
			 atadev->param.current_sectors :
			 atadev->param.sectors;
		rig_last_rc = rig_expect_chs(tag, rig_regs_a, eheads,
					     esectors) && rc == 0;
	} else {
		rig_last_rc = 0;
	}
}

/* ------------------------------------------------------------------ */
/* scenario dispatch + worker thread (DF-2489 pattern: sysctl posts,  */
/* dedicated kernel thread runs it holding the MP token)              */
/* ------------------------------------------------------------------ */
static int rig_fire_cmd;
static int rig_seq;

static void
rig_run(int cmd)
{
	rig_status_buf[0] = '\0';
	rig_last_rc = -1;
	switch (cmd) {
	case 1:
		rig_test_direct(0, GV_SANE, "go1");	/* control */
		break;
	case 2:
		rig_test_direct(0, GV_CSECT0, "go2");	/* FATAL :783 */
		break;
	case 3:
		rig_test_direct(0, GV_CHEAD0, "go3");	/* FATAL :785 */
		break;
	case 4:
		rig_test_direct(0, GV_LBA, "go4");	/* control */
		break;
	case 5:
		rig_test_direct(1, GV_CSECT0, "go5");	/* FATAL :783 */
		break;
	case 6:
		rig_test_direct(1, GV_SANE, "go6");	/* control */
		break;
	case 7:
		rig_test_funnel(FV_CSECT0, "go7");	/* FATAL :783 */
		break;
	case 8:
		rig_test_funnel(FV_SANE, "go8");	/* control */
		break;
	case 9:
		rig_test_funnel(FV_CHEAD0, "go9");	/* FATAL :785 */
		break;
	case 10:
		rig_test_direct(0, GV_RAWSECT0, "go10");/* FATAL :783 */
		break;
	default:
		P("unknown test %d", cmd);
		rig_last_rc = -2;
		return;
	}
	P("last_rc=%d", rig_last_rc);
}

static void
rig_worker(void *arg)
{
	kprintf("dfrig2491: worker thread ready\n");
	for (;;) {
		int cmd;

		tsleep(&rig_fire_cmd, 0, "dfrig2491w", 0);
		cmd = rig_fire_cmd;
		rig_fire_cmd = 0;
		if (cmd == 0)
			continue;
		kprintf("dfrig2491: worker firing scenario %d "
			"(under mp_token, like the boot attach context)\n",
			cmd);
		/* the funnel attaches through the real newbus machinery
		 * (atapicam's identify asserts the MP token), exactly
		 * like the DF-2489 rig */
		get_mplock();
		rig_run(cmd);
		rel_mplock();
		rig_seq++;
		wakeup(&rig_seq);
		kprintf("dfrig2491: scenario %d DONE (seq=%d)\n", cmd,
			rig_seq);
	}
}

static int
rig_go_sysctl(SYSCTL_HANDLER_ARGS)
{
	int val = 0, error;

	error = sysctl_handle_int(oidp, &val, 0, req);
	if (error || !req->newptr)
		return error;
	if (val < 1 || val > 10)
		return EINVAL;
	rig_fire_cmd = val;
	wakeup_one(&rig_fire_cmd);
	return 0;
}

static int
rig_status_sysctl(SYSCTL_HANDLER_ARGS)
{
	return SYSCTL_OUT(req, rig_status_buf, strlen(rig_status_buf) + 1);
}

static int
rig_seq_sysctl(SYSCTL_HANDLER_ARGS)
{
	int v = rig_seq;

	return sysctl_handle_int(oidp, &v, 0, req);
}

static SYSCTL_NODE(_kern, OID_AUTO, dfrig2491, CTLFLAG_RW, 0,
		   "DF-2491 ata_tf_write CHS divide rig");

SYSCTL_PROC(_kern_dfrig2491, OID_AUTO, go, CTLTYPE_INT | CTLFLAG_RW,
	    0, 0, rig_go_sysctl, "I", "fire DF-2491 scenario N");

SYSCTL_PROC(_kern_dfrig2491, OID_AUTO, status, CTLTYPE_STRING |
	    CTLFLAG_RD, 0, 0, rig_status_sysctl, "A",
	    "last DF-2491 scenario output");

SYSCTL_PROC(_kern_dfrig2491, OID_AUTO, seq, CTLTYPE_INT | CTLFLAG_RD,
	    0, 0, rig_seq_sysctl, "I", "completed scenario count");

/* ------------------------------------------------------------------ */
/* module (loaded at BOOT via /boot/loader.conf; runtime kldload GPFs */
/* on this INVARIANTS guest)                                           */
/* ------------------------------------------------------------------ */
static struct thread *rig_td;

static int
dfrig2491_modevent(module_t mod, int type, void *data)
{
	int error;

	switch (type) {
	case MOD_LOAD:
		if (rig_kobj_init())
			return ENXIO;
		lockinit(&fake_ch_a.state_mtx, "dfrig2491_sta", 0,
			 LK_CANRECURSE);
		lockinit(&fake_ch_a.queue_mtx, "dfrig2491_qa", 0,
			 LK_CANRECURSE);
		lockinit(&fake_ch_b.state_mtx, "dfrig2491_stb", 0,
			 LK_CANRECURSE);
		lockinit(&fake_ch_b.queue_mtx, "dfrig2491_qb", 0,
			 LK_CANRECURSE);
		error = kthread_create(rig_worker, NULL, &rig_td,
				       "dfrig2491");
		if (error)
			return error;
		error = kthread_create(rig_irq_thread, NULL, &rig_irq_td,
				       "dfrig2491irq");
		if (error)
			return error;
		kprintf("dfrig2491: loaded (production ata-lowlevel.c "
			"included verbatim; kernel own chain dispatched; "
			"scenarios kern.dfrig2491.go=1..10)\n");
		return 0;
	case MOD_UNLOAD:
		return EBUSY;	/* worker threads never exit */
	default:
		return EOPNOTSUPP;
	}
}

static moduledata_t dfrig2491_mod = { "dfrig2491", dfrig2491_modevent, 0 };

static struct mod_metadata _mod_metadata_dfrig2491 = {
	MDT_STRUCT_VERSION, MDT_MODULE, &dfrig2491_mod, "dfrig2491"
};
DATA_SET(modmetadata_set, _mod_metadata_dfrig2491);
SYSINIT(dfrig2491module, SI_SUB_DRIVERS, SI_ORDER_MIDDLE,
	module_register_init, &dfrig2491_mod);