/*
 * dfrig2489.c -- DF-2489 verification rig
 *                (sys/dev/disk/nata/ata-disk.c:147 and :540)
 *
 * Finding under test:
 *
 *	ad_get_geometry() (ata-disk.c:462-500) copies adp->heads and
 *	adp->sectors straight out of the device-supplied ATA IDENTIFY
 *	words (u_int16_t param.heads / param.sectors, else-branch at
 *	ata-disk.c:477-478; the current_* branch at :469-475 is guarded,
 *	the raw branch is not -- and only the raw branch is taken whenever
 *	word 53 does not report words 54-58 valid, or current_heads /
 *	current_sectors are zero).  A device reporting heads==0 or
 *	sectors==0 (word 3 / word 6) then makes ad_attach() execute
 *
 *	    info.d_ncylinders = adp->total_secs/(adp->heads*adp->sectors);
 *						 ^^ ata-disk.c:147
 *
 *	and, under bootverbose, ad_describe() execute the identical
 *	divisor
 *
 *	    adp->total_secs / (adp->heads * adp->sectors)
 *						 ^^ ata-disk.c:540
 *
 *	-- an integer divide by zero: x86 #DE, "Fatal trap 18: integer
 *	divide fault", at device attach (boot or hot-plug).  Nothing
 *	between ata_getparam() (ata-all.c:590, copies IDENTIFY verbatim,
 *	only requires a printable model byte) and ad_attach() validates
 *	the geometry words.
 *
 * Rig design (established nata verbatim-interposition class, cf.
 * DF-2210 ata-national, DF-249x/DF-2496 nata):
 *
 *  1. FULL PRODUCTION FUNNEL dispatch (tests 1/2/3): the real
 *     kernel's own ata_identify() (global, ata-all.c:683) is called
 *     on a fabricated ata channel.  It runs the REAL kmalloc'ed
 *     ata_device, the REAL ata_getparam() (ATA_ATA_IDENTIFY served by
 *     a hw.begin_transaction stub -- exactly the seam a QEMU IDE
 *     disk answers), the REAL le16toh/bswap/btrim string parse, the
 *     REAL newbus child creation (device_add_child on a channel
 *     carrying the real "ata" devclass), bus_generic_attach, the REAL
 *     ad_probe() and -- the point -- the RUNNING KERNEL'S OWN
 *     ad_attach() (nm: t ad_attach), whose inlined ad_get_geometry
 *     computes the divisor from the crafted IDENTIFY words and whose
 *     ata-disk.c:147 idiv traps.  Everything from the IDENTIFY bytes
 *     to the fault is shipped production code; the only fabrications
 *     are the channel HW stubs (begin_transaction / the ata_if setmode
 *     and locking defaults / the device graph), none of which sit
 *     between the cited division and its device-supplied input.
 *
 *  2. VERBATIM COPY dispatch (tests 4/5): the production
 *     sys/dev/disk/nata/ata-disk.c is #included UNMODIFIED (module
 *     plumbing neutralized, the unused global ad_devclass renamed --
 *     build.sh sha256-checks the file against the audited sys/ tree),
 *     so this KLD also carries its own byte-identical ad_attach /
 *     ad_describe / ad_get_geometry, driven directly on the same
 *     fabricated post-getparam state (DF-2496 test22 shape).
 *
 *  3. DESCRIBE variant (test 5): fabricated ad_softc with heads==0
 *     (exactly what production ad_get_geometry leaves after a
 *     heads==0 IDENTIFY) + bootverbose=1 -> ad_describe():540 traps;
 *     on the stock kernel this site is shadowed by :147 during
 *     attach, but it is the SAME unvalidated divisor and must be
 *     fixed with it (a :147-only fix still panics every bootverbose
 *     boot).
 *
 * Fabricated machine state (what a real ata channel looks like to
 * nata when a PIO-only disk sits on it):
 *
 *   rig bus (kobj null class) <- fake_pci_dev (devclass NULL)
 *        <- fake_ch_dev (struct ata_channel softc, devclass "ata",
 *           state=ATA_IDLE, dma=NULL [PIO-only channel, legitimate,
 *           ata-dma.c:153], hw.begin_transaction=rig stub,
 *           production ata_queue_init() locks/queue)
 *        <- [ata_identify creates the REAL ad child here]
 *
 * Tests (sysctl kern.dfrig2489.go=N; read kern.dfrig2489.status):
 *
 *   1  CONTROL, full funnel: crafted IDENTIFY with real geometry
 *      (16383/16/63, LBA 0x2000000) -> kernel own ad_attach runs to
 *      completion: ad0 created, describe line + bootverbose
 *      "[33230C/16H/63S]" line, subdisk child attaches, RAID
 *      metadata scan (reads served as zeros) finds nothing.  Expect
 *      RETURNED, no panic.
 *   2  THE FINDING, full funnel, heads==0 (IDENTIFY word 3 = 0):
 *      kernel own ad_attach -> trap 18 at ata-disk.c:147.
 *   3  THE FINDING, full funnel, sectors==0 (word 6 = 0): same trap.
 *   4  THE FINDING, verbatim rig copy ad_attach, heads==0: identical
 *      trap in the rig-compiled production copy (byte-identity).
 *   5  THE FINDING, verbatim rig copy ad_describe, heads==0 ad_softc,
 *      bootverbose: trap 18 at ata-disk.c:540.
 *
 *   2/3/4/5 are FATAL (one per boot); proof = serial console panic.
 *   Module is loaded AT BOOT via /boot/loader.conf (on this INVARIANTS
 *   guest runtime kldload GPFs); dispatch at runtime via sysctl onto a
 *   dedicated worker thread that holds the MP token (the boot-attach
 *   context -- the built-in atapicam driver's attach asserts it).
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <sys/bus.h>
#include <sys/bus_private.h>	/* struct bsd_device layout */
#include <sys/conf.h>		/* struct cdev / si_drv1 */
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/device.h>		/* struct dev_strategy_args */
#include <sys/callout.h>
#include <sys/lock.h>
#include <sys/kthread.h>
#include <sys/mplock2.h>		/* get_mplock: boot-attach context */

#include <sys/nata.h>

/* NOTE: nata headers (ata-all.h/ata-pci.h) have NO include guards;
 * the production ata-disk.c below includes ata-all.h itself, so all
 * further nata includes happen AFTER it, exactly once. */

/* ------------------------------------------------------------------ */
/* neutralize the production file's module/driver plumbing so this KLD */
/* registers nothing that clashes with the built-in ad driver, and     */
/* rename its (now unreferenced) global ad_devclass definition.        */
/* The functions under test keep their production names and bodies.    */
/* ------------------------------------------------------------------ */
#undef DRIVER_MODULE
#undef DECLARE_MODULE
#undef MODULE_DEPEND
#undef MODULE_VERSION
#define DRIVER_MODULE(name, busname, driver, devclass, evh, arg)	/* rig */
#define DECLARE_MODULE(name, data, sub, order)				/* rig */
#define MODULE_DEPEND(name, m, v1, v2, v3)				/* rig */
#define MODULE_VERSION(m, v)						/* rig */

/* rename the production file's global devclass definition (its only
 * reference, DRIVER_MODULE(ad, ata, ...), is neutralized above); the
 * extern in ata-disk.h and the definition in ata-disk.c rename in
 * lock inside this TU, avoiding a duplicate global against the
 * built-in kernel copy */
#define ad_devclass rig_unused_ad_devclass
#include "ata-disk.c"		/* <-- UNMODIFIED production code */
#undef ad_devclass

#include "ata-pci.h"		/* struct ata_pci_controller (once) */

#undef DRIVER_MODULE
#undef DECLARE_MODULE
#undef MODULE_DEPEND
#undef MODULE_VERSION

/* ------------------------------------------------------------------ */
/* channel HW seam: what the drive answers.  IDENTIFY is served from   */
/* the crafted image; data READs are served as zeros (a plain disk     */
/* with no ataraid metadata); control commands succeed.  Everything    */
/* else (queueing, locking, completion, taskqueue, retries, status     */
/* decoding) is the kernel's own ata_queue_request()/ata_start().      */
/* ------------------------------------------------------------------ */
static struct ata_params rig_params;	/* desired FINAL parsed words */
static u_int8_t rig_identify_img[512];	/* device-side IDENTIFY image */
static int rig_identify_served, rig_reads_served, rig_ctl_served;

/* device delivers ATA strings with each byte pair swapped;
 * ata_getparam() bswap()s them back (ata-all.c:631-635) */
static void
rig_swap_str(u_int8_t *dst, const char *src, int len)
{
	int i;

	for (i = 0; i < len; i += 2) {
		if (i + 1 < len) {
			dst[i] = src[i + 1];
			dst[i + 1] = src[i];
		} else {
			dst[i] = src[i];
		}
	}
}

/* variant: 0 = sane geometry (control), 1 = heads==0, 2 = sectors==0 */
static void
rig_build_params(int variant)
{

	bzero(&rig_params, sizeof(rig_params));
	rig_params.config = 0x0040;		/* non-ATAPI direct disk */
	rig_params.cylinders = 16383;
	rig_params.heads = (variant == 1) ? 0 : 16;	/* IDENTIFY word 3 */
	rig_params.sectors = (variant == 2) ? 0 : 63;	/* IDENTIFY word 6 */
	/* word 53: words 64-70 & 88 valid, 54-58 NOT valid -> forces the
	 * unguarded else-branch of ad_get_geometry (ata-disk.c:476-480) */
	rig_params.atavalid = ATA_FLAG_64_70 | ATA_FLAG_88;
	rig_params.apiomodes = 0x0002;		/* PIO4 */
	rig_params.sectors_intr = 16;		/* word 47: 16 secs/int */
	rig_params.lba_size_1 = 0x0000;		/* word 60/61 = 0x2000000 */
	rig_params.lba_size_2 = 0x0200;
	rig_params.version_major = 0x007e;	/* ATA/ATAPI-6 */
	bcopy("DF2489SERIAL00000001", rig_params.serial, 20);
	bcopy(variant == 0 ? "DF2489 SANE GEOMETRY" :
	      "DF2489 ZEROGEOM DISK", rig_params.model, 20);
	bcopy("FW000001", rig_params.revision, 8);
}

static void
rig_make_identify(void)
{
	size_t off;

	bzero(rig_identify_img, sizeof(rig_identify_img));
	bcopy(&rig_params, rig_identify_img,
	      umin(sizeof(rig_params), sizeof(rig_identify_img)));

	/* strings pairwise swapped, the way the wire delivers them */
	off = __offsetof(struct ata_params, serial);
	rig_swap_str(rig_identify_img + off, "DF2489SERIAL00000001", 20);
	off = __offsetof(struct ata_params, revision);
	rig_swap_str(rig_identify_img + off, "FW000001", 8);
	off = __offsetof(struct ata_params, model);
	rig_swap_str(rig_identify_img + off,
		     (const char *)&rig_params.model, 40);
}

static int
rig_begin_transaction(struct ata_request *request)
{
	kprintf("dfrig2489:   hw: cmd=0x%02x feature=0x%02x count=%u "
		"lba=%ju bytecount=%u flags=0x%x\n",
		request->u.ata.command, request->u.ata.feature,
		request->u.ata.count, (uintmax_t)request->u.ata.lba,
		request->bytecount, request->flags);
	if (request->flags & ATA_R_READ) {
		if (request->u.ata.command == ATA_ATA_IDENTIFY) {
			bcopy(rig_identify_img, request->data,
			      umin(request->bytecount,
				   sizeof(rig_identify_img)));
			rig_identify_served++;
		} else {
			/* plain-disk data: zeros, no ataraid magic */
			bzero(request->data, request->bytecount);
			rig_reads_served++;
		}
		request->donecount = request->bytecount;
	} else {
		rig_ctl_served++;	/* SETFEATURES / SET_MULTI accepted */
	}
	request->status = ATA_S_READY;		/* 0x40, no error bits */
	request->error = 0;
	request->result = 0;
	return ATA_OP_FINISHED;
}

/* ------------------------------------------------------------------ */
/* fabricated channel kobj ops: the ata_if defaults, exactly what a    */
/* channel without a chipset driver gets (ata_if.m CODE blocks)        */
/* ------------------------------------------------------------------ */
static int
rig_ata_locking(device_t dev, int mode)
{
	struct ata_channel *ch = device_get_softc(dev);

	return ch->unit;	/* interface DEFAULT ata_null_locking */
}

static void
rig_ata_setmode(device_t parent, device_t dev)
{
	struct ata_device *atadev = device_get_softc(dev);

	/* interface DEFAULT ata_null_setmode (ata_if.m) */
	atadev->mode = ata_limit_mode(dev, atadev->mode, ATA_PIO_MAX);
	kprintf("dfrig2489: rig ata setmode default -> mode=0x%02x\n",
		atadev->mode);
}

static struct kobj_method rig_ata_methods[] = {
	{ &ata_setmode_desc,	(kobjop_t)rig_ata_setmode },
	{ &ata_locking_desc,	(kobjop_t)rig_ata_locking },
	{ NULL,			NULL }
};
static struct kobj_class rig_ata_class = {
	"dfrig2489_atach", rig_ata_methods, 0, NULL, 0, NULL
};
static struct kobj_ops rig_ata_ops;

/* null class for the fabricated pci/bus devices (never dispatched on
 * these paths; kobj falls back to kobj_error_method) */
static struct kobj_method rig_null_methods[] = {
	{ NULL, NULL }
};
static struct kobj_class rig_null_class = {
	"dfrig2489_null", rig_null_methods, 0, NULL, 0, NULL
};
static struct kobj_ops rig_null_ops;
static struct kobj_method rig_dummy_method = { NULL, NULL };

/* ------------------------------------------------------------------ */
/* fabricated device graph                                              */
/* ------------------------------------------------------------------ */
static struct bsd_device fake_bus_dev, fake_pci_dev, fake_ch_dev;
static struct ata_pci_controller fake_ctlr;
static struct ata_channel fake_ch;
static char ch_nameunit[] = "ata9f";
static char pci_nameunit[] = "pcid9f";

static int
rig_prepare(int variant)
{
	int i;
	devclass_t ata_dc = devclass_find("ata");

	if (ata_dc == NULL) {
		kprintf("dfrig2489: devclass \"ata\" not found\n");
		return ENXIO;
	}

	{
		/* preserve the lockinit'ed locks across the reset */
		struct lock saved_state = fake_ch.state_mtx;
		struct lock saved_queue = fake_ch.queue_mtx;

		bzero(&fake_ch, sizeof(fake_ch));
		fake_ch.state_mtx = saved_state;
		fake_ch.queue_mtx = saved_queue;
	}
	bzero(&fake_bus_dev, sizeof(fake_bus_dev));
	bzero(&fake_pci_dev, sizeof(fake_pci_dev));
	bzero(&fake_ch_dev, sizeof(fake_ch_dev));
	bzero(&fake_ctlr, sizeof(fake_ctlr));

	/* kobj ops caches pre-filled so kernel kobj lookups through these
	 * devices resolve via the class method lists (DF-2210 pattern) */
	for (i = 0; i < KOBJ_CACHE_SIZE; i++) {
		rig_ata_ops.cache[i] = &rig_dummy_method;
		rig_null_ops.cache[i] = &rig_dummy_method;
	}
	rig_ata_ops.cls = &rig_ata_class;
	rig_null_ops.cls = &rig_null_class;

	fake_bus_dev.ops = (kobj_ops_t)&rig_null_ops;
	fake_pci_dev.ops = (kobj_ops_t)&rig_null_ops;
	fake_ch_dev.ops = (kobj_ops_t)&rig_ata_ops;

	fake_pci_dev.parent = &fake_bus_dev;
	fake_pci_dev.softc = &fake_ctlr;
	fake_pci_dev.devclass = NULL;	/* not pci/atapci: generic scan */
	fake_pci_dev.nameunit = pci_nameunit;
	TAILQ_INIT(&fake_pci_dev.children);

	fake_ch_dev.parent = &fake_pci_dev;
	fake_ch_dev.softc = &fake_ch;
	fake_ch_dev.devclass = ata_dc;	/* real "ata" bus devclass */
	fake_ch_dev.unit = 0;
	fake_ch_dev.nameunit = ch_nameunit;
	TAILQ_INIT(&fake_ch_dev.children);

	/* the channel a PIO-only disk lives on */
	fake_ch.dev = &fake_ch_dev;
	fake_ch.unit = 0;
	fake_ch.devices = ATA_ATA_MASTER;	/* disk present on master */
	fake_ch.state = ATA_IDLE;			/* normal channel state */
	fake_ch.dma = NULL;	/* PIO-only channel (ata-dma.c:153) */
	fake_ch.hw.begin_transaction = rig_begin_transaction;
	ata_queue_init(&fake_ch);			/* PRODUCTION */

	/* serve the crafted IDENTIFY */
	rig_build_params(variant);
	rig_make_identify();
	rig_identify_served = rig_reads_served = rig_ctl_served = 0;

	kprintf("dfrig2489: channel prepared: variant=%d "
		"(params.heads=%u params.sectors=%u cylinders=%u "
		"atavalid=0x%04x lba_size=0x%08x)\n",
		variant, rig_params.heads, rig_params.sectors,
		rig_params.cylinders, rig_params.atavalid,
		(rig_params.lba_size_2 << 16) | rig_params.lba_size_1);
	return 0;
}

/* ------------------------------------------------------------------ */
/* status buffer                                                        */
/* ------------------------------------------------------------------ */
static char rig_status_buf[8192];
static int rig_last_rc;

#define P(fmt, ...) \
	ksnprintf(rig_status_buf + strlen(rig_status_buf), \
		  sizeof(rig_status_buf) - strlen(rig_status_buf), \
		  fmt "\n", ## __VA_ARGS__)

/* ------------------------------------------------------------------ */
/* tests 1/2/3: the FULL production funnel -- kernel's own identify,   */
/* getparam parse, newbus child, ad_probe, ad_attach                    */
/* ------------------------------------------------------------------ */
static int
rig_test_funnel(int variant, const char *tag)
{
	int saved_bootverbose = bootverbose;
	int rc;

	rc = rig_prepare(variant);
	if (rc)
		return rc;

	P("== DF-2489 %s: PRODUCTION ata_identify() funnel -> "
	  "RUNNING KERNEL own ad_attach, variant=%d (%s) ==",
	  tag, variant,
	  variant == 0 ? "SANE 16383/16/63" :
	  variant == 1 ? "CRAFTED heads=0 (word 3)" :
			 "CRAFTED sectors=0 (word 6)");
	P("firing ata_identify(%p): real getparam parses the crafted "
	  "IDENTIFY, real newbus child, real ad_probe/ad_attach ...",
	  &fake_ch_dev);
	P("EXPECT%s: %s", variant == 0 ? "ED CLEAN ATTACH" :
	  "ING FATAL TRAP 18 (integer divide fault) at ata-disk.c:147",
	  variant == 0 ? "ad0 created, describe + bootverbose "
	  "[C/H/S] lines on console" :
	  "ad_attach:147 total_secs/(heads*sectors) with divisor 0");

	kprintf("dfrig2489[%s]: firing PRODUCTION ata_identify() "
		"(variant=%d) -- kernel own ad_attach follows\n", tag,
		variant);
	bootverbose = 1;	/* also exercise the :537-543 path */
	ata_identify(&fake_ch_dev);		/* PRODUCTION */
	bootverbose = saved_bootverbose;

	P("ata_identify returned (no panic); IDENTIFY served=%d "
	  "data reads served=%d control cmds served=%d",
	  rig_identify_served, rig_reads_served, rig_ctl_served);
	if (variant == 0) {
		struct ata_device *atadev;
		device_t child;

		child = TAILQ_FIRST(&fake_ch_dev.children);
		atadev = child ? device_get_softc(child) : NULL;
		if (child && atadev && device_get_ivars(child)) {
			struct ad_softc *adp = device_get_ivars(child);

			P("control attach state: child=%p ad_softc=%p "
			  "heads=%u sectors=%u total_secs=%ju "
			  "(disk_info d_ncylinders would be %ju)",
			  child, adp, adp->heads, adp->sectors,
			  (uintmax_t)adp->total_secs,
			  (uintmax_t)(adp->total_secs /
				      (adp->heads * adp->sectors)));
			rig_last_rc = (adp->heads == 16 &&
				       adp->sectors == 63);
		} else {
			P("no ad child attached?!");
			rig_last_rc = 0;
		}
	}
	return 0;
}

/* ------------------------------------------------------------------ */
/* test 4: verbatim rig copy ad_attach on the same post-getparam state */
/* ------------------------------------------------------------------ */
static struct bsd_device fake_ata_dev_b;
static struct ata_device fake_atadev_b;
static char b_nameunit[] = "ad9b";

static void
rig_test_verbatim_attach(void)
{
	devclass_t ad_dc = devclass_find("ad");

	if (rig_prepare(1) == 0) {
		bzero(&fake_ata_dev_b, sizeof(fake_ata_dev_b));
		bzero(&fake_atadev_b, sizeof(fake_atadev_b));

		fake_ata_dev_b.parent = &fake_ch_dev;
		fake_ata_dev_b.softc = &fake_atadev_b;
		fake_ata_dev_b.ivars = NULL;	/* virgin disk */
		fake_ata_dev_b.devclass = ad_dc;
		fake_ata_dev_b.unit = 7;
		fake_ata_dev_b.nameunit = b_nameunit;
		TAILQ_INIT(&fake_ata_dev_b.children);

		/* exactly what production ata_getparam leaves after the
		 * crafted heads==0 IDENTIFY (rig_params, variant 1) */
		fake_atadev_b.dev = &fake_ata_dev_b;
		fake_atadev_b.unit = ATA_MASTER;
		fake_atadev_b.mode = ATA_PIO_MAX;
		bcopy(&rig_params, &fake_atadev_b.param,
		      sizeof(struct ata_params));
	}

	P("== DF-2489 go4: VERBATIM #included copy ad_attach(), "
	  "heads=0 ==");
	P("ad_softc state entering ad_attach (from ad_get_geometry "
	  "else-branch, ata-disk.c:477-478): heads=0 sectors=63 "
	  "total_secs=0x2000000 (lba_size won, :489-490)");
	P("EXPECTING FATAL TRAP 18 at ata-disk.c:147 "
	  "`total_secs/(heads*sectors)' -- panic follows on serial");

	kprintf("dfrig2489[go4]: dispatching VERBATIM COPY ad_attach() "
		"(heads=0) -- divisor heads*sectors=0\n");
	ad_attach(&fake_ata_dev_b);		/* PRODUCTION (rig copy) */
	P("SURVIVED -- on a FIXED kernel this is the expected outcome "
	  "(clamp in ad_get_geometry -> no division by zero)");
	rig_last_rc = 0;
}

/* ------------------------------------------------------------------ */
/* test 5: verbatim rig copy ad_describe with a zero-geometry ad_softc */
/* ------------------------------------------------------------------ */
static struct bsd_device fake_ata_dev_c;
static struct ata_device fake_atadev_c;
static struct ad_softc fake_adsc_c;
static char c_nameunit[] = "ad9c";

static void
rig_test_describe(void)
{
	int saved_bootverbose = bootverbose;

	rig_prepare(0);
	bzero(&fake_ata_dev_c, sizeof(fake_ata_dev_c));
	bzero(&fake_atadev_c, sizeof(fake_atadev_c));
	bzero(&fake_adsc_c, sizeof(fake_adsc_c));

	fake_ata_dev_c.parent = &fake_ch_dev;
	fake_ata_dev_c.softc = &fake_atadev_c;
	fake_ata_dev_c.ivars = &fake_adsc_c;
	fake_ata_dev_c.unit = 0;
	fake_ata_dev_c.nameunit = c_nameunit;

	bcopy(&rig_params, &fake_atadev_c.param,
	      sizeof(struct ata_params));
	fake_atadev_c.dev = &fake_ata_dev_c;
	fake_atadev_c.unit = ATA_MASTER;
	fake_atadev_c.mode = ATA_PIO4;

	/* post-ad_get_geometry state of a heads==0 device */
	fake_adsc_c.total_secs = 100000;
	fake_adsc_c.heads = 0;		/* <-- crafted IDENTIFY word 3 */
	fake_adsc_c.sectors = 63;
	fake_adsc_c.num_tags = 0;

	P("== DF-2489 go5: VERBATIM #included copy ad_describe(), "
	  "bootverbose, heads=0 ==");
	P("ad_softc: total_secs=100000 heads=0 sectors=63 -- the state "
	  "ad_get_geometry leaves after a heads==0 IDENTIFY");
	P("EXPECTING FATAL TRAP 18 at ata-disk.c:540 "
	  "`total_secs/(heads*sectors)' in the bootverbose path");

	kprintf("dfrig2489[go5]: dispatching VERBATIM COPY ad_describe() "
		"(bootverbose, heads=0) -- :540 divisor heads*sectors=0\n");
	bootverbose = 1;
	ad_describe(&fake_ata_dev_c);		/* PRODUCTION (rig copy) */
	bootverbose = saved_bootverbose;
	P("SURVIVED -- on a FIXED kernel this is the expected outcome");
	rig_last_rc = 0;
}

/* ------------------------------------------------------------------ */
/* tests 6/7: semi-funnel diagnostics -- replicate ata_identify()'s    */
/* per-device body (ata-all.c:697-716) with the param installed        */
/* directly (fabricated post-getparse state), then run the REAL newbus */
/* probe/attach machinery stepwise to bracket any wedge               */
/* ------------------------------------------------------------------ */
static device_t rig_child;
static struct ata_device *rig_atadev;

static void
rig_test_semi(int variant, int direct_probe, const char *tag)
{
	int saved_bootverbose = bootverbose;

	if (rig_prepare(variant) != 0)
		return;

	rig_atadev = kmalloc(sizeof(struct ata_device), M_ATA,
			     M_INTWAIT | M_ZERO);
	rig_atadev->unit = ATA_MASTER;
	rig_child = device_add_child(&fake_ch_dev, NULL, -1);
	if (rig_child == NULL) {
		P("device_add_child failed");
		kfree(rig_atadev, M_ATA);
		rig_atadev = NULL;
		return;
	}
	device_set_softc(rig_child, rig_atadev);
	device_quiet(rig_child);
	rig_atadev->dev = rig_child;
	rig_atadev->max_iosize = DEV_BSIZE;
	rig_atadev->mode = ATA_PIO_MAX;
	bcopy(&rig_params, &rig_atadev->param, sizeof(struct ata_params));

	P("== DF-2489 %s: semi-funnel diagnostic (variant=%d) ==", tag,
	  variant);
	kprintf("dfrig2489[%s]: child=%p fabricated (ata-all.c:697-716 "
		"body); bus_generic_probe next\n", tag, rig_child);
	bus_generic_probe(&fake_ch_dev);
	kprintf("dfrig2489[%s]: bus_generic_probe RETURNED\n", tag);

	bootverbose = 1;
	if (direct_probe) {
		kprintf("dfrig2489[%s]: device_probe_and_attach(child) "
			"next\n", tag);
		device_probe_and_attach(rig_child);
	} else {
		kprintf("dfrig2489[%s]: bus_generic_attach(channel) "
			"next\n", tag);
		bus_generic_attach(&fake_ch_dev);
	}
	bootverbose = saved_bootverbose;

	kprintf("dfrig2489[%s]: newbus attach machinery RETURNED\n", tag);
	P("newbus attach machinery returned; hw stats: identify=%d "
	  "reads=%d ctl=%d", rig_identify_served, rig_reads_served,
	  rig_ctl_served);
	if (device_get_ivars(rig_child)) {
		struct ad_softc *adp = device_get_ivars(rig_child);

		P("child attached: ad_softc=%p heads=%u sectors=%u "
		  "total_secs=%ju", adp, adp->heads, adp->sectors,
		  (uintmax_t)adp->total_secs);
		rig_last_rc = (adp->heads != 0 && adp->sectors != 0);
	} else {
		P("child did not attach (no ivars)");
		rig_last_rc = 0;
	}
}

/* ------------------------------------------------------------------ */
/* scenario dispatch + worker thread (the sysctl handler only posts    */
/* the command; the scenario runs in a dedicated kernel thread so a    */
/* wedge cannot hold a sysctl/driverf-context lock)                    */
/* ------------------------------------------------------------------ */
static int rig_fire_cmd;	/* protected by being single-writer */
static int rig_seq;

static void
rig_run(int cmd)
{
	rig_status_buf[0] = '\0';
	rig_last_rc = -1;
	switch (cmd) {
	case 1:
		rig_test_funnel(0, "go1");
		break;
	case 2:
		rig_test_funnel(1, "go2");	/* fatal on stock */
		break;
	case 3:
		rig_test_funnel(2, "go3");	/* fatal on stock */
		break;
	case 4:
		rig_test_verbatim_attach();	/* fatal on stock */
		break;
	case 5:
		rig_test_describe();		/* fatal on stock */
		break;
	case 6:
		rig_test_semi(0, 0, "go6");	/* diagnostic, sane */
		break;
	case 7:
		rig_test_semi(0, 1, "go7");	/* diagnostic, sane */
		break;
	default:
		P("unknown test %d", cmd);
		rig_last_rc = -2;
		return;
	}
	P("last_rc=%d", rig_last_rc);
}

static void
rig_worker(void *arg)
{

	kprintf("dfrig2489: worker thread ready\n");
	for (;;) {
		int cmd;

		tsleep(&rig_fire_cmd, 0, "dfrig2489w", 0);
		cmd = rig_fire_cmd;
		rig_fire_cmd = 0;
		if (cmd == 0)
			continue;
		kprintf("dfrig2489: worker firing scenario %d "
			"(under mp_token, like the boot attach context)\n",
			cmd);
		/*
		 * The funnel attaches through the real newbus machinery,
		 * which on this kernel also attaches the built-in
		 * atapicam driver (its identify() adds a child to every
		 * probed channel); xpt_bus_register() asserts the MP
		 * token is held, exactly as the boot-time attach context
		 * holds it.  Take it -- this mirrors, not alters, the
		 * production attach context.
		 */
		get_mplock();
		rig_run(cmd);
		rel_mplock();
		rig_seq++;
		wakeup(&rig_seq);
		kprintf("dfrig2489: scenario %d DONE (seq=%d)\n", cmd,
			rig_seq);
	}
}

static int
rig_go_sysctl(SYSCTL_HANDLER_ARGS)
{
	int val = 0, error;

	error = sysctl_handle_int(oidp, &val, 0, req);
	if (error || !req->newptr)
		return error;

	if (val < 1 || val > 7)
		return EINVAL;
	rig_fire_cmd = val;
	wakeup_one(&rig_fire_cmd);
	return 0;
}

static int
rig_status_sysctl(SYSCTL_HANDLER_ARGS)
{
	return SYSCTL_OUT(req, rig_status_buf, strlen(rig_status_buf) + 1);
}

static int
rig_seq_sysctl(SYSCTL_HANDLER_ARGS)
{
	int v = rig_seq;

	return sysctl_handle_int(oidp, &v, 0, req);
}

static SYSCTL_NODE(_kern, OID_AUTO, dfrig2489, CTLFLAG_RW, 0,
		   "DF-2489 ata-disk zero-geometry rig");

SYSCTL_PROC(_kern_dfrig2489, OID_AUTO, go, CTLTYPE_INT | CTLFLAG_RW,
	    0, 0, rig_go_sysctl, "I", "fire DF-2489 scenario N");

SYSCTL_PROC(_kern_dfrig2489, OID_AUTO, status, CTLTYPE_STRING |
	    CTLFLAG_RD, 0, 0, rig_status_sysctl, "A",
	    "last DF-2489 scenario output");

SYSCTL_PROC(_kern_dfrig2489, OID_AUTO, seq, CTLTYPE_INT | CTLFLAG_RD,
	    0, 0, rig_seq_sysctl, "I", "completed scenario count");

/* ------------------------------------------------------------------ */
/* module (loaded at BOOT via /boot/loader.conf; runtime kldload GPFs  */
/* on this INVARIANTS guest)                                           */
/* ------------------------------------------------------------------ */
static struct thread *rig_td;

static int
dfrig2489_modevent(module_t mod, int type, void *data)
{
	int error;

	switch (type) {
	case MOD_LOAD:
		lockinit(&fake_ch.state_mtx, "dfrig2489_st", 0,
			 LK_CANRECURSE);
		lockinit(&fake_ch.queue_mtx, "dfrig2489_q", 0,
			 LK_CANRECURSE);
		error = kthread_create(rig_worker, NULL, &rig_td,
				       "dfrig2489");
		if (error)
			return error;
		kprintf("dfrig2489: loaded (production ata-disk.c "
			"included verbatim; scenarios "
			"kern.dfrig2489.go=1..7)\n");
		return 0;
	case MOD_UNLOAD:
		return EBUSY;	/* worker thread never exits */
	default:
		return EOPNOTSUPP;
	}
}

static moduledata_t dfrig2489_mod = { "dfrig2489", dfrig2489_modevent, 0 };

static struct mod_metadata _mod_metadata_dfrig2489 = {
	MDT_STRUCT_VERSION, MDT_MODULE, &dfrig2489_mod, "dfrig2489"
};
DATA_SET(modmetadata_set, _mod_metadata_dfrig2489);
SYSINIT(dfrig2489module, SI_SUB_DRIVERS, SI_ORDER_MIDDLE,
	module_register_init, &dfrig2489_mod);